use core::{cell::SyncUnsafeCell, cmp, fmt::Debug, ops::Range}; use crate::{ DYNAMIC_PROC_INFO, RtTcb, StaticProcInfo, arch::*, auxv_defs::*, protocol::{ProcCall, ThreadCall}, read_proc_meta, sys::{open, proc_call, thread_call}, }; use alloc::{boxed::Box, vec}; use goblin::elf::header::ET_DYN; //TODO: allow use of either 32-bit or 64-bit programs #[cfg(target_pointer_width = "32")] use goblin::elf32::{ header::Header, program_header::program_header32::{PF_R, PF_W, PF_X, PT_INTERP, PT_LOAD, ProgramHeader}, }; #[cfg(target_pointer_width = "64")] use goblin::elf64::{ header::Header, program_header::program_header64::{PF_R, PF_W, PF_X, PT_INTERP, PT_LOAD, ProgramHeader}, }; use syscall::{ CallFlags, F_GETFD, GrantDesc, GrantFlags, MAP_FIXED_NOREPLACE, MAP_SHARED, Map, O_CLOEXEC, PAGE_SIZE, PROT_EXEC, PROT_READ, PROT_WRITE, SetSighandlerData, error::*, flag::{MapFlags, SEEK_SET}, }; pub enum FexecResult { Interp { path: Box<[u8]>, interp_override: InterpOverride, }, } pub struct InterpOverride { phdrs_vaddr: usize, at_entry: usize, at_phnum: usize, at_phent: usize, name: Box<[u8]>, min_mmap_addr: usize, grants_fd: usize, } pub struct ExtraInfo<'a> { pub cwd: Option<&'a [u8]>, // POSIX states that while sigactions are reset, ignored sigactions will remain ignored. pub sigignmask: u64, // POSIX also states that the sigprocmask must be preserved across execs. pub sigprocmask: u64, /// File mode creation mask (POSIX) pub umask: u32, /// Thread handle pub thr_fd: usize, /// Process handle pub proc_fd: usize, /// Namespace handle pub ns_fd: Option, /// CWD handle pub cwd_fd: Option, } pub fn fexec_impl( image_file: FdGuardUpper, thread_fd: &FdGuardUpper, proc_fd: &FdGuardUpper, path: &[u8], args: &[&[u8]], envs: &[&[u8]], extrainfo: &ExtraInfo, interp_override: Option, ) -> Result { // Here, we do the minimum part of loading an application, which is what the kernel used to do. // We load the executable into memory (albeit at different offsets in this executable), fix // some misalignments, and then switch address space. let mut header_bytes = [0_u8; size_of::
()]; pread_all(&image_file, 0, &mut header_bytes)?; let header = Header::from_bytes(&header_bytes); let grants_fd = if let Some(interp) = interp_override.as_ref() { FdGuard::new(interp.grants_fd).to_upper()? } else { let current_addrspace_fd = thread_fd.dup(b"addrspace")?; current_addrspace_fd.dup(b"empty")?.to_upper()? }; // Never allow more than 1 MiB of program headers. const MAX_PH_SIZE: usize = 1024 * 1024; let phentsize = u64::from(header.e_phentsize) as usize; let phnum = u64::from(header.e_phnum) as usize; let pheaders_size = phentsize .saturating_mul(phnum) .saturating_add(size_of::
()); if pheaders_size > MAX_PH_SIZE { return Err(Error::new(E2BIG)); } let mut phs_raw = vec![0_u8; pheaders_size]; phs_raw[..size_of::
()].copy_from_slice(&header_bytes); let phs = &mut phs_raw[size_of::
()..]; // TODO: Remove clone, but this would require more as_refs and as_muts let mut min_mmap_addr = interp_override .as_ref() .map(|interp| interp.min_mmap_addr) .unwrap_or(PAGE_SIZE); let mut update_min_mmap_addr = |addr: usize, size: usize| { min_mmap_addr = cmp::max(min_mmap_addr, (addr + size).next_multiple_of(PAGE_SIZE)); }; pread_all(&image_file, u64::from(header.e_phoff), phs).map_err(|_| Error::new(EIO))?; let mut span: Option> = None; for ph_idx in 0..phnum { let ph_bytes = &phs[ph_idx * phentsize..(ph_idx + 1) * phentsize]; let segment: &ProgramHeader = plain::from_bytes(ph_bytes).map_err(|_| Error::new(EINVAL))?; if segment.p_type != PT_LOAD { continue; } let voff = segment.p_vaddr as usize % PAGE_SIZE; let vaddr = segment.p_vaddr as usize - voff; let vsize = (segment.p_memsz as usize + voff).next_multiple_of(segment.p_align as usize); let b = vaddr..vaddr + vsize; span = Some(if let Some(a) = span { a.start.min(b.start)..a.end.max(b.end) } else { b }); } let span = span.expect("ELF executables must contain at least one `PT_LOAD` segment"); let span_size = (span.end - span.start).next_multiple_of(PAGE_SIZE); let base_addr = if header.e_type == ET_DYN { // PIE let addr = mmap_anon_remote(&grants_fd, 0, 0, span_size, MapFlags::PROT_NONE)?; update_min_mmap_addr(addr, span_size); addr } else { mmap_anon_remote( &grants_fd, 0, span.start, span_size, MapFlags::MAP_FIXED_NOREPLACE, )?; update_min_mmap_addr(span.start, span_size); 0 }; let mut phdrs_vaddr = 0; let mut interpreter = None; for ph_idx in 0..phnum { let ph_bytes = &phs[ph_idx * phentsize..(ph_idx + 1) * phentsize]; let segment: &ProgramHeader = plain::from_bytes(ph_bytes).map_err(|_| Error::new(EINVAL))?; let mut flags = if segment.p_flags & PF_R == PF_R { syscall::PROT_READ } else { syscall::PROT_NONE }; // W ^ X. If it is executable, do not allow it to be writable, even if requested if segment.p_flags & PF_X == PF_X { flags |= syscall::PROT_EXEC; } else if segment.p_flags & PF_W == PF_W { flags |= syscall::PROT_WRITE; } match segment.p_type { // PT_INTERP must come before any PT_LOAD, so we don't have to iterate twice. PT_INTERP => { let mut interp = vec![0_u8; segment.p_filesz as usize]; pread_all(&image_file, u64::from(segment.p_offset), &mut interp)?; interpreter = Some(interp.into_boxed_slice()); } PT_LOAD => { let voff = segment.p_vaddr as usize % PAGE_SIZE; let vaddr = segment.p_vaddr as usize - voff; let filesz = segment.p_filesz as usize; let total_page_count = (segment.p_memsz as usize + voff).div_ceil(PAGE_SIZE); // The case where segments overlap so that they share one page, is not handled. // TODO: Should it be? if segment.p_filesz > segment.p_memsz { return Err(Error::new(ENOEXEC)); } mmap_anon_remote( &grants_fd, 0, base_addr + vaddr, total_page_count * PAGE_SIZE, flags | MapFlags::MAP_FIXED, )?; if segment.p_offset <= header.e_phoff && header.e_phoff < segment.p_offset + segment.p_filesz { phdrs_vaddr = (header.e_phoff - segment.p_offset + segment.p_vaddr) as usize + base_addr; } // TODO: Attempt to mmap with MAP_PRIVATE directly from the image file instead. if filesz > 0 { let (_guard, dst_memory) = unsafe { MmapGuard::map_mut_anywhere( &grants_fd, base_addr + vaddr, // offset (voff + filesz).next_multiple_of(PAGE_SIZE), // size )? }; pread_all( &image_file, u64::from(segment.p_offset), &mut dst_memory[voff..voff + filesz], )?; } } _ => continue, } } if let Some(interpreter_path) = interpreter { return Ok(FexecResult::Interp { path: interpreter_path, interp_override: InterpOverride { at_entry: base_addr + header.e_entry as usize, at_phnum: phnum, at_phent: phentsize, phdrs_vaddr, name: path.into(), min_mmap_addr, grants_fd: grants_fd.take(), }, }); } mmap_anon_remote( &grants_fd, 0, STACK_TOP - STACK_SIZE, STACK_SIZE, MapFlags::PROT_READ | MapFlags::PROT_WRITE | MapFlags::MAP_FIXED_NOREPLACE, )?; let mut sp = STACK_TOP; let mut stack_page = Option::::None; let mut push = |word: usize| { let old_page_no = sp / PAGE_SIZE; sp -= size_of::(); let new_page_no = sp / PAGE_SIZE; let new_page_off = sp % PAGE_SIZE; let page = if let Some(ref mut page) = stack_page && old_page_no == new_page_no { page } else if let Some(ref mut stack_page) = stack_page { stack_page.remap(new_page_no * PAGE_SIZE, PROT_READ | PROT_WRITE)?; stack_page } else { let new = MmapGuard::map( &grants_fd, &Map { offset: new_page_no * PAGE_SIZE, size: PAGE_SIZE, flags: PROT_READ | PROT_WRITE, address: 0, // let kernel decide }, )?; stack_page.insert(new) }; unsafe { page.as_mut_ptr_slice() .as_mut_ptr() .add(new_page_off) .cast::() .write(word); } Ok(()) }; push(0)?; push(AT_NULL)?; if let Some(ref r#override) = interp_override { push(r#override.at_entry)?; push(AT_ENTRY)?; push(base_addr)?; push(AT_BASE)?; push(r#override.phdrs_vaddr)?; push(AT_PHDR)?; } else { push(base_addr + header.e_entry as usize)?; push(AT_ENTRY)?; push(phdrs_vaddr)?; push(AT_PHDR)?; } push( interp_override .as_ref() .map_or(header.e_phnum as usize, |o| o.at_phnum), )?; push(AT_PHNUM)?; push( interp_override .as_ref() .map_or(header.e_phentsize as usize, |o| o.at_phent), )?; push(AT_PHENT)?; let total_args_envs_auxvpointee_size = args.iter().map(|arg| arg.len() + 1).sum::() + envs.iter().map(|env| env.len() + 1).sum::() + extrainfo.cwd.map_or(0, |s| s.len() + 1); let args_envs_size_aligned = total_args_envs_auxvpointee_size.next_multiple_of(PAGE_SIZE); let target_args_env_address = mmap_anon_remote( &grants_fd, 0, 0, args_envs_size_aligned, MapFlags::PROT_READ | MapFlags::PROT_WRITE, )?; update_min_mmap_addr(target_args_env_address, args_envs_size_aligned); let mut offset = 0; let mut argc = 0; { let mut append = |source_slice: &[u8]| { // TODO let address = target_args_env_address + offset; if !source_slice.is_empty() { let containing_page = address.div_floor(PAGE_SIZE) * PAGE_SIZE; let displacement = address - containing_page; let size = source_slice.len() + displacement; let aligned_size = size.next_multiple_of(PAGE_SIZE); let (_guard, memory) = unsafe { MmapGuard::map_mut_anywhere(&grants_fd, containing_page, aligned_size)? }; memory[displacement..][..source_slice.len()].copy_from_slice(source_slice); } offset += source_slice.len() + 1; Ok(address) }; if let Some(cwd) = extrainfo.cwd { push(append(cwd)?)?; push(AT_REDOX_INITIAL_CWD_PTR)?; push(cwd.len())?; push(AT_REDOX_INITIAL_CWD_LEN)?; } #[cfg(target_pointer_width = "32")] { push((extrainfo.sigignmask >> 32) as usize)?; push(AT_REDOX_INHERITED_SIGIGNMASK_HI)?; } push(extrainfo.sigignmask as usize)?; push(AT_REDOX_INHERITED_SIGIGNMASK)?; #[cfg(target_pointer_width = "32")] { push((extrainfo.sigprocmask >> 32) as usize)?; push(AT_REDOX_INHERITED_SIGPROCMASK_HI)?; } push(extrainfo.sigprocmask as usize)?; push(AT_REDOX_INHERITED_SIGPROCMASK)?; push(extrainfo.umask as usize)?; push(AT_REDOX_UMASK)?; push(extrainfo.thr_fd as usize)?; push(AT_REDOX_THR_FD)?; push(extrainfo.proc_fd as usize)?; push(AT_REDOX_PROC_FD)?; push(extrainfo.ns_fd.unwrap_or(usize::MAX))?; push(AT_REDOX_NS_FD)?; push(extrainfo.cwd_fd.unwrap_or(usize::MAX))?; push(AT_REDOX_CWD_FD)?; push(0)?; for env in envs.iter().rev() { push(append(env)?)?; } push(0)?; for arg in args.iter().rev() { push(append(arg)?)?; argc += 1; } } push(argc)?; if let Ok(sighandler_fd) = thread_fd.dup(b"sighandler") { let _ = sighandler_fd.write(&SetSighandlerData { user_handler: 0, excp_handler: 0, thread_control_addr: 0, proc_control_addr: 0, }); // TODO: sync with procmgr } // TODO: Restore old name if exec failed? { let mut buf = [0; 32]; let new_name = interp_override.as_ref().map_or(path, |o| &o.name); let len = new_name.len().min(32); buf[..len].copy_from_slice(&new_name[..len]); // XXX: takes &mut [] since it can mutate, but we could unsafe{}ly pass it directly // otherwise let _ = proc_call( proc_fd.as_raw_fd(), &mut buf, CallFlags::empty(), &[ProcCall::Rename as u64], ); } // TODO: Error handling let _ = proc_call( proc_fd.as_raw_fd(), &mut [], CallFlags::empty(), &[ProcCall::DisableSetpgid as u64], ); if interp_override.is_some() { let mmap_min_fd = grants_fd.dup(b"mmap-min-addr")?; let _ = mmap_min_fd.write(&usize::to_ne_bytes(min_mmap_addr)); } let addrspace_selection_fd = thread_fd.dup(b"current-addrspace")?.to_upper()?; let _ = addrspace_selection_fd.write(&create_set_addr_space_buf( grants_fd.as_raw_fd(), base_addr + header.e_entry as usize, sp, )); // Close all O_CLOEXEC file descriptors. TODO: close_range? { // NOTE: This approach of implementing O_CLOEXEC will not work in multithreaded // scenarios. While execve() is undefined according to POSIX if there exist sibling // threads, it could still be allowed by keeping certain file descriptors and instead // set the active file table. let files_fd = syscall::dup(thread_fd.as_raw_fd(), b"filetable-binary")?; let mut files_reader = FileBufReader::from_fd(files_fd); loop { let fd = match files_reader.read_le_u64()? { None => break, Some(fd) => fd, }; let fd = usize::try_from(fd).unwrap(); if fd == addrspace_selection_fd.as_raw_fd() || fd == files_fd { continue; // Will be closed below } let flags = syscall::fcntl(fd, F_GETFD, 0)?; if flags & O_CLOEXEC == O_CLOEXEC { let _ = syscall::close(fd); } } } unsafe { deactivate_tcb(&thread_fd)?; } // Dropping this FD will cause the address space switch. drop(addrspace_selection_fd); unreachable!(); } fn write_usizes(fd: &FdGuardUpper, usizes: [usize; N]) -> Result { fd.write(unsafe { plain::as_bytes(&usizes) }) } pub fn mmap_remote( addrspace_fd: &FdGuardUpper, fd: &FdGuardUpper, offset: usize, dst_addr: usize, len: usize, flags: MapFlags, ) -> Result { write_usizes( addrspace_fd, [ // op syscall::flag::ADDRSPACE_OP_MMAP, // fd fd.as_raw_fd(), // "offset" offset, // address dst_addr, // size len, // flags flags.bits(), ], ) } pub fn mmap_anon_remote( addrspace_fd: &FdGuardUpper, offset: usize, dst_addr: usize, len: usize, flags: MapFlags, ) -> Result { write_usizes( addrspace_fd, [ // op syscall::flag::ADDRSPACE_OP_MMAP, // fd !0, // "offset" offset, // address dst_addr, // size len, // flags flags.bits(), ], ) } pub fn mprotect_remote( addrspace_fd: &FdGuardUpper, addr: usize, len: usize, flags: MapFlags, ) -> Result<()> { write_usizes( addrspace_fd, [ // op syscall::flag::ADDRSPACE_OP_MPROTECT, // address addr, // size len, // flags flags.bits(), ], )?; Ok(()) } pub fn munmap_remote(addrspace_fd: &FdGuardUpper, addr: usize, len: usize) -> Result<()> { write_usizes( addrspace_fd, [ // op syscall::flag::ADDRSPACE_OP_MUNMAP, // address addr, // size len, ], )?; Ok(()) } pub fn munmap_transfer( src: &FdGuardUpper, dst: &FdGuardUpper, src_addr: usize, dst_addr: usize, len: usize, flags: MapFlags, ) -> Result<()> { write_usizes( dst, [ // op syscall::flag::ADDRSPACE_OP_TRANSFER, // fd src.as_raw_fd(), // "offset" (source address) src_addr, // address dst_addr, // size len, // flags (flags | MapFlags::MAP_FIXED_NOREPLACE).bits(), ], )?; Ok(()) } fn pread_all(fd: &FdGuardUpper, offset: u64, buf: &mut [u8]) -> Result<()> { fd.lseek(offset as isize, SEEK_SET)?; let mut total_bytes_read = 0; while total_bytes_read < buf.len() { total_bytes_read += match fd.read(&mut buf[total_bytes_read..])? { 0 => return Err(Error::new(ENOEXEC)), bytes_read => bytes_read, } } Ok(()) } pub struct MmapGuard<'a> { fd: &'a FdGuardUpper, base: usize, size: usize, } impl<'a> MmapGuard<'a> { pub fn map(fd: &'a FdGuardUpper, map: &Map) -> Result { let base = unsafe { syscall::fmap(fd.as_raw_fd(), map)? }; Ok(Self { fd, size: map.size, base, }) } pub fn remap(&mut self, offset: usize, mut flags: MapFlags) -> Result<()> { flags.remove(MapFlags::MAP_FIXED_NOREPLACE); flags.insert(MapFlags::MAP_FIXED); let _new_base = unsafe { syscall::fmap( self.fd.as_raw_fd(), &Map { offset, size: self.size, flags, address: self.base, }, )? }; Ok(()) } pub unsafe fn map_mut_anywhere( fd: &'a FdGuardUpper, offset: usize, size: usize, ) -> Result<(Self, &'a mut [u8])> { let mut this = Self::map( fd, &Map { size, offset, address: 0, flags: PROT_READ | PROT_WRITE, }, )?; let slice = unsafe { &mut *this.as_mut_ptr_slice() }; Ok((this, slice)) } pub fn addr(&self) -> usize { self.base } pub fn len(&self) -> usize { self.size } pub fn as_mut_ptr_slice(&mut self) -> *mut [u8] { core::ptr::slice_from_raw_parts_mut(self.base as *mut u8, self.size) } pub fn take(mut self) { self.size = 0; } } impl<'a> Drop for MmapGuard<'a> { fn drop(&mut self) { if self.size != 0 { let _ = unsafe { syscall::funmap(self.base, self.size) }; } } } struct FileBufReader { fd: usize, buf: [u8; 8192], pos: usize, cap: usize, } impl FileBufReader { pub fn from_fd(fd: usize) -> FileBufReader { FileBufReader { fd, buf: [0; 8192], pos: 0, cap: 0, } } } impl FileBufReader { fn read_le_u64(&mut self) -> syscall::Result> { if self.pos >= self.cap { debug_assert!(self.pos == self.cap); self.cap = crate::sys::posix_read(self.fd, &mut self.buf)?; self.pos = 0; } if self.cap == 0 { return Ok(None); } if self.cap - self.pos < 8 { unreachable!(); } let num = u64::from_le_bytes(self.buf[self.pos..self.pos + 8].try_into().unwrap()); self.pos += 8; Ok(Some(num)) } } #[repr(transparent)] pub struct FdGuard { fd: usize, } pub type FdGuardUpper = FdGuard; impl FdGuard { #[inline] pub fn new(fd: usize) -> Self { Self { fd } } #[inline] pub fn open>(path: T, flags: usize) -> Result { open(path, flags).map(Self::new) } #[inline] pub fn to_upper(self) -> Result { // Move to upper table if necessary let fd = if self.fd & syscall::UPPER_FDTBL_TAG == 0 { //TODO: use F_DUPFD_CLOEXEC? let fd = syscall::fcntl(self.fd, syscall::F_DUPFD, syscall::UPPER_FDTBL_TAG)?; drop(self); fd } else { self.take() }; Ok(FdGuard:: { fd }) } // Not implemented for UPPER to prevent misuse #[inline] pub fn as_c_fd(&self) -> Option { i32::try_from(self.fd).ok() } } impl FdGuard { #[inline] pub fn openat>( &self, path: T, flags: usize, fcntl_flags: usize, ) -> Result> { syscall::openat(self.fd, path, flags, fcntl_flags).map(FdGuard::new) } #[inline] pub fn dup(&self, buf: &[u8]) -> Result> { syscall::dup(self.fd, buf).map(FdGuard::new) } #[inline] pub fn fcntl(&self, cmd: usize, arg: usize) -> Result { syscall::fcntl(self.fd, cmd, arg) } #[inline] pub fn fstat(&self, stat: &mut syscall::Stat) -> Result { syscall::fstat(self.fd, stat) } #[inline] pub fn lseek(&self, offset: isize, whence: usize) -> Result { syscall::lseek(self.fd, offset, whence) } #[inline] pub fn read(&self, buf: &mut [u8]) -> Result { syscall::read(self.fd, buf) } #[inline] pub fn write(&self, buf: &[u8]) -> Result { syscall::write(self.fd, buf) } #[inline] pub fn call_ro(&self, payload: &mut [u8], flags: CallFlags, metadata: &[u64]) -> Result { syscall::call_ro(self.fd, payload, flags, metadata) } #[inline] pub fn call_wo(&self, payload: &[u8], flags: CallFlags, metadata: &[u64]) -> Result { syscall::call_wo(self.fd, payload, flags, metadata) } #[inline] pub fn call_rw(&self, payload: &mut [u8], flags: CallFlags, metadata: &[u64]) -> Result { syscall::call_rw(self.fd, payload, flags, metadata) } #[inline] pub fn as_raw_fd(&self) -> usize { self.fd } #[inline] pub fn take(self) -> usize { let fd = self.fd; core::mem::forget(self); fd } } impl Drop for FdGuard { #[inline] fn drop(&mut self) { let _ = syscall::close(self.fd); } } impl Debug for FdGuard { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "[fd {}]", self.fd) } } impl Debug for FdGuardUpper { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "[fd upper {}]", self.fd & !syscall::UPPER_FDTBL_TAG) } } pub fn create_set_addr_space_buf( space: usize, ip: usize, sp: usize, ) -> [u8; size_of::() * 3] { let mut buf = [0u8; size_of::() * 3]; buf.copy_from_slice([space, sp, ip].map(usize::to_ne_bytes).as_flattened()); buf } pub fn create_set_addr_space_buf_for_fork( space: usize, ip: usize, sp: usize, arg1: usize, ) -> [u8; size_of::() * 4] { let mut buf = [0u8; size_of::() * 4]; buf.copy_from_slice([space, sp, ip, arg1].map(usize::to_ne_bytes).as_flattened()); buf } /// Spawns a new context which will not share the same address space as the current one. File /// descriptors from other schemes are reobtained with `dup`, and grants referencing such file /// descriptors are reobtained through `fmap`. Other mappings are kept but duplicated using CoW. pub fn fork_impl(args: &ForkArgs<'_>) -> Result { let old_mask = crate::signal::get_sigmask()?; let pid = unsafe { Error::demux(__relibc_internal_fork_wrapper( args as *const ForkArgs as usize, ))? }; if pid == 0 { crate::signal::set_sigmask(Some(old_mask), None)?; } Ok(pid) } pub enum ForkArgs<'a> { Init { this_thr_fd: &'a FdGuardUpper, auth: &'a FdGuard, }, Managed, } pub fn fork_inner(initial_rsp: *mut usize, args: &ForkArgs) -> Result { let (cur_filetable_fd, new_proc_fd, new_thr_fd, new_pid); { let cur_thr_fd = match args { ForkArgs::Init { this_thr_fd, .. } => this_thr_fd, ForkArgs::Managed => RtTcb::current().thread_fd(), }; let NewChildProc { proc_fd, thr_fd, pid, } = new_child_process(args)?; new_proc_fd = proc_fd; new_thr_fd = thr_fd; new_pid = pid; // Copy existing files into new file table, but do not reuse the same file table (i.e. new // parent FDs will not show up for the child). let scratchpad = { cur_filetable_fd = cur_thr_fd.dup(b"filetable")?; // This must be done before the address space is copied. let proc_fd = new_proc_fd.as_ref().map_or(usize::MAX, |p| p.as_raw_fd()); //let _ = syscall::write(1, alloc::format!("FDTBL{}PROC{}THR{}\n", *cur_filetable_fd, proc_fd, *new_thr_fd).as_bytes()); ForkScratchpad { cur_filetable_fd: cur_filetable_fd.as_raw_fd(), new_proc_fd: proc_fd, new_thr_fd: new_thr_fd.as_raw_fd(), } }; #[cfg(any( target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64" ))] let arg1 = { let scratchpad_ptr: *const ForkScratchpad = &scratchpad; scratchpad_ptr as usize }; #[cfg(target_arch = "x86")] unsafe { let scratchpad_ptr = initial_rsp as *mut ForkScratchpad; scratchpad_ptr.write(scratchpad); } // CoW-duplicate address space. { let new_addr_space_sel_fd = new_thr_fd.dup(b"current-addrspace")?; let cur_addr_space_fd = cur_thr_fd.dup(b"addrspace")?.to_upper()?; let new_addr_space_fd = cur_addr_space_fd.dup(b"exclusive")?.to_upper()?; let mut grant_desc_buf = [GrantDesc::default(); 16]; loop { let bytes_read = { let buf = unsafe { core::slice::from_raw_parts_mut( grant_desc_buf.as_mut_ptr().cast(), grant_desc_buf.len() * size_of::(), ) }; cur_addr_space_fd.read(buf)? }; if bytes_read == 0 { break; } let grants = &grant_desc_buf[..bytes_read / size_of::()]; for grant in grants { if !grant.flags.contains(GrantFlags::GRANT_SCHEME) || !grant.flags.contains(GrantFlags::GRANT_SHARED) { continue; } let buf; // TODO: write! using some #![no_std] Cursor type (tracking the length)? #[cfg(target_pointer_width = "64")] { //buf = *b"grant-fd-AAAABBBBCCCCDDDD"; //write!(&mut buf, "grant-fd-{:>016x}", grant.base).unwrap(); buf = alloc::format!("grant-fd-{:>016x}", grant.base).into_bytes(); } #[cfg(target_pointer_width = "32")] { //buf = *b"grant-fd-AAAABBBB"; //write!(&mut buf[..], "grant-fd-{:>08x}", grant.base).unwrap(); buf = alloc::format!("grant-fd-{:>08x}", grant.base).into_bytes(); } let grant_fd = cur_addr_space_fd.dup(&buf)?.to_upper()?; let mut flags = MAP_SHARED | MAP_FIXED_NOREPLACE; flags.set(PROT_READ, grant.flags.contains(GrantFlags::GRANT_READ)); flags.set(PROT_WRITE, grant.flags.contains(GrantFlags::GRANT_WRITE)); flags.set(PROT_EXEC, grant.flags.contains(GrantFlags::GRANT_EXEC)); mmap_remote( &new_addr_space_fd, &grant_fd, grant.offset as usize, grant.base, grant.size, flags, )?; } } #[cfg(any( target_arch = "x86_64", target_arch = "aarch64", target_arch = "riscv64" ))] let buf = create_set_addr_space_buf_for_fork( new_addr_space_fd.as_raw_fd(), __relibc_internal_fork_ret as usize, initial_rsp as usize, arg1, ); #[cfg(target_arch = "x86")] let buf = create_set_addr_space_buf( new_addr_space_fd.as_raw_fd(), __relibc_internal_fork_ret as usize, initial_rsp as usize, ); new_addr_space_sel_fd.write(&buf)?; } { // Reuse the same sigaltstack and signal entry (all memory will be re-mapped CoW later). // // Do this after the address space is cloned, since the kernel will get a shared // reference to the TCB and whatever pages stores the signal proc control struct. { let new_sighandler_fd = new_thr_fd.dup(b"sighandler")?; new_sighandler_fd.write(&crate::signal::current_setsighandler_struct())?; } if let Some(ref proc_fd) = new_proc_fd { proc_call( proc_fd.as_raw_fd(), &mut [], CallFlags::empty(), &[ProcCall::SyncSigPctl as u64], )?; thread_call( new_thr_fd.as_raw_fd(), &mut [], CallFlags::empty(), &[ThreadCall::SyncSigTctl as u64], )?; } } { // Copy environment registers. let cur_env_regs_fd = cur_thr_fd.dup(b"regs/env")?; let new_env_regs_fd = new_thr_fd.dup(b"regs/env")?; let mut env_regs = syscall::EnvRegisters::default(); cur_env_regs_fd.read(&mut env_regs)?; new_env_regs_fd.write(&env_regs)?; } } // Copy the file table. We do this last to ensure that all previously used file descriptors are // closed. The only exception -- the filetable selection fd and the current filetable fd -- // will be closed by the child process. { // TODO: Use file descriptor forwarding or something similar to avoid copying the file // table in the kernel. let new_filetable_fd = cur_filetable_fd.dup(b"copy")?; let new_filetable_sel_fd = new_thr_fd.dup(b"current-filetable")?; new_filetable_sel_fd.write(&usize::to_ne_bytes(new_filetable_fd.as_raw_fd()))?; } let start_fd = new_thr_fd.dup(b"start")?; start_fd.write(&[0])?; Ok(new_pid) } pub struct NewChildProc { proc_fd: Option, thr_fd: FdGuardUpper, pid: usize, } pub fn new_child_process(args: &ForkArgs<'_>) -> Result { match *args { ForkArgs::Managed => { let proc_info = crate::static_proc_info(); let this_proc_fd = proc_info .proc_fd .as_ref() .expect("cannot use ForkArgs::Managed without an existing proc info"); let child_proc_fd = this_proc_fd.dup(b"fork")?.to_upper()?; let only_thread_fd = child_proc_fd.dup(b"thread-0")?.to_upper()?; let meta = read_proc_meta(&child_proc_fd)?; Ok(NewChildProc { proc_fd: Some(child_proc_fd), thr_fd: only_thread_fd, pid: meta.pid as usize, }) } #[cfg(feature = "proc")] ForkArgs::Init { .. } => unreachable!(), #[cfg(not(feature = "proc"))] ForkArgs::Init { this_thr_fd, auth } => { let thr_fd = auth.dup(b"new-context")?.to_upper()?; let buf = syscall::ProcSchemeAttrs { pid: 0, euid: 0, egid: 0, ens: 1, debug_name: { let mut buf = [0; 32]; let src = b"[init]"; buf[..src.len()].copy_from_slice(src); buf }, }; let attr_fd = thr_fd.dup(alloc::format!("auth-{}-attrs", auth.as_raw_fd()).as_bytes())?; attr_fd.write(&buf)?; Ok(NewChildProc { thr_fd, pid: 1, // dummy fd to distinguish child from parent proc_fd: None, }) } } } pub unsafe fn make_init(proc_cap: usize) -> (&'static FdGuardUpper, &'static FdGuardUpper) { let proc_fd = FdGuard::new( syscall::openat(proc_cap, "init", syscall::O_CLOEXEC, 0).expect("failed to create init"), ) .to_upper() .unwrap(); syscall::sendfd( proc_fd.as_raw_fd(), RtTcb::current().thread_fd().dup(&[]).unwrap().take(), 0, 0, ) .expect("failed to assign current thread to init process"); let managed_thr_fd = proc_fd .dup(b"thread-0") .expect("failed to get managed thread for init") .to_upper() .unwrap(); let managed_thr_fd = unsafe { (*RtTcb::current().thr_fd.get()).insert(managed_thr_fd) }; unsafe { STATIC_PROC_INFO.get().write(crate::StaticProcInfo { pid: 1, proc_fd: Some(proc_fd), }) }; *DYNAMIC_PROC_INFO.lock() = crate::DynamicProcInfo { pgid: 1, ruid: 0, euid: 0, suid: 0, rgid: 0, egid: 0, sgid: 0, ns_fd: None, }; ( unsafe { (*STATIC_PROC_INFO.get()).proc_fd.as_ref().unwrap() }, managed_thr_fd, ) } pub(crate) static STATIC_PROC_INFO: SyncUnsafeCell = SyncUnsafeCell::new(StaticProcInfo { pid: 0, proc_fd: None, });