diff --git a/Cargo.lock b/Cargo.lock index be23923f3b..6901c00bf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -234,8 +234,8 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7" [[package]] name = "redox_syscall" -version = "0.5.12" -source = "git+https://gitlab.redox-os.org/redox-os/syscall.git?branch=master#fe32c6b89dae51e609d5c53880dec1834ec9bde0" +version = "0.5.17" +source = "git+https://gitlab.redox-os.org/redox-os/syscall.git?branch=master#a9880ccf50d093e662f3b31c1aa13f0951cc2b4d" dependencies = [ "bitflags 2.9.0", ] diff --git a/src/context/context.rs b/src/context/context.rs index b7ba80e0f5..a3126b4b84 100644 --- a/src/context/context.rs +++ b/src/context/context.rs @@ -1,4 +1,4 @@ -use alloc::{sync::Arc, vec::Vec}; +use alloc::{collections::BTreeSet, sync::Arc, vec::Vec}; use arrayvec::ArrayString; use core::{ mem::{self, size_of}, @@ -6,6 +6,7 @@ use core::{ sync::atomic::{AtomicU32, Ordering}, }; use spin::RwLock; +use syscall::UPPER_FDTBL_TAG; use syscall::{SigProcControl, Sigcontrol}; #[cfg(feature = "sys_stat")] @@ -19,10 +20,10 @@ use crate::{ memory::{allocate_p2frame, deallocate_p2frame, Enomem, Frame, RaiiFrame}, paging::{RmmA, RmmArch}, percpu::PercpuBlock, - scheme::{CallerCtx, FileHandle, SchemeNamespace}, + scheme::{CallerCtx, FileHandle, SchemeId, SchemeNamespace}, }; -use crate::syscall::error::{Error, Result, EAGAIN, ESRCH}; +use crate::syscall::error::{Error, Result, EAGAIN, EBADF, EEXIST, EINVAL, EMFILE, ESRCH}; use super::{ empty_cr3, @@ -121,7 +122,7 @@ pub struct Context { /// The name of the context pub name: ArrayString, /// The open files in the scheme - pub files: Arc>>>, + pub files: Arc>, /// All contexts except kmain will primarily live in userspace, and enter the kernel only when /// interrupts or syscalls occur. This flag is set for all contexts but kmain. pub userspace: bool, @@ -177,7 +178,7 @@ impl Context { kstack: None, addr_space: None, name: ArrayString::new(), - files: Arc::new(RwLock::new(Vec::new())), + files: Arc::new(RwLock::new(FdTbl::new())), userspace: false, fmap_ret: None, being_sigkilled: false, @@ -255,64 +256,61 @@ impl Context { /// Add a file to the lowest available slot greater than or equal to min. /// Return the file descriptor number or None if no slot was found pub fn add_file_min(&self, file: FileDescriptor, min: usize) -> Option { - let mut files = self.files.write(); - for (i, file_option) in files.iter_mut().enumerate() { - if file_option.is_none() && i >= min { - *file_option = Some(file); - return Some(FileHandle::from(i)); - } - } - let len = files.len(); - if len < super::CONTEXT_MAX_FILES { - if len >= min { - files.push(Some(file)); - Some(FileHandle::from(len)) - } else { - drop(files); - self.insert_file(FileHandle::from(min), file) - } - } else { - None - } + self.files.write().add_file_min(file, min) + } + + /// Bulk-add multiple files to the POSIX file table + pub fn bulk_add_files_posix( + &self, + files_to_add: Vec, + ) -> Option> { + self.files.write().bulk_add_files_posix(files_to_add) + } + + /// Bulk-insert multiple files into to the upper file table contiguously + pub fn bulk_insert_files_upper( + &self, + files_to_insert: Vec, + ) -> Option> { + self.files.write().bulk_insert_files_upper(files_to_insert) + } + + /// Bulk-insert multiple files into to the upper file table manually + pub fn bulk_insert_files_upper_manual( + &self, + files_to_insert: Vec, + handles: &[FileHandle], + ) -> Result<()> { + self.files + .write() + .bulk_insert_files_upper_manual(files_to_insert, handles) } /// Get a file pub fn get_file(&self, i: FileHandle) -> Option { - let files = self.files.read(); - if i.get() < files.len() { - files[i.get()].clone() - } else { - None - } + self.files.read().get_file(i) + } + + /// Bulk get files + pub fn bulk_get_files(&self, handles: &[FileHandle]) -> Result> { + self.files.read().bulk_get_files(handles) } /// Insert a file with a specific handle number. This is used by dup2 /// Return the file descriptor number or None if the slot was not empty, or i was invalid pub fn insert_file(&self, i: FileHandle, file: FileDescriptor) -> Option { - let mut files = self.files.write(); - if i.get() >= super::CONTEXT_MAX_FILES { - return None; - } - if i.get() >= files.len() { - files.resize_with(i.get() + 1, || None); - } - if files[i.get()].is_none() { - files[i.get()] = Some(file); - Some(i) - } else { - None - } + self.files.write().insert_file(i, file) } /// Remove a file // TODO: adjust files vector to smaller size if possible pub fn remove_file(&self, i: FileHandle) -> Option { - let mut files = self.files.write(); - if i.get() < files.len() { - files[i.get()].take() - } else { - None - } + self.files.write().remove_file(i) + } + + /// Bulk remove files + pub fn bulk_remove_files(&self, handles: &[FileHandle]) -> Result> { + self.files.write().bulk_remove_files(handles) } pub fn is_current_context(&self) -> bool { @@ -541,3 +539,359 @@ impl core::fmt::Debug for Kstack { write!(f, "[kstack at {:?}]", self.base) } } + +#[derive(Clone, Debug, Default)] +pub struct FdTbl { + pub posix_fdtbl: Vec>, + pub upper_fdtbl: Vec>, + active_count: usize, +} + +impl FdTbl { + pub fn new() -> Self { + Self { + posix_fdtbl: Vec::new(), + upper_fdtbl: Vec::new(), + active_count: 0, + } + } + + fn strip_tags(index: usize) -> usize { + index & !UPPER_FDTBL_TAG + } + + fn select_fdtbl(&self, index: usize) -> (&Vec>, usize) { + if index & UPPER_FDTBL_TAG == 0 { + (&self.posix_fdtbl, index) + } else { + (&self.upper_fdtbl, Self::strip_tags(index)) + } + } + + fn select_fdtbl_mut(&mut self, index: usize) -> (&mut Vec>, usize) { + if index & UPPER_FDTBL_TAG == 0 { + (&mut self.posix_fdtbl, index) + } else { + (&mut self.upper_fdtbl, Self::strip_tags(index)) + } + } + + fn validate_handles(&self, handles: &[FileHandle]) -> Result<()> { + let mut checked_handles = BTreeSet::new(); + for i in handles { + let index = i.get(); + if Self::strip_tags(index) >= super::CONTEXT_MAX_FILES { + return Err(Error::new(EMFILE)); + } + if !checked_handles.insert(index) { + return Err(Error::new(EBADF)); // Duplicate handle + } + if !matches!(self.get(index), Some(Some(_))) { + return Err(Error::new(EBADF)); + } + } + + Ok(()) + } + + fn validate_free_slots(&self, handles: &[FileHandle]) -> Result<()> { + let mut checked_slots = BTreeSet::new(); + for i in handles { + let index = i.get(); + if Self::strip_tags(index) >= super::CONTEXT_MAX_FILES { + return Err(Error::new(EMFILE)); + } + if !checked_slots.insert(index) { + return Err(Error::new(EINVAL)); // Duplicate slots + } + if matches!(self.get(index), Some(Some(_))) { + return Err(Error::new(EEXIST)); + } + } + + Ok(()) + } + + pub fn add_file_min(&mut self, file: FileDescriptor, min: usize) -> Option { + if self.active_count >= super::CONTEXT_MAX_FILES { + return None; + } + + let tag = min & UPPER_FDTBL_TAG; + + let (fdtbl, min) = self.select_fdtbl_mut(min); + + // Find the first empty slot in the posix_fdtbl starting from `min`. + if let Some((pos, slot)) = fdtbl + .iter_mut() + .enumerate() + .skip(min) + .find(|(_, slot)| slot.is_none()) + { + *slot = Some(file); + self.active_count += 1; + return Some(FileHandle::from(pos | tag)); + }; + + let len = fdtbl.len(); + + // If no empty slot was found, we need to allocate a new slot. + if len >= min { + fdtbl.push(Some(file)); + self.active_count += 1; + Some(FileHandle::from(len | tag)) + } else { + self.insert_file(FileHandle::from(min | tag), file) + } + } + + fn bulk_add_files_posix( + &mut self, + files_to_add: Vec, + ) -> Option> { + let count = files_to_add.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let handles = self.find_free_posix_slots(count); + let max_index = handles[count - 1].get(); + if self.posix_fdtbl.len() <= max_index { + // Resize the posix_fdtbl to accommodate the new files. + self.posix_fdtbl.resize(max_index + 1, None); + } + + for (&handle, file) in handles.iter().zip(files_to_add) { + let index = handle.get(); + self.posix_fdtbl[index] = Some(file); + } + + self.active_count += count; + Some(handles) + } + + fn insert_file(&mut self, i: FileHandle, file: FileDescriptor) -> Option { + if self.active_count >= super::CONTEXT_MAX_FILES { + return None; + } + let index = i.get(); + let (fdtbl, real_index) = self.select_fdtbl_mut(index); + + if real_index >= super::CONTEXT_MAX_FILES { + return None; + } + + if real_index >= fdtbl.len() { + fdtbl.resize_with(real_index + 1, || None); + } + + if let Some(slot @ None) = fdtbl.get_mut(real_index) { + *slot = Some(file); + self.active_count += 1; + Some(i) + } else { + None + } + } + + fn bulk_insert_files_upper( + &mut self, + files_to_insert: Vec, + ) -> Option> { + let count = files_to_insert.len(); + if count == 0 { + return Some(Vec::new()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return None; + } + + let index = Self::strip_tags(self.find_free_upper_block(count).get()); + let mut handles = Vec::with_capacity(count); + for (i, file) in files_to_insert.into_iter().enumerate() { + let current_index = index + i; + self.upper_fdtbl[current_index] = Some(file); + handles.push(FileHandle::from(current_index | UPPER_FDTBL_TAG)); + } + + self.active_count += count; + Some(handles) + } + + fn bulk_insert_files_upper_manual( + &mut self, + files_to_insert: Vec, + handles: &[FileHandle], + ) -> Result<()> { + if handles.len() != files_to_insert.len() { + return Err(Error::new(EINVAL)); + } + let count = files_to_insert.len(); + if count == 0 { + return Ok(()); + } + if self.active_count + count > super::CONTEXT_MAX_FILES { + return Err(Error::new(EMFILE)); + } + self.validate_free_slots(handles)?; + + let max_index = handles + .iter() + .map(|h| Self::strip_tags(h.get())) + .max() + .unwrap_or(0); + if self.upper_fdtbl.len() <= max_index { + self.upper_fdtbl.resize_with(max_index + 1, || None); + } + for (file, &handle) in files_to_insert.into_iter().zip(handles) { + let index = Self::strip_tags(handle.get()); + self.upper_fdtbl[index] = Some(file); + } + + self.active_count += count; + Ok(()) + } + + pub fn get(&self, index: usize) -> Option<&Option> { + let (fdtbl, real_index) = self.select_fdtbl(index); + + fdtbl.get(real_index) + } + + pub fn get_mut(&mut self, index: usize) -> Option<&mut Option> { + let (fdtbl, real_index) = self.select_fdtbl_mut(index); + + fdtbl.get_mut(real_index) + } + + pub fn get_file(&self, i: FileHandle) -> Option { + self.get(i.get()).cloned().flatten() + } + + fn bulk_get_files(&self, handles: &[FileHandle]) -> Result> { + // Validate that all handles are valid before proceeding to avoid partial results. + self.validate_handles(handles)?; + + let files = handles + .iter() + .map(|&i| self.get_file(i).expect("File should exist")) + .collect(); + + Ok(files) + } + + // TODO: Faster, cleaner mechanism to get descriptor + // Find a file descriptor by scheme id and number. + pub fn find_by_scheme( + &self, + scheme_id: SchemeId, + scheme_number: usize, + ) -> Result { + self.iter() + .flatten() + .find(|&context_fd| { + let desc = context_fd.description.read(); + desc.scheme == scheme_id && desc.number == scheme_number + }) + .map(|fd| fd.clone()) + .ok_or(Error::new(EBADF)) + } + + fn remove_file(&mut self, i: FileHandle) -> Option { + let index = i.get(); + let (fdtbl, real_index) = self.select_fdtbl_mut(index); + + let removed_file_opt = fdtbl.get_mut(real_index).and_then(|opt| opt.take()); + if removed_file_opt.is_some() { + self.active_count -= 1; + } + + removed_file_opt + } + + fn bulk_remove_files(&mut self, handles: &[FileHandle]) -> Result> { + // Validate that all handles are valid before proceeding to avoid partial results. + self.validate_handles(handles)?; + + let files = handles + .iter() + .map(|&i| self.remove_file(i).expect("File should exist")) + .collect(); + + Ok(files) + } + + fn find_free_posix_slots(&self, count: usize) -> Vec { + let mut free_slots = Vec::with_capacity(count); + + for (i, slot) in self.posix_fdtbl.iter().enumerate() { + if slot.is_none() { + free_slots.push(FileHandle::from(i)); + if free_slots.len() == count { + return free_slots; + } + } + } + + let mut current_len = self.posix_fdtbl.len(); + while free_slots.len() < count { + free_slots.push(FileHandle::from(current_len)); + current_len += 1; + } + free_slots + } + + fn find_free_upper_block(&mut self, len: usize) -> FileHandle { + let mut start = 0; + let mut count = 0; + + for (i, file_opt) in self.upper_fdtbl.iter().enumerate() { + if file_opt.is_none() { + if count == 0 { + start = i; + } + count += 1; + if count == len { + break; + } + } else { + count = 0; + } + } + + if count < len { + if count == 0 { + start = self.upper_fdtbl.len(); + } + let needed = len - count; + self.upper_fdtbl + .resize(self.upper_fdtbl.len() + needed, None); + } + + FileHandle::from(start | UPPER_FDTBL_TAG) + } + + pub fn force_close_all(&mut self) { + for file_opt in self.iter_mut() { + if let Some(file) = file_opt.take() { + let _ = file.close(); + } + } + self.active_count = 0; + } +} + +impl FdTbl { + pub fn iter(&self) -> impl Iterator> { + self.posix_fdtbl.iter().chain(self.upper_fdtbl.iter()) + } + + pub fn iter_mut(&mut self) -> impl Iterator> { + self.posix_fdtbl + .iter_mut() + .chain(self.upper_fdtbl.iter_mut()) + } +} diff --git a/src/scheme/mod.rs b/src/scheme/mod.rs index d409deda72..21bd50a852 100644 --- a/src/scheme/mod.rs +++ b/src/scheme/mod.rs @@ -13,7 +13,7 @@ use core::{hash::BuildHasherDefault, sync::atomic::AtomicUsize}; use hashbrown::{hash_map::DefaultHashBuilder, HashMap}; use indexmap::IndexMap; use spin::{Once, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use syscall::{CallFlags, EventFlags, MunmapFlags, SendFdFlags}; +use syscall::{CallFlags, EventFlags, MunmapFlags}; use crate::{ context::{ @@ -468,15 +468,6 @@ pub trait KernelScheme: Send + Sync + 'static { Err(Error::new(EBADF)) } - fn ksendfd( - &self, - id: usize, - desc: Arc>, - flags: SendFdFlags, - arg: u64, - ) -> Result { - Err(Error::new(EOPNOTSUPP)) - } fn getdents( &self, id: usize, @@ -535,6 +526,25 @@ pub trait KernelScheme: Send + Sync + 'static { ) -> Result { Err(Error::new(EOPNOTSUPP)) } + fn kfdwrite( + &self, + id: usize, + descs: Vec>>, + flags: CallFlags, + args: u64, + metadata: &[u64], + ) -> Result { + Err(Error::new(EOPNOTSUPP)) + } + fn kfdread( + &self, + id: usize, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], + ) -> Result { + Err(Error::new(EOPNOTSUPP)) + } } #[derive(Debug)] diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index 8865dbd7e9..b59e2407a7 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -3,7 +3,7 @@ use crate::{ context::{ self, context::{HardBlockedReason, SignalState}, - file::{FileDescriptor, InternalFlags}, + file::InternalFlags, memory::{handle_notify_files, AddrSpace, AddrSpaceWrapper, Grant, PageSpan}, Context, Status, }, @@ -19,6 +19,8 @@ use crate::{ }, }; +use crate::context::context::FdTbl; + use super::{CallerCtx, GlobalSchemes, KernelSchemes, OpenResult}; use ::syscall::{ProcSchemeAttrs, SigProcControl, Sigcontrol}; use alloc::{ @@ -108,11 +110,11 @@ enum ContextHandle { Sighandler, Start, NewFiletable { - filetable: Arc>>>, + filetable: Arc>, data: Box<[u8]>, }, Filetable { - filetable: Weak>>>, + filetable: Weak>, data: Box<[u8]>, }, AddrSpace { @@ -129,7 +131,7 @@ enum ContextHandle { CurrentFiletable, AwaitingFiletableChange { - new_ft: Arc>>>, + new_ft: Arc>, }, // TODO: Remove this once openat is implemented, or allow openat-via-dup via e.g. the top-level @@ -320,8 +322,10 @@ impl ProcScheme { use core::fmt::Write; let mut data = String::new(); + // Only the posix file table is targeted. for index in filetable .read() + .posix_fdtbl .iter() .enumerate() .filter_map(|(idx, val)| val.as_ref().map(|_| idx)) diff --git a/src/scheme/root.rs b/src/scheme/root.rs index 603a392b55..8dd4dbc467 100644 --- a/src/scheme/root.rs +++ b/src/scheme/root.rs @@ -1,4 +1,4 @@ -use alloc::{boxed::Box, string::ToString, sync::Arc}; +use alloc::{boxed::Box, string::ToString, sync::Arc, vec::Vec}; use core::{ str, sync::atomic::{AtomicUsize, Ordering}, @@ -15,13 +15,13 @@ use crate::{ scheme::{ self, user::{UserInner, UserScheme}, - SchemeId, SchemeNamespace, + FileDescription, SchemeId, SchemeNamespace, }, syscall::{ data::Stat, error::*, - flag::{EventFlags, MODE_DIR, MODE_FILE, O_CREAT}, - usercopy::{UserSliceRo, UserSliceWo}, + flag::{CallFlags, EventFlags, MODE_DIR, MODE_FILE, O_CREAT}, + usercopy::{UserSliceRo, UserSliceRw, UserSliceWo}, }, }; @@ -330,4 +330,45 @@ impl KernelScheme for RootScheme { Ok(()) } + + fn kfdwrite( + &self, + id: usize, + descs: Vec>>, + flags: CallFlags, + arg: u64, + metadata: &[u64], + ) -> Result { + let handle = { + let handles = self.handles.read(); + let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + handle.clone() + }; + + match handle { + Handle::Scheme(inner) => inner.call_fdwrite(descs, flags, arg, metadata), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::List { .. } => Err(Error::new(EISDIR)), + } + } + + fn kfdread( + &self, + id: usize, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], + ) -> Result { + let handle = { + let handles = self.handles.read(); + let handle = handles.get(&id).ok_or(Error::new(EBADF))?; + handle.clone() + }; + + match handle { + Handle::Scheme(inner) => inner.call_fdread(payload, flags, metadata), + Handle::File(_) => Err(Error::new(EBADF)), + Handle::List { .. } => Err(Error::new(EISDIR)), + } + } } diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 5c4f77da12..11321a3842 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -15,8 +15,9 @@ use spin::{Mutex, RwLock}; use spinning_top::RwSpinlock; use syscall::{ schemev2::{Cqe, CqeOpcode, Opcode, Sqe, SqeFlags}, - CallFlags, FobtainFdFlags, MunmapFlags, SendFdFlags, F_SETFL, KSMSG_CANCEL, - MAP_FIXED_NOREPLACE, SKMSG_FOBTAINFD, SKMSG_FRETURNFD, SKMSG_PROVIDE_MMAP, + CallFlags, FmoveFdFlags, FobtainFdFlags, MunmapFlags, RecvFdFlags, SchemeSocketCall, + SendFdFlags, F_SETFL, KSMSG_CANCEL, MAP_FIXED_NOREPLACE, SKMSG_FOBTAINFD, SKMSG_FRETURNFD, + SKMSG_PROVIDE_MMAP, }; use crate::{ @@ -65,7 +66,7 @@ pub struct UserInner { enum State { Waiting { context: Weak>, - fd: Option>>, + fds: Option>>>, callee_responsible: PageSpan, canceling: bool, }, @@ -78,6 +79,7 @@ enum State { pub enum Response { Regular(usize, u8), Fd(Arc>), + MultipleFds(Option>>>), } const ONE: NonZeroUsize = match NonZeroUsize::new(1) { @@ -99,6 +101,10 @@ enum ParsedCqe { tag: u32, fd: usize, }, + ResponseWithMultipleFds { + tag: u32, + num_fds: usize, + }, ObtainFd { tag: u32, flags: FobtainFdFlags, @@ -167,7 +173,7 @@ impl ParsedCqe { } fn parse_cqe(cqe: &Cqe) -> Result { Ok( - match CqeOpcode::try_from_raw(cqe.flags & 0b11).ok_or(Error::new(EINVAL))? { + match CqeOpcode::try_from_raw(cqe.flags & 0b111).ok_or(Error::new(EINVAL))? { CqeOpcode::RespondRegular => Self::RegularResponse { tag: cqe.tag, code: cqe.result as usize, @@ -177,6 +183,10 @@ impl ParsedCqe { tag: cqe.tag, fd: cqe.result as usize, }, + CqeOpcode::RespondWithMultipleFds => Self::ResponseWithMultipleFds { + tag: cqe.tag, + num_fds: cqe.result as usize, + }, CqeOpcode::SendFevent => Self::TriggerFevent { number: cqe.result as usize, flags: EventFlags::from_bits(cqe.tag as usize).ok_or(Error::new(EINVAL))?, @@ -249,24 +259,26 @@ impl UserInner { match self.call_extended(ctx, None, opcode, args, caller_responsible)? { Response::Regular(code, _) => Error::demux(code), Response::Fd(_) => Err(Error::new(EIO)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } pub fn call_extended( &self, ctx: CallerCtx, - fd: Option>>, + fds: Option>>>, opcode: Opcode, args: impl Args, caller_responsible: &mut PageSpan, ) -> Result { + let next_id = self.next_id()?; self.call_extended_inner( - fd, + fds, Sqe { opcode: opcode as u8, sqe_flags: SqeFlags::empty(), _rsvd: 0, - tag: self.next_id()?, + tag: next_id, caller: ctx.pid as u64, args: { let mut a = args.args(); @@ -280,7 +292,7 @@ impl UserInner { fn call_extended_inner( &self, - fd: Option>>, + fds: Option>>>, sqe: Sqe, caller_responsible: &mut PageSpan, ) -> Result { @@ -295,7 +307,7 @@ impl UserInner { current_context.write().block("UserScheme::call"); states[sqe.tag as usize] = State::Waiting { context: Arc::downgrade(¤t_context), - fd, + fds, canceling: false, // This is the part that the scheme handler will deallocate when responding. It @@ -337,14 +349,14 @@ impl UserInner { canceling: true, mut callee_responsible, context, - fd, + fds, } => { let maybe_eintr = eintr_if_sigkill(&mut callee_responsible); *o = State::Waiting { canceling: true, callee_responsible, context, - fd, + fds, }; drop(states); maybe_eintr?; @@ -354,14 +366,14 @@ impl UserInner { // spurious wakeup State::Waiting { canceling: false, - fd, + fds, context, mut callee_responsible, } => { let maybe_eintr = eintr_if_sigkill(&mut callee_responsible); *o = State::Waiting { canceling: true, - fd, + fds, context, callee_responsible, }; @@ -929,6 +941,9 @@ impl UserInner { .description, ), )?, + ParsedCqe::ResponseWithMultipleFds { tag, num_fds } => { + self.respond(tag, Response::MultipleFds(None))?; + } ParsedCqe::ObtainFd { tag, flags, @@ -940,7 +955,9 @@ impl UserInner { .get_mut(tag as usize) .ok_or(Error::new(EINVAL))? { - State::Waiting { ref mut fd, .. } => fd.take().ok_or(Error::new(ENOENT))?, + State::Waiting { ref mut fds, .. } => { + fds.take().ok_or(Error::new(ENOENT))?.remove(0) + } _ => return Err(Error::new(ENOENT)), }; @@ -1039,7 +1056,7 @@ impl UserInner { Ok(()) } fn respond(&self, tag: u32, mut response: Response) -> Result<()> { - let to_close; + let to_close: Vec; let mut states = self.states.lock(); match states.get_mut(tag as usize) { @@ -1054,7 +1071,7 @@ impl UserInner { State::Waiting { context, - fd, + mut fds, canceling, callee_responsible, } => { @@ -1078,9 +1095,15 @@ impl UserInner { *code = Error::mux(Err(Error::new(EIO))); } - to_close = fd - .and_then(|f| Arc::try_unwrap(f).ok()) - .map(RwLock::into_inner); + if let Response::MultipleFds(ref mut response_fds) = response { + *response_fds = fds.take(); + } + to_close = fds + .into_iter() + .flatten() + .filter_map(|f| Arc::try_unwrap(f).ok()) + .map(RwLock::into_inner) + .collect(); if let Some(context) = context.upgrade() { context.write().unblock(); @@ -1097,8 +1120,8 @@ impl UserInner { None => return Err(Error::new(EBADFD)), } - if let Some(to_close) = to_close { - let _ = to_close.try_close(); + for fd in to_close { + let _ = fd.try_close(); } Ok(()) } @@ -1155,19 +1178,7 @@ impl UserInner { let (pid, desc) = { let context_lock = context::current(); let context = context_lock.read(); - // TODO: Faster, cleaner mechanism to get descriptor - let mut desc_res = Err(Error::new(EBADF)); - for context_file in context.files.read().iter().flatten() { - let (context_scheme, context_number) = { - let desc = context_file.description.read(); - (desc.scheme, desc.number) - }; - if context_scheme == self.scheme_id && context_number == file { - desc_res = Ok(context_file.clone()); - break; - } - } - let desc = desc_res?; + let desc = context.files.read().find_by_scheme(self.scheme_id, file)?; (context.pid, desc.description) }; @@ -1221,6 +1232,7 @@ impl UserInner { return Err(Error::new(EIO)); } + Response::MultipleFds(_) => return Err(Error::new(EIO)), }; let file_ref = GrantFileRef { @@ -1275,6 +1287,196 @@ impl UserInner { Ok(dst_base.start_address().data()) } + + pub fn call_fdwrite( + &self, + descs: Vec>>, + flags: CallFlags, + _arg: u64, + metadata: &[u64], + ) -> Result { + if metadata.is_empty() { + return Err(Error::new(EINVAL)); + } + let Some(verb) = SchemeSocketCall::try_from_raw(metadata[0] as usize) else { + return Err(Error::new(EINVAL)); + }; + + match verb { + SchemeSocketCall::MoveFd => { + if metadata.len() != 2 { + return Err(Error::new(EINVAL)); + } + let mut movefd_flags = FmoveFdFlags::empty(); + if flags.contains(CallFlags::FD_EXCLUSIVE) { + movefd_flags |= FmoveFdFlags::EXCLUSIVE; + } + if flags.contains(CallFlags::FD_CLONE) { + movefd_flags |= FmoveFdFlags::CLONE; + } + self.handle_movefd(descs, metadata[1] as usize, movefd_flags) + } + _ => Err(Error::new(EINVAL)), + } + } + + fn handle_movefd( + &self, + descs: Vec>>, + request_id: usize, + _flags: FmoveFdFlags, + ) -> Result { + let num_fds = descs.len(); + match self + .states + .lock() + .get_mut(request_id) + .ok_or(Error::new(EINVAL))? + { + State::Waiting { ref mut fds, .. } => *fds = Some(descs), + _ => return Err(Error::new(ENOENT)), + }; + + Ok(num_fds) + } + + pub fn call_fdread( + &self, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], + ) -> Result { + if metadata.is_empty() { + return Err(Error::new(EINVAL)); + } + log::debug!( + "call_fdread: payload: {} metadata: {}", + payload.len(), + metadata.len() + ); + + let Some(verb) = SchemeSocketCall::try_from_raw(metadata[0] as usize) else { + return Err(Error::new(EINVAL)); + }; + + match verb { + SchemeSocketCall::ObtainFd => { + if metadata.len() != 2 { + return Err(Error::new(EINVAL)); + } + let mut obtainfd_flags = FobtainFdFlags::empty(); + if flags.contains(CallFlags::FD_UPPER) { + obtainfd_flags |= FobtainFdFlags::UPPER_TBL; + } + if flags.contains(CallFlags::FD_EXCLUSIVE) { + obtainfd_flags |= FobtainFdFlags::EXCLUSIVE; + } + self.handle_obtainfd(payload, metadata[1] as usize, obtainfd_flags) + } + _ => Err(Error::new(EINVAL)), + } + } + + fn handle_obtainfd( + &self, + payload: UserSliceRw, + request_id: usize, + flags: FobtainFdFlags, + ) -> Result { + let descriptions = match self + .states + .lock() + .get_mut(request_id) + .ok_or(Error::new(EINVAL))? + { + State::Waiting { ref mut fds, .. } => fds.take().ok_or(Error::new(ENOENT))?, + _ => return Err(Error::new(ENOENT)), + }; + + let num_fds = if flags.contains(FobtainFdFlags::UPPER_TBL) { + Self::bulk_insert_fds(descriptions, payload)? + } else { + Self::bulk_add_fds(descriptions, payload)? + }; + + Ok(num_fds) + } + + fn bulk_add_fds( + descriptions: Vec>>, + payload: UserSliceRw, + ) -> Result { + let cnt = descriptions.len(); + if payload.len() != cnt * size_of::() { + return Err(Error::new(EINVAL)); + } + if descriptions.is_empty() { + return Ok(0); + } + let current_lock = context::current(); + let current = current_lock.write(); + + let files: Vec = descriptions + .into_iter() + .map(|description| FileDescriptor { + description, + cloexec: true, + }) + .collect(); + let handles = current + .bulk_add_files_posix(files) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } + Ok(handles.len()) + } + + fn bulk_insert_fds( + descriptions: Vec>>, + payload: UserSliceRw, + ) -> Result { + let cnt = descriptions.len(); + if payload.len() != cnt * size_of::() { + return Err(Error::new(EINVAL)); + } + if descriptions.is_empty() { + return Ok(0); + } + let files_iter = descriptions.into_iter().map(|description| FileDescriptor { + description, + cloexec: true, + }); + let first_fd = payload + .in_exact_chunks(size_of::()) + .next() + .ok_or(Error::new(EINVAL))? + .read_usize()?; + + let current_lock = context::current(); + let current = current_lock.write(); + + if first_fd == usize::MAX { + let files = files_iter.collect::>(); + let handles = current + .bulk_insert_files_upper(files) + .ok_or(Error::new(EMFILE))?; + let payload_chunks = payload.in_exact_chunks(size_of::()); + for (handle, chunk) in handles.iter().zip(payload_chunks) { + chunk.copy_from_slice(&handle.get().to_ne_bytes())?; + } + Ok(handles.len()) + } else { + let handles: Vec = payload + .usizes() + .map(|res| res.map(|i| FileHandle::from(i | syscall::UPPER_FDTBL_TAG))) + .collect::>()?; + let files = files_iter.collect::>(); + current.bulk_insert_files_upper_manual(files, &handles)?; + Ok(handles.len()) + } + } } pub struct CaptureGuard { destroyed: bool, @@ -1386,6 +1588,7 @@ impl KernelScheme for UserScheme { ) }), Response::Fd(desc) => Ok(OpenResult::External(desc)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } @@ -1418,6 +1621,7 @@ impl KernelScheme for UserScheme { ) }), Response::Fd(desc) => Ok(OpenResult::External(desc)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } @@ -1570,6 +1774,7 @@ impl KernelScheme for UserScheme { ) }), Response::Fd(desc) => Ok(OpenResult::External(desc)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } fn kfpath(&self, file: usize, buf: UserSliceWo) -> Result { @@ -1749,29 +1954,7 @@ impl KernelScheme for UserScheme { match res { Response::Regular(_, _) => Ok(()), Response::Fd(_) => Err(Error::new(EIO)), - } - } - fn ksendfd( - &self, - number: usize, - desc: Arc>, - flags: SendFdFlags, - arg: u64, - ) -> Result { - let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; - - let ctx = context::current().read().caller_ctx(); - let res = inner.call_extended( - ctx, - Some(desc), - Opcode::Sendfd, - [number, flags.bits(), arg as usize], - &mut PageSpan::empty(), - )?; - - match res { - Response::Regular(res, _) => Error::demux(res), - Response::Fd(_) => Err(Error::new(EIO)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } fn kcall( @@ -1811,8 +1994,90 @@ impl KernelScheme for UserScheme { match res { Response::Regular(res, _) => Error::demux(res), Response::Fd(_) => Err(Error::new(EIO)), + Response::MultipleFds(_) => Err(Error::new(EIO)), } } + fn kfdwrite( + &self, + number: usize, + descs: Vec>>, + flags: CallFlags, + arg: u64, + _metadata: &[u64], + ) -> Result { + let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; + + let mut sendfd_flags = SendFdFlags::empty(); + if flags.contains(CallFlags::FD_EXCLUSIVE) { + sendfd_flags |= SendFdFlags::EXCLUSIVE; + } + + let ctx = context::current().read().caller_ctx(); + let len = descs.len(); + let res = inner.call_extended( + ctx, + Some(descs), + Opcode::Sendfd, + [number, sendfd_flags.bits(), arg as usize, len], + &mut PageSpan::empty(), + )?; + + match res { + Response::Regular(res, _) => Error::demux(res), + Response::Fd(_) => Err(Error::new(EIO)), + Response::MultipleFds(_) => Err(Error::new(EIO)), + } + } + fn kfdread( + &self, + id: usize, + payload: UserSliceRw, + flags: CallFlags, + _metadata: &[u64], + ) -> Result { + let inner = self.inner.upgrade().ok_or(Error::new(ENODEV))?; + if payload.len() % mem::size_of::() != 0 { + return Err(Error::new(EINVAL)); + } + + let mut recvfd_flags = RecvFdFlags::empty(); + if flags.contains(CallFlags::FD_UPPER) { + recvfd_flags |= RecvFdFlags::UPPER_TBL; + } + + let ctx = context::current().read().caller_ctx(); + let len = payload.len() / mem::size_of::(); + let res = inner.call_extended( + ctx, + None, + Opcode::Recvfd, + [id, recvfd_flags.bits(), len], + &mut PageSpan::empty(), + )?; + + let descriptions_opt = match res { + Response::Regular(res, _) => { + return match Error::demux(res) { + Ok(_) => Err(Error::new(EIO)), + Err(e) => Err(e), + } + } + Response::Fd(_) => return Err(Error::new(EIO)), + Response::MultipleFds(fds) => fds, + }; + + let num_fds = if let Some(descriptions) = descriptions_opt { + if recvfd_flags.contains(RecvFdFlags::UPPER_TBL) { + UserInner::bulk_insert_fds(descriptions, payload)? + } else { + UserInner::bulk_add_fds(descriptions, payload)? + } + } else { + 0 + }; + + Ok(num_fds) + } } pub trait Args: Copy { diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 06cd5c2714..0f9c48718b 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -1,4 +1,5 @@ //! Filesystem syscalls +use core::mem::size_of; use core::num::NonZeroUsize; use alloc::{string::String, sync::Arc, vec::Vec}; @@ -319,6 +320,23 @@ pub fn call( core::slice::from_raw_parts_mut(meta.as_mut_ptr().cast(), meta.len() * 8) })?; + match flags { + f if f.contains(CallFlags::WRITE | CallFlags::FD) => { + call_fdwrite(fd, payload, flags, &meta[..copied / 8]) + } + f if f.contains(CallFlags::READ | CallFlags::FD) => { + call_fdread(fd, payload, flags, &meta[..copied / 8]) + } + _ => call_normal(fd, payload, flags, &meta[..copied / 8]), + } +} + +fn call_normal( + fd: FileHandle, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], +) -> Result { let file = (match ( context::current().read(), flags.contains(CallFlags::CONSUME), @@ -337,18 +355,42 @@ pub fn call( .ok_or(Error::new(EBADFD))? .clone(); - scheme.kcall(number, payload, flags, &meta[..copied / 8]) + scheme.kcall(number, payload, flags, metadata) } -pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> Result { - let requested_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?; +fn call_fdwrite( + fd: FileHandle, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], +) -> Result { + let payload_chunks = payload.in_exact_chunks(size_of::()); + let fds = payload_chunks + .map(|chunk| { + let fd = chunk.read_usize()?; + Ok(FileHandle::from(fd)) + }) + .collect::>>()?; - let (scheme, number, desc_to_send) = { + let len = fds.len(); + + fdwrite_inner(fd, fds, flags, 0, metadata)?; + + Ok(len) +} + +fn fdwrite_inner( + socket: FileHandle, + target_fds: Vec, + flags: CallFlags, + arg: u64, + metadata: &[u64], +) -> Result { + // TODO: Ensure deadlocks can't happen + let (scheme, number, descs_to_send) = { let current_lock = context::current(); let current = current_lock.read(); - // TODO: Ensure deadlocks can't happen - let (scheme, number) = match current .get_file(socket) .ok_or(Error::new(EBADF))? @@ -365,27 +407,74 @@ pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> ( scheme, number, - current - .remove_file(fd) - .ok_or(Error::new(EBADF))? - .description, + if flags.contains(CallFlags::FD_CLONE) { + current.bulk_get_files(&target_fds) + } else { + current.bulk_remove_files(&target_fds) + }? + .into_iter() + .map(|f| f.description) + .collect(), ) }; - // Inform the scheme whether there are still references to the file description to be sent, - // either in the current file table or in other file tables, regardless of whether EXCLUSIVE is - // requested. - - let flags_to_scheme = if Arc::strong_count(&desc_to_send) == 1 { - SendFdFlags::EXCLUSIVE - } else { - if requested_flags.contains(SendFdFlags::EXCLUSIVE) { - return Err(Error::new(EBUSY)); + // Inform the scheme whether there are still references to the file description to be sent, + // either in the current file table or in other file tables, regardless of whether EXCLUSIVE is + // requested. + let flags_to_scheme = if flags.contains(CallFlags::FD_EXCLUSIVE) { + for desc in &descs_to_send { + if Arc::strong_count(desc) > 1 { + return Err(Error::new(EBUSY)); + } } - SendFdFlags::empty() + + CallFlags::FD_EXCLUSIVE + } else { + CallFlags::empty() }; - scheme.ksendfd(number, desc_to_send, flags_to_scheme, arg) + scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata) +} + +fn call_fdread( + fd: FileHandle, + payload: UserSliceRw, + flags: CallFlags, + metadata: &[u64], +) -> Result { + let (scheme, number) = { + let current_lock = context::current(); + let current = current_lock.read(); + + let (scheme, number) = match current + .get_file(fd) + .ok_or(Error::new(EBADF))? + .description + .read() + { + ref desc => (desc.scheme, desc.number), + }; + let scheme = scheme::schemes() + .get(scheme) + .ok_or(Error::new(ENODEV))? + .clone(); + + (scheme, number) + }; + + scheme.kfdread(number, payload, flags, metadata) +} + +pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> Result { + let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?; + let mut call_flags = CallFlags::FD | CallFlags::WRITE; + if sendfd_flags.contains(SendFdFlags::CLONE) { + call_flags |= CallFlags::FD_CLONE; + } + if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) { + call_flags |= CallFlags::FD_EXCLUSIVE; + } + fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[]) } /// File descriptor controls diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 5f4777eda3..0f39de0828 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -14,6 +14,7 @@ use crate::{ syscall::EventFlags, }; +use crate::context::context::FdTbl; use crate::{ context, paging::{Page, VirtualAddress, PAGE_SIZE}, @@ -24,14 +25,14 @@ use crate::{ use super::usercopy::UserSliceWo; pub fn exit_this_context(excp: Option) -> ! { - let close_files; + let mut close_files; let addrspace_opt; let context_lock = context::current(); { let mut context = context_lock.write(); close_files = Arc::try_unwrap(mem::take(&mut context.files)) - .map_or_else(|_| Vec::new(), RwLock::into_inner); + .map_or_else(|_| FdTbl::new(), RwLock::into_inner); addrspace_opt = context .set_addr_space(None) .and_then(|a| Arc::try_unwrap(a).ok()); @@ -40,11 +41,7 @@ pub fn exit_this_context(excp: Option) -> ! { } // Files must be closed while context is valid so that messages can be passed - for file_opt in close_files.into_iter() { - if let Some(file) = file_opt { - let _ = file.close(); - } - } + close_files.force_close_all(); drop(addrspace_opt); // TODO: Should status == Status::HardBlocked be handled differently? let owner = {