lock ordering

This commit is contained in:
Jeremy Soller
2025-09-22 07:48:48 -06:00
parent e7358e3e5b
commit 5dc6f7c3ba
79 changed files with 2420 additions and 1181 deletions
+15 -6
View File
@@ -9,7 +9,7 @@ use super::{
usercopy::UserSlice,
};
use crate::syscall::error::Result;
use crate::{sync::CleanLockToken, syscall::error::Result};
struct ByteStr<'a>(&'a [u8]);
@@ -217,12 +217,17 @@ impl SyscallDebugInfo {
}
#[cfg_attr(feature = "syscall_debug", inline)]
pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) {
pub fn debug_start([a, b, c, d, e, f]: [usize; 6], token: &mut CleanLockToken) {
if cfg!(not(feature = "syscall_debug")) {
return;
}
let do_debug = if false && crate::context::current().read().name.contains("init") {
let do_debug = if false
&& crate::context::current()
.read(token.token())
.name
.contains("init")
{
if a == SYS_CLOCK_GETTIME || a == SYS_YIELD || a == SYS_FUTEX {
false
} else if (a == SYS_WRITE || a == SYS_FSYNC) && (b == 1 || b == 2) {
@@ -237,7 +242,7 @@ pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) {
let debug_start = if do_debug {
let context_lock = crate::context::current();
{
let context = context_lock.read();
let context = context_lock.read(token.token());
print!("{} (*{}*): ", context.name, context.pid,);
}
@@ -261,7 +266,11 @@ pub fn debug_start([a, b, c, d, e, f]: [usize; 6]) {
}
#[cfg_attr(feature = "syscall_debug", inline)]
pub fn debug_end([a, b, c, d, e, f]: [usize; 6], result: Result<usize>) {
pub fn debug_end(
[a, b, c, d, e, f]: [usize; 6],
result: Result<usize>,
token: &mut CleanLockToken,
) {
if cfg!(not(feature = "syscall_debug")) {
return;
}
@@ -278,7 +287,7 @@ pub fn debug_end([a, b, c, d, e, f]: [usize; 6], result: Result<usize>) {
let context_lock = crate::context::current();
{
let context = context_lock.read();
let context = context_lock.read(token.token());
print!("{} (*{}*): ", context.name, context.pid,);
}
+167 -125
View File
@@ -13,6 +13,7 @@ use crate::{
},
paging::{Page, VirtualAddress, PAGE_SIZE},
scheme::{self, CallerCtx, FileHandle, KernelScheme, OpenResult, StrOrBytes},
sync::CleanLockToken,
syscall::{data::Stat, error::*, flag::*},
};
@@ -20,29 +21,36 @@ use super::usercopy::{UserSlice, UserSliceRo, UserSliceRw, UserSliceWo};
pub fn file_op_generic<T>(
fd: FileHandle,
op: impl FnOnce(&dyn KernelScheme, usize) -> Result<T>,
token: &mut CleanLockToken,
op: impl FnOnce(&dyn KernelScheme, usize, &mut CleanLockToken) -> Result<T>,
) -> Result<T> {
file_op_generic_ext(fd, |s, _, desc| op(s, desc.number))
file_op_generic_ext(fd, token, |s, _, desc, token| op(s, desc.number, token))
}
pub fn file_op_generic_ext<T>(
fd: FileHandle,
op: impl FnOnce(&dyn KernelScheme, Arc<RwLock<FileDescription>>, FileDescription) -> Result<T>,
token: &mut CleanLockToken,
op: impl FnOnce(
&dyn KernelScheme,
Arc<RwLock<FileDescription>>,
FileDescription,
&mut CleanLockToken,
) -> Result<T>,
) -> Result<T> {
let (file, desc) = {
let file = context::current()
.read()
.read(token.token())
.get_file(fd)
.ok_or(Error::new(EBADF))?;
let desc = *file.description.read();
(file, desc)
};
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(desc.scheme)
.ok_or(Error::new(EBADF))?
.clone();
op(&*scheme, file.description, desc)
op(&*scheme, file.description, desc, token)
}
pub fn copy_path_to_buf(raw_path: UserSliceRo, max_len: usize) -> Result<String> {
let mut path_buf = vec![0_u8; max_len];
@@ -67,8 +75,8 @@ fn is_legacy(path_buf: &String) -> bool {
}
/// Open syscall
pub fn open(raw_path: UserSliceRo, flags: usize) -> Result<FileHandle> {
let (pid, uid, gid, scheme_ns) = match context::current().read() {
pub fn open(raw_path: UserSliceRo, flags: usize, token: &mut CleanLockToken) -> Result<FileHandle> {
let (pid, uid, gid, scheme_ns) = match context::current().read(token.token()) {
ref cx => (cx.pid.into(), cx.euid, cx.egid, cx.ens),
};
@@ -83,7 +91,7 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result<FileHandle> {
// Display a deprecation warning for any usage of the legacy scheme syntax (scheme:/path)
// FIXME remove entries from this list as the respective programs get updated
if path_buf.contains(':') && !is_legacy(&path_buf) {
let name = context::current().read().name.clone();
let name = context::current().read(token.token()).name.clone();
if name.contains("cosmic") && (path_buf == "event:" || path_buf.starts_with("time:")) {
// FIXME cosmic apps likely need crate updates
} else {
@@ -95,14 +103,19 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result<FileHandle> {
let description = {
let (scheme_id, scheme) = {
let schemes = scheme::schemes();
let schemes = scheme::schemes(token.token());
let (scheme_id, scheme) = schemes
.get_name(scheme_ns, scheme_name.as_ref())
.ok_or(Error::new(ENODEV))?;
(scheme_id, scheme.clone())
};
match scheme.kopen(reference.as_ref(), flags, CallerCtx { uid, gid, pid })? {
match scheme.kopen(
reference.as_ref(),
flags,
CallerCtx { uid, gid, pid },
token,
)? {
OpenResult::SchemeLocal(number, internal_flags) => {
Arc::new(RwLock::new(FileDescription {
scheme: scheme_id,
@@ -117,7 +130,7 @@ pub fn open(raw_path: UserSliceRo, flags: usize) -> Result<FileHandle> {
};
//drop(path_buf);
context::current()
.read()
.read(token.token())
.add_file(FileDescriptor {
description,
cloexec: flags & O_CLOEXEC == O_CLOEXEC,
@@ -130,6 +143,7 @@ pub fn openat(
raw_path: UserSliceRo,
flags: usize,
fcntl_flags: u32,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
let path_buf = copy_path_to_buf(raw_path, PATH_MAX)?;
@@ -139,16 +153,16 @@ pub fn openat(
}
let pipe = context::current()
.read()
.read(token.token())
.get_file(fh)
.ok_or(Error::new(EBADF))?;
let description = pipe.description.read();
let caller_ctx = context::current().read().caller_ctx();
let caller_ctx = context::current().read(token.token()).caller_ctx();
let new_description = {
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(description.scheme)
.ok_or(Error::new(EBADF))?
.clone();
@@ -159,6 +173,7 @@ pub fn openat(
flags,
fcntl_flags,
caller_ctx,
token,
);
match res? {
@@ -176,7 +191,7 @@ pub fn openat(
};
context::current()
.read()
.read(token.token())
.add_file(FileDescriptor {
description: new_description,
cloexec: false,
@@ -184,8 +199,8 @@ pub fn openat(
.ok_or(Error::new(EMFILE))
}
/// rmdir syscall
pub fn rmdir(raw_path: UserSliceRo) -> Result<()> {
let (scheme_ns, caller_ctx) = match context::current().read() {
pub fn rmdir(raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> {
let (scheme_ns, caller_ctx) = match context::current().read(token.token()) {
ref cx => (cx.ens, cx.caller_ctx()),
};
@@ -198,18 +213,18 @@ pub fn rmdir(raw_path: UserSliceRo) -> Result<()> {
let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let scheme = {
let schemes = scheme::schemes();
let schemes = scheme::schemes(token.token());
let (_scheme_id, scheme) = schemes
.get_name(scheme_ns, scheme_name.as_ref())
.ok_or(Error::new(ENODEV))?;
scheme.clone()
};
scheme.rmdir(reference.as_ref(), caller_ctx)
scheme.rmdir(reference.as_ref(), caller_ctx, token)
}
/// Unlink syscall
pub fn unlink(raw_path: UserSliceRo) -> Result<()> {
let (scheme_ns, caller_ctx) = match context::current().read() {
pub fn unlink(raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> {
let (scheme_ns, caller_ctx) = match context::current().read(token.token()) {
ref cx => (cx.ens, cx.caller_ctx()),
};
/*
@@ -221,30 +236,34 @@ pub fn unlink(raw_path: UserSliceRo) -> Result<()> {
let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let scheme = {
let schemes = scheme::schemes();
let schemes = scheme::schemes(token.token());
let (_scheme_id, scheme) = schemes
.get_name(scheme_ns, scheme_name.as_ref())
.ok_or(Error::new(ENODEV))?;
scheme.clone()
};
scheme.unlink(reference.as_ref(), caller_ctx)
scheme.unlink(reference.as_ref(), caller_ctx, token)
}
/// Close syscall
pub fn close(fd: FileHandle) -> Result<()> {
pub fn close(fd: FileHandle, token: &mut CleanLockToken) -> Result<()> {
let file = {
let context_lock = context::current();
let context = context_lock.read();
let context = context_lock.read(token.token());
context.remove_file(fd).ok_or(Error::new(EBADF))?
};
file.close()
file.close(token)
}
fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result<FileDescriptor> {
fn duplicate_file(
fd: FileHandle,
user_buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileDescriptor> {
let (caller_ctx, file) = {
let context_lock = context::current();
let context = context_lock.read();
let context = context_lock.read(token.token());
(
context.caller_ctx(),
context.get_file(fd).ok_or(Error::new(EBADF))?,
@@ -260,12 +279,12 @@ fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result<FileDescripto
let description = { *file.description.read() };
let new_description = {
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(description.scheme)
.ok_or(Error::new(EBADF))?
.clone();
match scheme.kdup(description.number, user_buf, caller_ctx)? {
match scheme.kdup(description.number, user_buf, caller_ctx, token)? {
OpenResult::SchemeLocal(number, internal_flags) => {
Arc::new(RwLock::new(FileDescription {
offset: 0,
@@ -287,25 +306,30 @@ fn duplicate_file(fd: FileHandle, user_buf: UserSliceRo) -> Result<FileDescripto
}
/// Duplicate file descriptor
pub fn dup(fd: FileHandle, buf: UserSliceRo) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf)?;
pub fn dup(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<FileHandle> {
let new_file = duplicate_file(fd, buf, token)?;
context::current()
.read()
.read(token.token())
.add_file(new_file)
.ok_or(Error::new(EMFILE))
}
/// Duplicate file descriptor, replacing another
pub fn dup2(fd: FileHandle, new_fd: FileHandle, buf: UserSliceRo) -> Result<FileHandle> {
pub fn dup2(
fd: FileHandle,
new_fd: FileHandle,
buf: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<FileHandle> {
if fd == new_fd {
Ok(new_fd)
} else {
let _ = close(new_fd);
let new_file = duplicate_file(fd, buf)?;
let _ = close(new_fd, token);
let new_file = duplicate_file(fd, buf, token)?;
let context_ref = context::current();
let context = context_ref.read();
let context = context_ref.read(token.token());
context
.insert_file(new_fd, new_file)
@@ -317,6 +341,7 @@ pub fn call(
payload: UserSliceRw,
flags: CallFlags,
metadata: UserSliceRo,
token: &mut CleanLockToken,
) -> Result<usize> {
let mut meta = [0_u64; 3];
@@ -327,12 +352,12 @@ pub fn call(
match flags {
f if f.contains(CallFlags::WRITE | CallFlags::FD) => {
call_fdwrite(fd, payload, flags, &meta[..copied / 8])
call_fdwrite(fd, payload, flags, &meta[..copied / 8], token)
}
f if f.contains(CallFlags::READ | CallFlags::FD) => {
call_fdread(fd, payload, flags, &meta[..copied / 8])
call_fdread(fd, payload, flags, &meta[..copied / 8], token)
}
_ => call_normal(fd, payload, flags, &meta[..copied / 8]),
_ => call_normal(fd, payload, flags, &meta[..copied / 8], token),
}
}
@@ -341,9 +366,10 @@ fn call_normal(
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let file = (match (
context::current().read(),
context::current().read(token.token()),
flags.contains(CallFlags::CONSUME),
) {
(ctxt, true) => ctxt.remove_file(fd),
@@ -355,12 +381,12 @@ fn call_normal(
let desc = file.description.read();
(desc.scheme, desc.number)
};
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(scheme_id)
.ok_or(Error::new(EBADFD))?
.clone();
scheme.kcall(number, payload, flags, metadata)
scheme.kcall(number, payload, flags, metadata, token)
}
fn call_fdwrite(
@@ -368,6 +394,7 @@ fn call_fdwrite(
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let payload_chunks = payload.in_exact_chunks(size_of::<usize>());
let fds = payload_chunks
@@ -379,7 +406,7 @@ fn call_fdwrite(
let len = fds.len();
fdwrite_inner(fd, fds, flags, 0, metadata)?;
fdwrite_inner(fd, fds, flags, 0, metadata, token)?;
Ok(len)
}
@@ -390,25 +417,29 @@ fn fdwrite_inner(
flags: CallFlags,
arg: u64,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
// TODO: Ensure deadlocks can't happen
let (scheme, number, descs_to_send) = {
let current_lock = context::current();
let current = current_lock.read();
let (scheme, number) = match current
.get_file(socket)
.ok_or(Error::new(EBADF))?
.description
.read()
{
ref desc => (desc.scheme, desc.number),
let (scheme, number) = {
let current_lock = context::current();
let current = current_lock.read(token.token());
match current
.get_file(socket)
.ok_or(Error::new(EBADF))?
.description
.read()
{
ref desc => (desc.scheme, desc.number),
}
};
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(scheme)
.ok_or(Error::new(ENODEV))?
.clone();
let current_lock = context::current();
let current = current_lock.read(token.token());
(
scheme,
number,
@@ -438,7 +469,7 @@ fn fdwrite_inner(
CallFlags::empty()
};
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata)
scheme.kfdwrite(number, descs_to_send, flags_to_scheme, arg, metadata, token)
}
fn call_fdread(
@@ -446,20 +477,22 @@ fn call_fdread(
payload: UserSliceRw,
flags: CallFlags,
metadata: &[u64],
token: &mut CleanLockToken,
) -> Result<usize> {
let (scheme, number) = {
let current_lock = context::current();
let current = current_lock.read();
let (scheme, number) = match current
.get_file(fd)
.ok_or(Error::new(EBADF))?
.description
.read()
{
ref desc => (desc.scheme, desc.number),
let (scheme, number) = {
let current_lock = context::current();
let current = current_lock.read(token.token());
match current
.get_file(fd)
.ok_or(Error::new(EBADF))?
.description
.read()
{
ref desc => (desc.scheme, desc.number),
}
};
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(scheme)
.ok_or(Error::new(ENODEV))?
.clone();
@@ -467,10 +500,16 @@ fn call_fdread(
(scheme, number)
};
scheme.kfdread(number, payload, flags, metadata)
scheme.kfdread(number, payload, flags, metadata, token)
}
pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) -> Result<usize> {
pub fn sendfd(
socket: FileHandle,
fd: FileHandle,
flags_raw: usize,
arg: u64,
token: &mut CleanLockToken,
) -> Result<usize> {
let sendfd_flags = SendFdFlags::from_bits(flags_raw).ok_or(Error::new(EINVAL))?;
let mut call_flags = CallFlags::FD | CallFlags::WRITE;
if sendfd_flags.contains(SendFdFlags::CLONE) {
@@ -479,13 +518,13 @@ pub fn sendfd(socket: FileHandle, fd: FileHandle, flags_raw: usize, arg: u64) ->
if sendfd_flags.contains(SendFdFlags::EXCLUSIVE) {
call_flags |= CallFlags::FD_EXCLUSIVE;
}
fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[])
fdwrite_inner(socket, Vec::from([fd]), call_flags, arg, &[], token)
}
/// File descriptor controls
pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result<usize> {
pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize, token: &mut CleanLockToken) -> Result<usize> {
let file = context::current()
.read()
.read(token.token())
.get_file(fd)
.ok_or(Error::new(EBADF))?;
@@ -493,10 +532,10 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result<usize> {
if cmd == F_DUPFD {
// Not in match because 'files' cannot be locked
let new_file = duplicate_file(fd, UserSlice::empty())?;
let new_file = duplicate_file(fd, UserSlice::empty(), token)?;
let context_lock = context::current();
let context = context_lock.read();
let context = context_lock.read(token.token());
return context
.add_file_min(new_file, arg)
@@ -506,18 +545,18 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result<usize> {
// Communicate fcntl with scheme
if cmd != F_GETFD && cmd != F_SETFD {
let scheme = scheme::schemes()
let scheme = scheme::schemes(token.token())
.get(description.scheme)
.ok_or(Error::new(EBADF))?
.clone();
scheme.fcntl(description.number, cmd, arg)?;
scheme.fcntl(description.number, cmd, arg, token)?;
};
// Perform kernel operation if scheme agrees
{
let context_lock = context::current();
let context = context_lock.read();
let context = context_lock.read(token.token());
let mut files = context.files.write();
match *files.get_mut(fd.get()).ok_or(Error::new(EBADF))? {
@@ -548,12 +587,12 @@ pub fn fcntl(fd: FileHandle, cmd: usize, arg: usize) -> Result<usize> {
}
}
pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
let (caller_ctx, scheme_ns) = match context::current().read() {
pub fn flink(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> {
let (caller_ctx, scheme_ns) = match context::current().read(token.token()) {
ref cx => (cx.caller_ctx(), cx.ens),
};
let file = context::current()
.read()
.read(token.token())
.get_file(fd)
.ok_or(Error::new(EBADF))?;
@@ -566,7 +605,7 @@ pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let (scheme_id, scheme) = {
let schemes = scheme::schemes();
let schemes = scheme::schemes(token.token());
let (scheme_id, scheme) = schemes
.get_name(scheme_ns, scheme_name.as_ref())
.ok_or(Error::new(ENODEV))?;
@@ -579,15 +618,15 @@ pub fn flink(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
return Err(Error::new(EXDEV));
}
scheme.flink(description.number, reference.as_ref(), caller_ctx)
scheme.flink(description.number, reference.as_ref(), caller_ctx, token)
}
pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
let (caller_ctx, scheme_ns) = match context::current().read() {
pub fn frename(fd: FileHandle, raw_path: UserSliceRo, token: &mut CleanLockToken) -> Result<()> {
let (caller_ctx, scheme_ns) = match context::current().read(token.token()) {
ref cx => (cx.caller_ctx(), cx.ens),
};
let file = context::current()
.read()
.read(token.token())
.get_file(fd)
.ok_or(Error::new(EBADF))?;
@@ -600,7 +639,7 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
let (scheme_name, reference) = path.as_parts().ok_or(Error::new(EINVAL))?;
let (scheme_id, scheme) = {
let schemes = scheme::schemes();
let schemes = scheme::schemes(token.token());
let (scheme_id, scheme) = schemes
.get_name(scheme_ns, scheme_name.as_ref())
.ok_or(Error::new(ENODEV))?;
@@ -613,13 +652,13 @@ pub fn frename(fd: FileHandle, raw_path: UserSliceRo) -> Result<()> {
return Err(Error::new(EXDEV));
}
scheme.frename(description.number, reference.as_ref(), caller_ctx)
scheme.frename(description.number, reference.as_ref(), caller_ctx, token)
}
/// File status
pub fn fstat(fd: FileHandle, user_buf: UserSliceWo) -> Result<()> {
file_op_generic_ext(fd, |scheme, _, desc| {
scheme.kfstat(desc.number, user_buf)?;
pub fn fstat(fd: FileHandle, user_buf: UserSliceWo, token: &mut CleanLockToken) -> Result<()> {
file_op_generic_ext(fd, token, |scheme, _, desc, token| {
scheme.kfstat(desc.number, user_buf, token)?;
// TODO: Ensure only the kernel can access the stat when st_dev is set, or use another API
// for retrieving the scheme ID from a file descriptor.
@@ -639,7 +678,7 @@ pub fn fstat(fd: FileHandle, user_buf: UserSliceWo) -> Result<()> {
})
}
pub fn funmap(virtual_address: usize, length: usize) -> Result<usize> {
pub fn funmap(virtual_address: usize, length: usize, token: &mut CleanLockToken) -> Result<usize> {
// Partial lengths in funmap are allowed according to POSIX, but not particularly meaningful;
// since the memory needs to SIGSEGV if later read, the entire page needs to disappear.
//
@@ -654,14 +693,14 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result<usize> {
);
}
let addr_space = Arc::clone(context::current().read().addr_space()?);
let addr_space = Arc::clone(context::current().read(token.token()).addr_space()?);
let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned)
.ok_or(Error::new(EINVAL))?;
let unpin = false;
let notify = addr_space.munmap(span, unpin)?;
for map in notify {
let _ = map.unmap();
let _ = map.unmap(token);
}
Ok(0)
@@ -673,6 +712,7 @@ pub fn mremap(
new_address: usize,
new_size: usize,
flags: usize,
token: &mut CleanLockToken,
) -> Result<usize> {
if old_address % PAGE_SIZE != 0
|| old_size % PAGE_SIZE != 0
@@ -712,7 +752,7 @@ pub fn mremap(
return Err(Error::new(EOPNOTSUPP));
}
let raii_frame = addr_space.borrow_frame_enforce_rw_allocated(src_span.base)?;
let raii_frame = addr_space.borrow_frame_enforce_rw_allocated(src_span.base, token)?;
let base = addr_space.acquire_write().mmap(
&addr_space,
@@ -752,17 +792,17 @@ pub fn mremap(
}
}
pub fn lseek(fd: FileHandle, pos: i64, whence: usize) -> Result<usize> {
pub fn lseek(fd: FileHandle, pos: i64, whence: usize, token: &mut CleanLockToken) -> Result<usize> {
enum Ret {
Legacy(usize),
Fsize((Option<u64>, Arc<RwLock<FileDescription>>)),
}
let fsize_or_legacy = file_op_generic_ext(fd, |scheme, desc_arc, desc| {
let fsize_or_legacy = file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| {
Ok(
if let Some(new_off) = scheme.legacy_seek(desc.number, pos as isize, whence) {
if let Some(new_off) = scheme.legacy_seek(desc.number, pos as isize, whence, token) {
Ret::Legacy(new_off?)
} else if whence == SEEK_END {
Ret::Fsize((Some(scheme.fsize(desc.number)?), desc_arc))
Ret::Fsize((Some(scheme.fsize(desc.number, token)?), desc_arc))
} else {
Ret::Fsize((None, desc_arc))
},
@@ -789,19 +829,20 @@ pub fn lseek(fd: FileHandle, pos: i64, whence: usize) -> Result<usize> {
Ok(guard.offset as usize)
}
pub fn sys_read(fd: FileHandle, buf: UserSliceWo) -> Result<usize> {
let (bytes_read, desc_arc, desc) = file_op_generic_ext(fd, |scheme, desc_arc, desc| {
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
};
Ok((
scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags)?,
desc_arc,
desc,
))
})?;
pub fn sys_read(fd: FileHandle, buf: UserSliceWo, token: &mut CleanLockToken) -> Result<usize> {
let (bytes_read, desc_arc, desc) =
file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| {
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
};
Ok((
scheme.kreadoff(desc.number, buf, offset, desc.flags, desc.flags, token)?,
desc_arc,
desc,
))
})?;
if desc.internal_flags.contains(InternalFlags::POSITIONED) {
match desc_arc.write().offset {
ref mut offset => *offset = offset.saturating_add(bytes_read as u64),
@@ -809,19 +850,20 @@ pub fn sys_read(fd: FileHandle, buf: UserSliceWo) -> Result<usize> {
}
Ok(bytes_read)
}
pub fn sys_write(fd: FileHandle, buf: UserSliceRo) -> Result<usize> {
let (bytes_written, desc_arc, desc) = file_op_generic_ext(fd, |scheme, desc_arc, desc| {
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
};
Ok((
scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags)?,
desc_arc,
desc,
))
})?;
pub fn sys_write(fd: FileHandle, buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
let (bytes_written, desc_arc, desc) =
file_op_generic_ext(fd, token, |scheme, desc_arc, desc, token| {
let offset = if desc.internal_flags.contains(InternalFlags::POSITIONED) {
desc.offset
} else {
u64::MAX
};
Ok((
scheme.kwriteoff(desc.number, buf, offset, desc.flags, desc.flags, token)?,
desc_arc,
desc,
))
})?;
if desc.internal_flags.contains(InternalFlags::POSITIONED) {
match desc_arc.write().offset {
ref mut offset => *offset = offset.saturating_add(bytes_written as u64),
+20 -12
View File
@@ -8,18 +8,17 @@ use alloc::{
};
use core::sync::atomic::{AtomicU32, Ordering};
use rmm::Arch;
use spin::RwLock;
use spinning_top::RwSpinlock;
use syscall::EINTR;
use crate::{
context::{
self,
memory::{AddrSpace, AddrSpaceWrapper},
Context,
ContextLock,
},
memory::PhysicalAddress,
paging::{Page, VirtualAddress},
sync::{CleanLockToken, Mutex, L1},
time,
};
@@ -42,7 +41,7 @@ pub struct FutexEntry {
// TODO: FUTEX_REQUEUE
target_virtaddr: VirtualAddress,
// Context to wake up, and compare address spaces.
context_lock: Arc<RwSpinlock<Context>>,
context_lock: Arc<ContextLock>,
// address space to check against if virt matches but not phys
addr_space: Weak<AddrSpaceWrapper>,
}
@@ -53,7 +52,7 @@ pub struct FutexEntry {
// lwp_park/lwp_unpark from NetBSD) could be a simpler replacement.
//
// TODO: Use an actual hash table.
static FUTEXES: RwLock<FutexList> = RwLock::new(FutexList::new());
static FUTEXES: Mutex<L1, FutexList> = Mutex::new(FutexList::new());
fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Option<PhysicalAddress> {
// TODO: Move this elsewhere!
@@ -69,7 +68,14 @@ fn validate_and_translate_virt(space: &AddrSpace, addr: VirtualAddress) -> Optio
Some(frame.add(off))
}
pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) -> Result<usize> {
pub fn futex(
addr: usize,
op: usize,
val: usize,
val2: usize,
_addr2: usize,
token: &mut CleanLockToken,
) -> Result<usize> {
let current_addrsp = AddrSpace::current()?;
// Keep the address space locked so we can safely read from the physical address. Unlock it
@@ -89,7 +95,8 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) ->
.transpose()?;
{
let mut futexes = FUTEXES.write();
let mut futexes = FUTEXES.lock(token.token());
let (futexes, mut token) = futexes.token_split();
let context_lock = context::current();
@@ -137,7 +144,7 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) ->
}
{
let mut context = context_lock.write();
let mut context = context_lock.write(token.token());
context.wake = timeout_opt.map(|TimeSpec { tv_sec, tv_nsec }| {
tv_sec as u128 * time::NANOS_PER_SEC + tv_nsec as u128
@@ -161,10 +168,10 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) ->
drop(addr_space_guard);
context::switch();
context::switch(token);
if timeout_opt.is_some() {
context::current().write().wake = None;
context::current().write(token.token()).wake = None;
Err(Error::new(ETIMEDOUT))
} else {
Ok(0)
@@ -174,7 +181,8 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) ->
let mut woken = 0;
{
let mut futexes = FUTEXES.write();
let mut futexes = FUTEXES.lock(token.token());
let (futexes, mut token) = futexes.token_split();
let mut i = 0;
@@ -187,7 +195,7 @@ pub fn futex(addr: usize, op: usize, val: usize, val2: usize, _addr2: usize) ->
i += 1;
continue;
}
futexes[i].context_lock.write().unblock();
futexes[i].context_lock.write(token.token()).unblock();
futexes.swap_remove_back(i);
woken += 1;
}
+102 -67
View File
@@ -23,11 +23,11 @@ use self::{
usercopy::UserSlice,
};
use crate::percpu::PercpuBlock;
use crate::{
context::memory::AddrSpace,
percpu::PercpuBlock,
scheme::{memory::MemoryScheme, FileHandle},
sync::CleanLockToken,
};
/// Debug
@@ -54,13 +54,29 @@ pub mod usercopy;
/// This function is the syscall handler of the kernel, it is composed of an inner function that returns a `Result<usize>`. After the inner function runs, the syscall
/// function calls [`Error::mux`] on it.
#[must_use]
pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> usize {
pub fn syscall(
a: usize,
b: usize,
c: usize,
d: usize,
e: usize,
f: usize,
token: &mut CleanLockToken,
) -> usize {
#[inline(always)]
fn inner(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> Result<usize> {
fn inner(
a: usize,
b: usize,
c: usize,
d: usize,
e: usize,
f: usize,
token: &mut CleanLockToken,
) -> Result<usize> {
let fd = FileHandle::from(b);
//SYS_* is declared in kernel/syscall/src/number.rs
match a {
SYS_WRITE2 => file_op_generic_ext(fd, |scheme, _, desc| {
SYS_WRITE2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
@@ -77,17 +93,18 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
}),
SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?),
SYS_WRITE => sys_write(fd, UserSlice::ro(c, d)?, token),
SYS_FMAP => {
let addrspace = AddrSpace::current()?;
let map = unsafe { UserSlice::ro(c, d)?.read_exact::<Map>()? };
if b == !0 {
MemoryScheme::fmap_anonymous(&addrspace, &map, false)
MemoryScheme::fmap_anonymous(&addrspace, &map, false, token)
} else {
file_op_generic(fd, |scheme, number| {
scheme.kfmap(number, &addrspace, &map, false)
file_op_generic(fd, token, |scheme, number, token| {
scheme.kfmap(number, &addrspace, &map, false, token)
})
}
}
@@ -99,15 +116,15 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us
return Err(Error::new(EINVAL));
}
file_op_generic(fd, |scheme, number| {
scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64)
file_op_generic(fd, token, |scheme, number, token| {
scheme.getdents(number, UserSlice::wo(c, d)?, header_size, f as u64, token)
})
}
SYS_FUTIMENS => file_op_generic(fd, |scheme, number| {
scheme.kfutimens(number, UserSlice::ro(c, d)?)
SYS_FUTIMENS => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfutimens(number, UserSlice::ro(c, d)?, token)
}),
SYS_READ2 => file_op_generic_ext(fd, |scheme, _, desc| {
SYS_READ2 => file_op_generic_ext(fd, token, |scheme, _, desc, token| {
let flags = if f == usize::MAX {
None
} else {
@@ -124,79 +141,97 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us
e as u64,
flags.map_or(desc.flags, |f| desc.rw_flags(f)),
desc.flags,
token,
)
}),
SYS_READ => sys_read(fd, UserSlice::wo(c, d)?),
SYS_FPATH => file_op_generic(fd, |scheme, number| {
scheme.kfpath(number, UserSlice::wo(c, d)?)
SYS_READ => sys_read(fd, UserSlice::wo(c, d)?, token),
SYS_FPATH => file_op_generic(fd, token, |scheme, number, token| {
scheme.kfpath(number, UserSlice::wo(c, d)?, token)
}),
SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?).map(|()| 0),
SYS_FSTATVFS => file_op_generic(fd, |scheme, number| {
scheme.kfstatvfs(number, UserSlice::wo(c, d)?).map(|()| 0)
SYS_FSTAT => fstat(fd, UserSlice::wo(c, d)?, token).map(|()| 0),
SYS_FSTATVFS => file_op_generic(fd, token, |scheme, number, token| {
scheme
.kfstatvfs(number, UserSlice::wo(c, d)?, token)
.map(|()| 0)
}),
SYS_DUP => dup(fd, UserSlice::ro(c, d)?).map(FileHandle::into),
SYS_DUP2 => dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?).map(FileHandle::into),
#[cfg(target_pointer_width = "32")]
SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64 | ((f as u64) << 32)),
#[cfg(target_pointer_width = "64")]
SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64),
SYS_LSEEK => lseek(fd, c as i64, d),
SYS_FCHMOD => file_op_generic(fd, |scheme, number| {
scheme.fchmod(number, c as u16).map(|()| 0)
}),
SYS_FCHOWN => file_op_generic(fd, |scheme, number| {
scheme.fchown(number, c as u32, d as u32).map(|()| 0)
}),
SYS_FCNTL => fcntl(fd, c, d),
SYS_FEVENT => file_op_generic(fd, |scheme, number| {
Ok(scheme
.fevent(number, EventFlags::from_bits_truncate(c))?
.bits())
}),
SYS_FLINK => flink(fd, UserSlice::ro(c, d)?).map(|()| 0),
SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?).map(|()| 0),
SYS_FUNMAP => funmap(b, c),
SYS_FSYNC => file_op_generic(fd, |scheme, number| scheme.fsync(number).map(|()| 0)),
// TODO: 64-bit lengths on 32-bit platforms
SYS_FTRUNCATE => {
file_op_generic(fd, |scheme, number| scheme.ftruncate(number, c).map(|()| 0))
SYS_DUP => dup(fd, UserSlice::ro(c, d)?, token).map(FileHandle::into),
SYS_DUP2 => {
dup2(fd, FileHandle::from(c), UserSlice::ro(d, e)?, token).map(FileHandle::into)
}
SYS_CLOSE => close(fd).map(|()| 0),
#[cfg(target_pointer_width = "32")]
SYS_SENDFD => sendfd(
fd,
FileHandle::from(c),
d,
e as u64 | ((f as u64) << 32),
token,
),
#[cfg(target_pointer_width = "64")]
SYS_SENDFD => sendfd(fd, FileHandle::from(c), d, e as u64, token),
SYS_LSEEK => lseek(fd, c as i64, d, token),
SYS_FCHMOD => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchmod(number, c as u16, token).map(|()| 0)
}),
SYS_FCHOWN => file_op_generic(fd, token, |scheme, number, token| {
scheme.fchown(number, c as u32, d as u32, token).map(|()| 0)
}),
SYS_FCNTL => fcntl(fd, c, d, token),
SYS_FEVENT => file_op_generic(fd, token, |scheme, number, token| {
Ok(scheme
.fevent(number, EventFlags::from_bits_truncate(c), token)?
.bits())
}),
SYS_FLINK => flink(fd, UserSlice::ro(c, d)?, token).map(|()| 0),
SYS_FRENAME => frename(fd, UserSlice::ro(c, d)?, token).map(|()| 0),
SYS_FUNMAP => funmap(b, c, token),
SYS_FSYNC => file_op_generic(fd, token, |scheme, number, token| {
scheme.fsync(number, token).map(|()| 0)
}),
// TODO: 64-bit lengths on 32-bit platforms
SYS_FTRUNCATE => file_op_generic(fd, token, |scheme, number, token| {
scheme.ftruncate(number, c, token).map(|()| 0)
}),
SYS_CLOSE => close(fd, token).map(|()| 0),
SYS_CALL => call(
fd,
UserSlice::rw(c, d)?,
CallFlags::from_bits(e & !0xff).ok_or(Error::new(EINVAL))?,
UserSlice::ro(f, (e & 0xff) * 8)?,
token,
),
SYS_OPEN => open(UserSlice::ro(b, c)?, d).map(FileHandle::into),
SYS_OPENAT => openat(fd, UserSlice::ro(c, d)?, e, f as _).map(FileHandle::into),
SYS_RMDIR => rmdir(UserSlice::ro(b, c)?).map(|()| 0),
SYS_UNLINK => unlink(UserSlice::ro(b, c)?).map(|()| 0),
SYS_YIELD => sched_yield().map(|()| 0),
SYS_OPEN => open(UserSlice::ro(b, c)?, d, token).map(FileHandle::into),
SYS_OPENAT => openat(fd, UserSlice::ro(c, d)?, e, f as _, token).map(FileHandle::into),
SYS_RMDIR => rmdir(UserSlice::ro(b, c)?, token).map(|()| 0),
SYS_UNLINK => unlink(UserSlice::ro(b, c)?, token).map(|()| 0),
SYS_YIELD => sched_yield(token).map(|()| 0),
SYS_NANOSLEEP => nanosleep(
UserSlice::ro(b, core::mem::size_of::<TimeSpec>())?,
UserSlice::wo(c, core::mem::size_of::<TimeSpec>())?.none_if_null(),
token,
)
.map(|()| 0),
SYS_CLOCK_GETTIME => {
clock_gettime(b, UserSlice::wo(c, core::mem::size_of::<TimeSpec>())?).map(|()| 0)
}
SYS_FUTEX => futex(b, c, d, e, f),
SYS_FUTEX => futex(b, c, d, e, f, token),
SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)).map(|()| 0),
SYS_MKNS => mkns(UserSlice::ro(
b,
c.checked_mul(core::mem::size_of::<[usize; 2]>())
.ok_or(Error::new(EOVERFLOW))?,
)?),
SYS_MREMAP => mremap(b, c, d, e, f),
SYS_MKNS => mkns(
UserSlice::ro(
b,
c.checked_mul(core::mem::size_of::<[usize; 2]>())
.ok_or(Error::new(EOVERFLOW))?,
)?,
token,
),
SYS_MREMAP => mremap(b, c, d, e, f, token),
_ => return Err(Error::new(ENOSYS)),
}
@@ -204,17 +239,17 @@ pub fn syscall(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize) -> us
PercpuBlock::current().inside_syscall.set(true);
debug_start([a, b, c, d, e, f]);
debug_start([a, b, c, d, e, f], token);
let result = inner(a, b, c, d, e, f);
let result = inner(a, b, c, d, e, f, token);
debug_end([a, b, c, d, e, f], result);
debug_end([a, b, c, d, e, f], result, token);
let percpu = PercpuBlock::current();
percpu.inside_syscall.set(false);
if percpu.switch_internals.being_sigkilled.get() {
exit_this_context(None);
exit_this_context(None, token);
}
// errormux turns Result<usize> into -errno
+4 -4
View File
@@ -1,14 +1,14 @@
use alloc::vec::Vec;
use crate::{context, scheme, syscall::error::*};
use crate::{context, scheme, sync::CleanLockToken, syscall::error::*};
use super::{
copy_path_to_buf,
usercopy::{UserSlice, UserSliceRo},
};
pub fn mkns(mut user_buf: UserSliceRo) -> Result<usize> {
let (uid, from) = match context::current().read() {
pub fn mkns(mut user_buf: UserSliceRo, token: &mut CleanLockToken) -> Result<usize> {
let (uid, from) = match context::current().read(token.token()) {
ref cx => (cx.euid, cx.ens),
};
@@ -36,6 +36,6 @@ pub fn mkns(mut user_buf: UserSliceRo) -> Result<usize> {
user_buf = next_part;
}
let to = scheme::schemes_mut().make_ns(from, names)?;
let to = scheme::schemes_mut(token.token()).make_ns(from, names)?;
Ok(to.into())
}
+10 -9
View File
@@ -11,6 +11,7 @@ use crate::{
},
event,
scheme::GlobalSchemes,
sync::CleanLockToken,
syscall::EventFlags,
};
@@ -24,13 +25,13 @@ use crate::{
use super::usercopy::UserSliceWo;
pub fn exit_this_context(excp: Option<syscall::Exception>) -> ! {
pub fn exit_this_context(excp: Option<syscall::Exception>, token: &mut CleanLockToken) -> ! {
let mut close_files;
let addrspace_opt;
let context_lock = context::current();
{
let mut context = context_lock.write();
let mut context = context_lock.write(token.token());
close_files = Arc::try_unwrap(mem::take(&mut context.files))
.map_or_else(|_| FdTbl::new(), RwLock::into_inner);
addrspace_opt = context
@@ -41,11 +42,11 @@ pub fn exit_this_context(excp: Option<syscall::Exception>) -> ! {
}
// Files must be closed while context is valid so that messages can be passed
close_files.force_close_all();
close_files.force_close_all(token);
drop(addrspace_opt);
// TODO: Should status == Status::HardBlocked be handled differently?
let owner = {
let mut guard = context_lock.write();
let mut guard = context_lock.write(token.token());
guard.status = context::Status::Dead { excp };
guard.owner_proc_id
};
@@ -57,9 +58,9 @@ pub fn exit_this_context(excp: Option<syscall::Exception>) -> ! {
);
}
{
let _ = context::contexts_mut().remove(&ContextRef(context_lock));
let _ = context::contexts_mut(token.token()).remove(&ContextRef(context_lock));
}
context::switch();
context::switch(token);
unreachable!();
}
@@ -72,13 +73,13 @@ pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<()> {
AddrSpace::current()?.mprotect(span, flags)
}
pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) {
pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap, token: &mut CleanLockToken) {
assert_ne!(bootstrap.page_count, 0);
{
let addr_space = Arc::clone(
context::current()
.read()
.read(token.token())
.addr_space()
.expect("expected bootstrap context to have an address space"),
);
@@ -128,7 +129,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) {
// Start in a minimal environment without any stack.
match context::current()
.write()
.write(token.token())
.regs_mut()
.expect("bootstrap needs registers to be available")
{
+12 -7
View File
@@ -1,5 +1,6 @@
use crate::{
context,
sync::CleanLockToken,
syscall::{
data::TimeSpec,
error::*,
@@ -24,7 +25,11 @@ pub fn clock_gettime(clock: usize, buf: UserSliceWo) -> Result<()> {
}
/// Nanosleep will sleep by switching the current context
pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option<UserSliceWo>) -> Result<()> {
pub fn nanosleep(
req_buf: UserSliceRo,
rem_buf_opt: Option<UserSliceWo>,
token: &mut CleanLockToken,
) -> Result<()> {
let req = unsafe { req_buf.read_exact::<TimeSpec>()? };
let start = time::monotonic();
@@ -32,7 +37,7 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option<UserSliceWo>) -> Resu
let current_context = context::current();
{
let mut context = current_context.write();
let mut context = current_context.write(token.token());
if let Some((tctl, pctl, _)) = context.sigcontrol() {
if tctl.currently_pending_unblocked(pctl) != 0 {
@@ -46,9 +51,9 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option<UserSliceWo>) -> Resu
// TODO: The previous wakeup reason was most likely signals, but is there any other possible
// reason?
context::switch();
context::switch(token);
let was_interrupted = current_context.write().wake.take().is_some();
let was_interrupted = current_context.write(token.token()).wake.take().is_some();
if let Some(rem_buf) = rem_buf_opt {
let current = time::monotonic();
@@ -74,9 +79,9 @@ pub fn nanosleep(req_buf: UserSliceRo, rem_buf_opt: Option<UserSliceWo>) -> Resu
}
}
pub fn sched_yield() -> Result<()> {
context::switch();
pub fn sched_yield(token: &mut CleanLockToken) -> Result<()> {
context::switch(token);
// TODO: Do this check in userspace
context::signal::signal_handler();
context::signal::signal_handler(token);
Ok(())
}