vasilito/RedBear-OS
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build the foundation for TLB shootdown.

Browse Source
This commit is contained in:
4lDO2
2024-02-26 23:18:31 +01:00
parent 156017a25d
commit 6bf90f9eee
11 changed files with 201 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/arch/x86_64/gdt.rs
+2 -1
View File
@@ -1,6 +1,6 @@
//! Global descriptor table
use core::{convert::TryInto, mem};
use core::{convert::TryInto, mem, sync::atomic::AtomicUsize};
use crate::{
cpu_set::LogicalCpuId,
@@ -239,6 +239,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
pcr.percpu = PercpuBlock {
cpu_id,
switch_internals: Default::default(),
nmi_flags_lock: AtomicUsize::new(0),
#[cfg(feature = "profiling")]
profiling: None,
src/arch/x86_shared/ipi.rs
+15
View File
@@ -37,3 +37,18 @@ pub fn ipi(kind: IpiKind, target: IpiTarget) {
let icr = (target as u64) << 18 | 1 << 14 | (kind as u64);
unsafe { LOCAL_APIC.set_icr(icr) };
}
#[cfg(feature = "multi_core")]
#[inline(always)]
pub fn ipi_single(kind: IpiKind, target: u32) {
use crate::device::local_apic::LOCAL_APIC;
#[cfg(feature = "profiling")]
if matches!(kind, IpiKind::Profile) {
let icr = (target as u64) << 18 | 1 << 14 | 0b100 << 8;
unsafe { LOCAL_APIC.set_icr(icr) };
return;
}
let icr = u64::from(target) << 18 | 1 << 14 | (kind as u64);
unsafe { LOCAL_APIC.set_icr(icr) };
}
src/context/memory.rs
+92 -52
View File
@@ -98,22 +98,21 @@ pub struct AddrSpace {
/// against null pointers, so fixed mmaps to address zero are still allowed.
pub mmap_min: usize,
}
impl AddrSpace {
pub fn current() -> Result<Arc<AddrSpaceWrapper>> {
Ok(Arc::clone(super::current()?.read().addr_space()?))
}
impl AddrSpaceWrapper {
/// Attempt to clone an existing address space so that all mappings are copied (CoW).
pub fn try_clone(&mut self) -> Result<Arc<AddrSpaceWrapper>> {
pub fn try_clone(&self) -> Result<Arc<AddrSpaceWrapper>> {
let mut guard = self.inner.write();
let guard = &mut *guard;
let mut new_arc = AddrSpaceWrapper::new()?;
let new = Arc::get_mut(&mut new_arc)
.expect("expected new address space Arc not to be aliased");
let this_mapper = &mut self.table.utable;
let mut this_flusher = Flusher::with_cpu_set(&mut self.used_by);
let this_mapper = &mut guard.table.utable;
let mut this_flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack);
for (grant_base, grant_info) in self.grants.iter() {
for (grant_base, grant_info) in guard.grants.iter() {
let new_grant = match grant_info.provider {
// No, your temporary UserScheme mappings will not be kept across forks.
Provider::External {
@@ -191,23 +190,15 @@ impl AddrSpace {
}
Ok(new_arc)
}
pub fn new() -> Result<Self> {
Ok(Self {
grants: UserGrants::new(),
table: setup_new_utable()?,
mmap_min: MMAP_MIN_DEFAULT,
used_by: LogicalCpuSet::empty(),
})
}
pub fn is_current(&self) -> bool {
self.table.utable.is_current()
}
pub fn mprotect(&mut self, requested_span: PageSpan, flags: MapFlags) -> Result<()> {
let mapper = &mut self.table.utable;
let mut flusher = Flusher::with_cpu_set(&mut self.used_by);
pub fn mprotect(&self, requested_span: PageSpan, flags: MapFlags) -> Result<()> {
let mut guard = self.inner.write();
let guard = &mut *guard;
let mapper = &mut guard.table.utable;
let mut flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack);
// TODO: Remove allocation (might require BTreeMap::set_key or interior mutability).
let regions = self
let regions = guard
.grants
.conflicts(requested_span)
.map(|(base, info)| {
@@ -222,7 +213,7 @@ impl AddrSpace {
for grant_span_res in regions {
let grant_span = grant_span_res?;
let grant = self
let grant = guard
.grants
.remove(grant_span.base)
.expect("grant cannot magically disappear while we hold the lock!");
@@ -235,14 +226,14 @@ impl AddrSpace {
//log::info!("Sliced into\n\n{:#?}\n\n{:#?}\n\n{:#?}", before, grant, after);
if let Some(before) = before {
self.grants.insert(before);
guard.grants.insert(before);
}
if let Some(after) = after {
self.grants.insert(after);
guard.grants.insert(after);
}
if !grant.info.can_have_flags(flags) {
self.grants.insert(grant);
guard.grants.insert(grant);
return Err(Error::new(EACCES));
}
@@ -259,18 +250,38 @@ impl AddrSpace {
grant.remap(mapper, &mut flusher, new_flags);
//log::info!("Mprotect grant became {:#?}", grant);
self.grants.insert(grant);
guard.grants.insert(grant);
}
Ok(())
}
#[must_use = "needs to notify files"]
pub fn munmap(
&mut self,
&self,
mut requested_span: PageSpan,
unpin: bool,
) -> Result<Vec<UnmapResult>> {
let mut flusher = Flusher::with_cpu_set(&mut self.used_by);
Self::munmap_inner(&mut self.grants, &mut self.table.utable, &mut flusher, requested_span, unpin)
let mut guard = self.inner.write();
let guard = &mut *guard;
let mut flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack);
AddrSpace::munmap_inner(&mut guard.grants, &mut guard.table.utable, &mut flusher, requested_span, unpin)
}
}
impl AddrSpace {
pub fn current() -> Result<Arc<AddrSpaceWrapper>> {
Ok(Arc::clone(super::current()?.read().addr_space()?))
}
pub fn new() -> Result<Self> {
Ok(Self {
grants: UserGrants::new(),
table: setup_new_utable()?,
mmap_min: MMAP_MIN_DEFAULT,
used_by: LogicalCpuSet::empty(),
})
}
pub fn is_current(&self) -> bool {
self.table.utable.is_current()
}
fn munmap_inner(
this_grants: &mut UserGrants,
@@ -341,6 +352,7 @@ impl AddrSpace {
}
pub fn mmap_anywhere(
&mut self,
dst_lock: &AddrSpaceWrapper,
page_count: NonZeroUsize,
flags: MapFlags,
map: impl FnOnce(
@@ -350,10 +362,11 @@ impl AddrSpace {
&mut Flusher,
) -> Result<Grant>,
) -> Result<Page> {
self.mmap(None, page_count, flags, &mut Vec::new(), map)
self.mmap(dst_lock, None, page_count, flags, &mut Vec::new(), map)
}
pub fn mmap(
&mut self,
dst_lock: &AddrSpaceWrapper,
requested_base_opt: Option<Page>,
page_count: NonZeroUsize,
flags: MapFlags,
@@ -378,7 +391,7 @@ impl AddrSpace {
requested_span
} else if flags.contains(MapFlags::MAP_FIXED) {
let unpin = false;
let mut notify_files = self.munmap(requested_span, unpin)?;
let mut notify_files = Self::munmap_inner(&mut self.grants, &mut self.table.utable, &mut Flusher::with_cpu_set(&mut self.used_by, &dst_lock.tlb_ack), requested_span, unpin)?;
notify_files_out.append(&mut notify_files);
requested_span
@@ -403,24 +416,25 @@ impl AddrSpace {
selected_span.base,
page_flags(flags),
&mut self.table.utable,
&mut Flusher::with_cpu_set(&mut self.used_by),
&mut Flusher::with_cpu_set(&mut self.used_by, &dst_lock.tlb_ack),
)?;
self.grants.insert(grant);
Ok(selected_span.base)
}
pub fn r#move(
dst_lock: &AddrSpaceWrapper,
dst: &mut AddrSpace,
mut src_opt: Option<&mut AddrSpace>,
mut src_opt: Option<(&AddrSpaceWrapper, &mut AddrSpace)>,
src_span: PageSpan,
requested_dst_base: Option<Page>,
new_page_count: usize,
new_flags: MapFlags,
notify_files: &mut Vec<UnmapResult>,
) -> Result<Page> {
let mut src_owned_opt = src_opt.as_mut().map(|a| (&mut a.grants, &mut a.table.utable, Flusher::with_cpu_set(&mut a.used_by)));
let mut src_owned_opt = src_opt.as_mut().map(|(aw, a)| (&mut a.grants, &mut a.table.utable, Flusher::with_cpu_set(&mut a.used_by, &aw.tlb_ack)));
let mut src_opt = src_owned_opt.as_mut().map(|(g, m, f)| (&mut *g, &mut *m, &mut *f));
let mut dst_flusher = Flusher::with_cpu_set(&mut dst.used_by);
let mut dst_flusher = Flusher::with_cpu_set(&mut dst.used_by, &dst_lock.tlb_ack);
let dst_base = match requested_dst_base {
Some(base) if new_flags.contains(MapFlags::MAP_FIXED_NOREPLACE) => {
@@ -1155,13 +1169,14 @@ impl Grant {
new_flags: PageFlags<RmmA>,
file_ref: GrantFileRef,
src: Option<BorrowedFmapSource<'_>>,
lock: &AddrSpaceWrapper,
mapper: &mut PageMapper,
mut flusher: &mut Flusher,
) -> Result<Self> {
if let Some(src) = src {
let mut guard = src.addr_space_guard;
let mut src_addrspace = &mut *guard;
let mut src_flusher_state = Flusher::with_cpu_set(&mut src_addrspace.used_by).detach();
let mut src_flusher_state = Flusher::with_cpu_set(&mut src_addrspace.used_by, &lock.tlb_ack).detach();
for dst_page in span.pages() {
let src_page = src.src_base.next_by(dst_page.offset_from(span.base));
@@ -2463,37 +2478,62 @@ impl GenericFlusher for NopFlusher {
core::mem::forget(flush);
}
}
impl GenericFlusher for Flusher<'_> {
impl GenericFlusher for Flusher<'_, '_> {
fn consume(&mut self, flush: PageFlush<RmmA>) {
self.state.dirty = true;
unsafe {
flush.ignore();
}
}
}
struct FlusherState {
struct FlusherState<'addrsp> {
// TODO: Track *which* pages shall be flushed, to avoid expensive cr3 reloads. Linux defaults
// to approx. 32 INVLPGs before a CR3 reload is worth it.
dirty: bool,
ackword: &'addrsp AtomicU32,
}
pub struct Flusher<'asp> {
active_cpus: &'asp mut LogicalCpuSet,
state: FlusherState,
pub struct Flusher<'guard, 'addrsp> {
active_cpus: &'guard mut LogicalCpuSet,
state: FlusherState<'addrsp>,
}
impl<'asp> Flusher<'asp> {
fn with_cpu_set(set: &'asp mut LogicalCpuSet) -> Self {
impl<'guard, 'addrsp> Flusher<'guard, 'addrsp> {
fn with_cpu_set(set: &'guard mut LogicalCpuSet, ackword: &'addrsp AtomicU32) -> Self {
Self {
active_cpus: set,
state: FlusherState {
dirty: false,
ackword,
},
}
}
fn detach(mut self) -> FlusherState {
let state = core::mem::replace(&mut self.state, FlusherState { dirty: false });
fn detach(mut self) -> FlusherState<'addrsp> {
static DUMMY: AtomicU32 = AtomicU32::new(0);
let state = core::mem::replace(&mut self.state, FlusherState { dirty: false, ackword: &DUMMY });
core::mem::forget(self);
state
}
}
impl Drop for Flusher<'_> {
fn drop(&mut self) {
todo!()
// NOTE: Lock must be held, which must be guaranteed by the caller.
fn flush(&mut self) {
self.state.ackword.store(0, Ordering::Relaxed);
let mut affected_cpu_count = 0;
for cpu_id in self.active_cpus.iter_mut() {
crate::percpu::shootdown_tlb_ipi(Some(cpu_id));
affected_cpu_count += 1;
}
while self.state.ackword.load(Ordering::Relaxed) < affected_cpu_count {
core::hint::spin_loop();
}
self.state.dirty = false;
}
}
impl Drop for Flusher<'_, '_> {
fn drop(&mut self) {
self.flush();
}
}
src/cpu_set.rs
+11 -2
View File
@@ -35,10 +35,10 @@ impl core::fmt::Display for LogicalCpuId {
}
#[cfg(target_pointer_width = "64")]
const MAX_CPU_COUNT: u32 = 128;
pub const MAX_CPU_COUNT: u32 = 128;
#[cfg(target_pointer_width = "32")]
const MAX_CPU_COUNT: u32 = 32;
pub const MAX_CPU_COUNT: u32 = 32;
const SET_WORDS: usize = (MAX_CPU_COUNT / usize::BITS) as usize;
@@ -86,6 +86,15 @@ impl LogicalCpuSet {
pub fn to_raw(&self) -> RawMask {
self.0.each_ref().map(|w| w.load(Ordering::Acquire))
}
pub fn iter_mut(&mut self) -> impl Iterator<Item = LogicalCpuId> + '_ {
// TODO: Will this be optimized away?
self.0.iter_mut().enumerate().flat_map(move |(i, w)| (0..usize::BITS).filter_map(move |b| if *w.get_mut() & 1 << b != 0 {
Some(LogicalCpuId::new(i as u32 * usize::BITS + b))
} else {
None
}))
}
}
impl ToString for LogicalCpuSet {
src/percpu.rs
+49
View File
@@ -1,3 +1,6 @@
use core::sync::atomic::{AtomicUsize, AtomicPtr, Ordering};
use crate::cpu_set::MAX_CPU_COUNT;
use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId};
/// The percpu block, that stored all percpu variables.
@@ -8,10 +11,56 @@ pub struct PercpuBlock {
/// Context management
pub switch_internals: ContextSwitchPercpu,
// TODO: This lock can probably be relaxed further, but verify correctness first.
// The NMI lock. Can be set by any CPU, but can only be cleared by the CPU this percpu block
// refers to. Multiple flags can be set simultaneously, but NMI senders need to wait for the
// flag to be cleared if it was already set.
pub nmi_flags_lock: AtomicUsize,
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
// first to avoid cache invalidation.
#[cfg(feature = "profiling")]
pub profiling: Option<&'static crate::profiling::RingBuffer>,
}
const NULL: AtomicPtr<PercpuBlock> = AtomicPtr::new(core::ptr::null_mut());
static ALL_PERCPU_BLOCKS: [AtomicPtr<PercpuBlock>; MAX_CPU_COUNT as usize] = [NULL; MAX_CPU_COUNT as usize];
// PercpuBlock::current() is implemented somewhere in the arch-specific modules
bitflags::bitflags! {
struct NmiReasons: usize {
const TLB_SHOOTDOWN = 1;
// TODO: Profiling code wakes all CPUs up, so use a global and percpu ack counter for that.
//
//#[cfg(feature = "profiling")]
//const PROFILING = 1 << 32;
}
}
#[cfg(not(feature = "multi_core"))]
pub fn shootdown_tlb_ipi(_target: Option<LogicalCpuId>) {}
#[cfg(feature = "multi_core")]
pub fn shootdown_tlb_ipi(target: Option<LogicalCpuId>) {
if let Some(target) = target {
let Some(percpublock) = (unsafe { ALL_PERCPU_BLOCKS[target.get() as usize].load(Ordering::Acquire).as_ref() }) else {
return;
};
let bit = NmiReasons::TLB_SHOOTDOWN.bits();
while percpublock.nmi_flags_lock.fetch_or(bit, Ordering::Release) & bit == bit {
// Load is faster than CAS or on x86, LOCK BTS
while percpublock.nmi_flags_lock.load(Ordering::Relaxed) & bit == bit {
core::hint::spin_loop();
}
}
} else {
for id in 0..crate::cpu_count() {
// TODO: Optimize: use global counter and percpu ack counters.
shootdown_tlb_ipi(Some(LogicalCpuId::new(id)));
}
}
}
src/scheme/memory.rs
+7 -4
View File
@@ -84,6 +84,7 @@ impl MemoryScheme {
}
let page = addr_space.inner.write().mmap(
&addr_space,
(map.address != 0).then_some(span.base),
page_count,
map.flags,
@@ -131,10 +132,12 @@ impl MemoryScheme {
}
let page_count = NonZeroUsize::new(size.div_ceil(PAGE_SIZE)).ok_or(Error::new(EINVAL))?;
AddrSpace::current()?
.inner
let current_addrsp = AddrSpace::current()?;
let base_page = current_addrsp.inner
.write()
.mmap_anywhere(
&current_addrsp,
page_count,
flags,
|dst_page, mut page_flags, dst_mapper, dst_flusher| {
@@ -169,8 +172,8 @@ impl MemoryScheme {
dst_flusher,
)
},
)
.map(|page| page.start_address().data())
)?;
Ok(base_page.start_address().data())
}
}
impl KernelScheme for MemoryScheme {
src/scheme/proc.rs
+9 -11
View File
@@ -718,7 +718,7 @@ impl<const FULL: bool> KernelScheme for ProcScheme<FULL> {
let requested_dst_base = (map.address != 0).then_some(requested_dst_page);
let mut src_addr_space = addrspace.inner.write();
let mut dst_addr_space = dst_addr_space.inner.write();
let mut dst_addrsp_guard = dst_addr_space.inner.write();
let src_page_count = NonZeroUsize::new(src_span.count).ok_or(Error::new(EINVAL))?;
@@ -727,8 +727,9 @@ impl<const FULL: bool> KernelScheme for ProcScheme<FULL> {
// TODO: Validate flags
let result_base = if consume {
AddrSpace::r#move(
&mut *dst_addr_space,
Some(&mut *src_addr_space),
&*dst_addr_space,
&mut *dst_addrsp_guard,
Some((&addrspace, &mut *src_addr_space)),
src_span,
requested_dst_base,
src_page_count.get(),
@@ -736,7 +737,8 @@ impl<const FULL: bool> KernelScheme for ProcScheme<FULL> {
&mut notify_files,
)?
} else {
dst_addr_space.mmap(
dst_addrsp_guard.mmap(
&dst_addr_space,
requested_dst_base,
src_page_count,
map.flags,
@@ -1052,18 +1054,14 @@ impl<const FULL: bool> KernelScheme for ProcScheme<FULL> {
crate::syscall::validate_region(next()??, next()??)?;
let unpin = false;
addrspace
.inner.write()
.munmap(PageSpan::new(page, page_count), unpin)?;
addrspace.munmap(PageSpan::new(page, page_count), unpin)?;
}
ADDRSPACE_OP_MPROTECT => {
let (page, page_count) =
crate::syscall::validate_region(next()??, next()??)?;
let flags = MapFlags::from_bits(next()??).ok_or(Error::new(EINVAL))?;
addrspace
.inner.write()
.mprotect(PageSpan::new(page, page_count), flags)?;
addrspace.mprotect(PageSpan::new(page, page_count), flags)?;
}
_ => return Err(Error::new(EINVAL)),
}
@@ -1444,7 +1442,7 @@ impl<const FULL: bool> KernelScheme for ProcScheme<FULL> {
addrspace: AddrSpaceWrapper::new()?,
},
b"exclusive" => Operation::AddrSpace {
addrspace: addrspace.inner.write().try_clone()?,
addrspace: addrspace.try_clone()?,
},
b"mmap-min-addr" => Operation::MmapMinAddr(Arc::clone(addrspace)),
src/scheme/user.rs
+10 -6
View File
@@ -243,6 +243,7 @@ impl UserInner {
let is_pinned = true;
let dst_page = dst_addr_space.inner.write().mmap_anywhere(
&dst_addr_space,
ONE,
PROT_READ,
|dst_page, flags, mapper, flusher| {
@@ -375,6 +376,7 @@ impl UserInner {
}
dst_space.mmap(
&dst_space_lock,
Some(free_span.base),
ONE,
map_flags | MAP_FIXED_NOREPLACE,
@@ -413,7 +415,9 @@ impl UserInner {
.expect("split must succeed");
if let Some(middle_page_count) = NonZeroUsize::new(middle_page_count) {
let cur_space_lock_clone = Arc::clone(&cur_space_lock);
dst_space.mmap(
&cur_space_lock,
Some(first_middle_dst_page),
middle_page_count,
map_flags | MAP_FIXED_NOREPLACE,
@@ -433,8 +437,8 @@ impl UserInner {
let is_pinned_userscheme_borrow = true;
Ok(Grant::borrow(
Arc::clone(&cur_space_lock),
&mut *cur_space_lock.inner.write(),
Arc::clone(&cur_space_lock_clone),
&mut *cur_space_lock_clone.inner.write(),
first_middle_src_page,
dst_page,
middle_page_count.get(),
@@ -473,6 +477,7 @@ impl UserInner {
}
dst_space.mmap(
&dst_space_lock,
Some(tail_dst_page),
ONE,
map_flags | MAP_FIXED_NOREPLACE,
@@ -851,6 +856,7 @@ impl UserInner {
let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0");
let mut notify_files = Vec::new();
let dst_base = dst_addr_space.inner.write().mmap(
&dst_addr_space,
dst_base,
page_count_nz,
map.flags,
@@ -861,6 +867,7 @@ impl UserInner {
flags,
file_ref,
src,
&dst_addr_space,
mapper,
flusher,
)
@@ -935,10 +942,7 @@ impl<const READ: bool, const WRITE: bool> CaptureGuard<READ, WRITE> {
let (first_page, page_count, _offset) = page_range_containing(self.base, self.len);
let unpin = true;
space
.inner
.write()
.munmap(PageSpan::new(first_page, page_count), unpin)?;
space.munmap(PageSpan::new(first_page, page_count), unpin)?;
result
}
src/syscall/fs.rs
+2 -1
View File
@@ -433,7 +433,7 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result<usize> {
let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned)
.ok_or(Error::new(EINVAL))?;
let unpin = false;
let notify = addr_space.inner.write().munmap(span, unpin)?;
let notify = addr_space.munmap(span, unpin)?;
for map in notify {
let _ = map.unmap();
@@ -483,6 +483,7 @@ pub fn mremap(
let mut guard = addr_space.inner.write();
let base = AddrSpace::r#move(
&addr_space,
&mut *guard,
None,
src_span,
src/syscall/mod.rs
+1 -1
View File
@@ -200,7 +200,7 @@ pub fn syscall(
SYS_GETGID => getgid(),
SYS_GETNS => getns(),
SYS_GETUID => getuid(),
SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)),
SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)).map(|()| 0),
SYS_MKNS => mkns(UserSlice::ro(
b,
c.checked_mul(core::mem::size_of::<[usize; 2]>())
src/syscall/process.rs
+3 -7
View File
@@ -254,17 +254,13 @@ pub fn kill(pid: ContextId, sig: usize) -> Result<usize> {
}
}
pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<usize> {
pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<()> {
// println!("mprotect {:#X}, {}, {:#X}", address, size, flags);
let span = PageSpan::validate_nonempty(VirtualAddress::new(address), size)
.ok_or(Error::new(EINVAL))?;
AddrSpace::current()?
.inner
.write()
.mprotect(span, flags)
.map(|()| 0)
AddrSpace::current()?.mprotect(span, flags)
}
pub fn setpgid(pid: ContextId, pgid: ContextId) -> Result<usize> {
@@ -591,7 +587,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! {
let page_count = NonZeroUsize::new(bootstrap.page_count)
.expect("bootstrap contained no pages!");
let _base_page = addr_space.inner.write().mmap(Some(base), page_count, flags, &mut Vec::new(), |page, flags, mapper, flusher| {
let _base_page = addr_space.inner.write().mmap(&addr_space, Some(base), page_count, flags, &mut Vec::new(), |page, flags, mapper, flusher| {
let shared = false;
Ok(Grant::zeroed(PageSpan::new(page, bootstrap.page_count), flags, mapper, flusher, shared)?)
});