diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index 3529580eb5..f611d351fe 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,6 +1,6 @@ //! Global descriptor table -use core::{convert::TryInto, mem}; +use core::{convert::TryInto, mem, sync::atomic::AtomicUsize}; use crate::{ cpu_set::LogicalCpuId, @@ -239,6 +239,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { pcr.percpu = PercpuBlock { cpu_id, switch_internals: Default::default(), + nmi_flags_lock: AtomicUsize::new(0), #[cfg(feature = "profiling")] profiling: None, diff --git a/src/arch/x86_shared/ipi.rs b/src/arch/x86_shared/ipi.rs index d5b11df906..b453c0b695 100644 --- a/src/arch/x86_shared/ipi.rs +++ b/src/arch/x86_shared/ipi.rs @@ -37,3 +37,18 @@ pub fn ipi(kind: IpiKind, target: IpiTarget) { let icr = (target as u64) << 18 | 1 << 14 | (kind as u64); unsafe { LOCAL_APIC.set_icr(icr) }; } +#[cfg(feature = "multi_core")] +#[inline(always)] +pub fn ipi_single(kind: IpiKind, target: u32) { + use crate::device::local_apic::LOCAL_APIC; + + #[cfg(feature = "profiling")] + if matches!(kind, IpiKind::Profile) { + let icr = (target as u64) << 18 | 1 << 14 | 0b100 << 8; + unsafe { LOCAL_APIC.set_icr(icr) }; + return; + } + + let icr = u64::from(target) << 18 | 1 << 14 | (kind as u64); + unsafe { LOCAL_APIC.set_icr(icr) }; +} diff --git a/src/context/memory.rs b/src/context/memory.rs index 9372950412..5a40f30663 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -98,22 +98,21 @@ pub struct AddrSpace { /// against null pointers, so fixed mmaps to address zero are still allowed. pub mmap_min: usize, } -impl AddrSpace { - pub fn current() -> Result> { - Ok(Arc::clone(super::current()?.read().addr_space()?)) - } - +impl AddrSpaceWrapper { /// Attempt to clone an existing address space so that all mappings are copied (CoW). - pub fn try_clone(&mut self) -> Result> { + pub fn try_clone(&self) -> Result> { + let mut guard = self.inner.write(); + let guard = &mut *guard; + let mut new_arc = AddrSpaceWrapper::new()?; let new = Arc::get_mut(&mut new_arc) .expect("expected new address space Arc not to be aliased"); - let this_mapper = &mut self.table.utable; - let mut this_flusher = Flusher::with_cpu_set(&mut self.used_by); + let this_mapper = &mut guard.table.utable; + let mut this_flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack); - for (grant_base, grant_info) in self.grants.iter() { + for (grant_base, grant_info) in guard.grants.iter() { let new_grant = match grant_info.provider { // No, your temporary UserScheme mappings will not be kept across forks. Provider::External { @@ -191,23 +190,15 @@ impl AddrSpace { } Ok(new_arc) } - pub fn new() -> Result { - Ok(Self { - grants: UserGrants::new(), - table: setup_new_utable()?, - mmap_min: MMAP_MIN_DEFAULT, - used_by: LogicalCpuSet::empty(), - }) - } - pub fn is_current(&self) -> bool { - self.table.utable.is_current() - } - pub fn mprotect(&mut self, requested_span: PageSpan, flags: MapFlags) -> Result<()> { - let mapper = &mut self.table.utable; - let mut flusher = Flusher::with_cpu_set(&mut self.used_by); + pub fn mprotect(&self, requested_span: PageSpan, flags: MapFlags) -> Result<()> { + let mut guard = self.inner.write(); + let guard = &mut *guard; + + let mapper = &mut guard.table.utable; + let mut flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack); // TODO: Remove allocation (might require BTreeMap::set_key or interior mutability). - let regions = self + let regions = guard .grants .conflicts(requested_span) .map(|(base, info)| { @@ -222,7 +213,7 @@ impl AddrSpace { for grant_span_res in regions { let grant_span = grant_span_res?; - let grant = self + let grant = guard .grants .remove(grant_span.base) .expect("grant cannot magically disappear while we hold the lock!"); @@ -235,14 +226,14 @@ impl AddrSpace { //log::info!("Sliced into\n\n{:#?}\n\n{:#?}\n\n{:#?}", before, grant, after); if let Some(before) = before { - self.grants.insert(before); + guard.grants.insert(before); } if let Some(after) = after { - self.grants.insert(after); + guard.grants.insert(after); } if !grant.info.can_have_flags(flags) { - self.grants.insert(grant); + guard.grants.insert(grant); return Err(Error::new(EACCES)); } @@ -259,18 +250,38 @@ impl AddrSpace { grant.remap(mapper, &mut flusher, new_flags); //log::info!("Mprotect grant became {:#?}", grant); - self.grants.insert(grant); + guard.grants.insert(grant); } Ok(()) } #[must_use = "needs to notify files"] pub fn munmap( - &mut self, + &self, mut requested_span: PageSpan, unpin: bool, ) -> Result> { - let mut flusher = Flusher::with_cpu_set(&mut self.used_by); - Self::munmap_inner(&mut self.grants, &mut self.table.utable, &mut flusher, requested_span, unpin) + let mut guard = self.inner.write(); + let guard = &mut *guard; + + let mut flusher = Flusher::with_cpu_set(&mut guard.used_by, &self.tlb_ack); + AddrSpace::munmap_inner(&mut guard.grants, &mut guard.table.utable, &mut flusher, requested_span, unpin) + } +} +impl AddrSpace { + pub fn current() -> Result> { + Ok(Arc::clone(super::current()?.read().addr_space()?)) + } + + pub fn new() -> Result { + Ok(Self { + grants: UserGrants::new(), + table: setup_new_utable()?, + mmap_min: MMAP_MIN_DEFAULT, + used_by: LogicalCpuSet::empty(), + }) + } + pub fn is_current(&self) -> bool { + self.table.utable.is_current() } fn munmap_inner( this_grants: &mut UserGrants, @@ -341,6 +352,7 @@ impl AddrSpace { } pub fn mmap_anywhere( &mut self, + dst_lock: &AddrSpaceWrapper, page_count: NonZeroUsize, flags: MapFlags, map: impl FnOnce( @@ -350,10 +362,11 @@ impl AddrSpace { &mut Flusher, ) -> Result, ) -> Result { - self.mmap(None, page_count, flags, &mut Vec::new(), map) + self.mmap(dst_lock, None, page_count, flags, &mut Vec::new(), map) } pub fn mmap( &mut self, + dst_lock: &AddrSpaceWrapper, requested_base_opt: Option, page_count: NonZeroUsize, flags: MapFlags, @@ -378,7 +391,7 @@ impl AddrSpace { requested_span } else if flags.contains(MapFlags::MAP_FIXED) { let unpin = false; - let mut notify_files = self.munmap(requested_span, unpin)?; + let mut notify_files = Self::munmap_inner(&mut self.grants, &mut self.table.utable, &mut Flusher::with_cpu_set(&mut self.used_by, &dst_lock.tlb_ack), requested_span, unpin)?; notify_files_out.append(&mut notify_files); requested_span @@ -403,24 +416,25 @@ impl AddrSpace { selected_span.base, page_flags(flags), &mut self.table.utable, - &mut Flusher::with_cpu_set(&mut self.used_by), + &mut Flusher::with_cpu_set(&mut self.used_by, &dst_lock.tlb_ack), )?; self.grants.insert(grant); Ok(selected_span.base) } pub fn r#move( + dst_lock: &AddrSpaceWrapper, dst: &mut AddrSpace, - mut src_opt: Option<&mut AddrSpace>, + mut src_opt: Option<(&AddrSpaceWrapper, &mut AddrSpace)>, src_span: PageSpan, requested_dst_base: Option, new_page_count: usize, new_flags: MapFlags, notify_files: &mut Vec, ) -> Result { - let mut src_owned_opt = src_opt.as_mut().map(|a| (&mut a.grants, &mut a.table.utable, Flusher::with_cpu_set(&mut a.used_by))); + let mut src_owned_opt = src_opt.as_mut().map(|(aw, a)| (&mut a.grants, &mut a.table.utable, Flusher::with_cpu_set(&mut a.used_by, &aw.tlb_ack))); let mut src_opt = src_owned_opt.as_mut().map(|(g, m, f)| (&mut *g, &mut *m, &mut *f)); - let mut dst_flusher = Flusher::with_cpu_set(&mut dst.used_by); + let mut dst_flusher = Flusher::with_cpu_set(&mut dst.used_by, &dst_lock.tlb_ack); let dst_base = match requested_dst_base { Some(base) if new_flags.contains(MapFlags::MAP_FIXED_NOREPLACE) => { @@ -1155,13 +1169,14 @@ impl Grant { new_flags: PageFlags, file_ref: GrantFileRef, src: Option>, + lock: &AddrSpaceWrapper, mapper: &mut PageMapper, mut flusher: &mut Flusher, ) -> Result { if let Some(src) = src { let mut guard = src.addr_space_guard; let mut src_addrspace = &mut *guard; - let mut src_flusher_state = Flusher::with_cpu_set(&mut src_addrspace.used_by).detach(); + let mut src_flusher_state = Flusher::with_cpu_set(&mut src_addrspace.used_by, &lock.tlb_ack).detach(); for dst_page in span.pages() { let src_page = src.src_base.next_by(dst_page.offset_from(span.base)); @@ -2463,37 +2478,62 @@ impl GenericFlusher for NopFlusher { core::mem::forget(flush); } } -impl GenericFlusher for Flusher<'_> { +impl GenericFlusher for Flusher<'_, '_> { fn consume(&mut self, flush: PageFlush) { + self.state.dirty = true; + + unsafe { + flush.ignore(); + } } } -struct FlusherState { +struct FlusherState<'addrsp> { // TODO: Track *which* pages shall be flushed, to avoid expensive cr3 reloads. Linux defaults // to approx. 32 INVLPGs before a CR3 reload is worth it. dirty: bool, + ackword: &'addrsp AtomicU32, } -pub struct Flusher<'asp> { - active_cpus: &'asp mut LogicalCpuSet, - state: FlusherState, +pub struct Flusher<'guard, 'addrsp> { + active_cpus: &'guard mut LogicalCpuSet, + state: FlusherState<'addrsp>, } -impl<'asp> Flusher<'asp> { - fn with_cpu_set(set: &'asp mut LogicalCpuSet) -> Self { +impl<'guard, 'addrsp> Flusher<'guard, 'addrsp> { + fn with_cpu_set(set: &'guard mut LogicalCpuSet, ackword: &'addrsp AtomicU32) -> Self { Self { active_cpus: set, state: FlusherState { dirty: false, + ackword, }, } } - fn detach(mut self) -> FlusherState { - let state = core::mem::replace(&mut self.state, FlusherState { dirty: false }); + fn detach(mut self) -> FlusherState<'addrsp> { + static DUMMY: AtomicU32 = AtomicU32::new(0); + let state = core::mem::replace(&mut self.state, FlusherState { dirty: false, ackword: &DUMMY }); core::mem::forget(self); state } -} -impl Drop for Flusher<'_> { - fn drop(&mut self) { - todo!() + // NOTE: Lock must be held, which must be guaranteed by the caller. + fn flush(&mut self) { + self.state.ackword.store(0, Ordering::Relaxed); + + let mut affected_cpu_count = 0; + + for cpu_id in self.active_cpus.iter_mut() { + crate::percpu::shootdown_tlb_ipi(Some(cpu_id)); + affected_cpu_count += 1; + } + + while self.state.ackword.load(Ordering::Relaxed) < affected_cpu_count { + core::hint::spin_loop(); + } + + self.state.dirty = false; + } +} +impl Drop for Flusher<'_, '_> { + fn drop(&mut self) { + self.flush(); } } diff --git a/src/cpu_set.rs b/src/cpu_set.rs index 40aeb928cc..54180eaf34 100644 --- a/src/cpu_set.rs +++ b/src/cpu_set.rs @@ -35,10 +35,10 @@ impl core::fmt::Display for LogicalCpuId { } #[cfg(target_pointer_width = "64")] -const MAX_CPU_COUNT: u32 = 128; +pub const MAX_CPU_COUNT: u32 = 128; #[cfg(target_pointer_width = "32")] -const MAX_CPU_COUNT: u32 = 32; +pub const MAX_CPU_COUNT: u32 = 32; const SET_WORDS: usize = (MAX_CPU_COUNT / usize::BITS) as usize; @@ -86,6 +86,15 @@ impl LogicalCpuSet { pub fn to_raw(&self) -> RawMask { self.0.each_ref().map(|w| w.load(Ordering::Acquire)) } + + pub fn iter_mut(&mut self) -> impl Iterator + '_ { + // TODO: Will this be optimized away? + self.0.iter_mut().enumerate().flat_map(move |(i, w)| (0..usize::BITS).filter_map(move |b| if *w.get_mut() & 1 << b != 0 { + Some(LogicalCpuId::new(i as u32 * usize::BITS + b)) + } else { + None + })) + } } impl ToString for LogicalCpuSet { diff --git a/src/percpu.rs b/src/percpu.rs index 43810e0475..fd2684c285 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,3 +1,6 @@ +use core::sync::atomic::{AtomicUsize, AtomicPtr, Ordering}; + +use crate::cpu_set::MAX_CPU_COUNT; use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId}; /// The percpu block, that stored all percpu variables. @@ -8,10 +11,56 @@ pub struct PercpuBlock { /// Context management pub switch_internals: ContextSwitchPercpu, + // TODO: This lock can probably be relaxed further, but verify correctness first. + + // The NMI lock. Can be set by any CPU, but can only be cleared by the CPU this percpu block + // refers to. Multiple flags can be set simultaneously, but NMI senders need to wait for the + // flag to be cleared if it was already set. + pub nmi_flags_lock: AtomicUsize, + // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. #[cfg(feature = "profiling")] pub profiling: Option<&'static crate::profiling::RingBuffer>, } +const NULL: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); +static ALL_PERCPU_BLOCKS: [AtomicPtr; MAX_CPU_COUNT as usize] = [NULL; MAX_CPU_COUNT as usize]; + // PercpuBlock::current() is implemented somewhere in the arch-specific modules + +bitflags::bitflags! { + struct NmiReasons: usize { + const TLB_SHOOTDOWN = 1; + + // TODO: Profiling code wakes all CPUs up, so use a global and percpu ack counter for that. + // + //#[cfg(feature = "profiling")] + //const PROFILING = 1 << 32; + } +} + +#[cfg(not(feature = "multi_core"))] +pub fn shootdown_tlb_ipi(_target: Option) {} + +#[cfg(feature = "multi_core")] +pub fn shootdown_tlb_ipi(target: Option) { + if let Some(target) = target { + let Some(percpublock) = (unsafe { ALL_PERCPU_BLOCKS[target.get() as usize].load(Ordering::Acquire).as_ref() }) else { + return; + }; + let bit = NmiReasons::TLB_SHOOTDOWN.bits(); + + while percpublock.nmi_flags_lock.fetch_or(bit, Ordering::Release) & bit == bit { + // Load is faster than CAS or on x86, LOCK BTS + while percpublock.nmi_flags_lock.load(Ordering::Relaxed) & bit == bit { + core::hint::spin_loop(); + } + } + } else { + for id in 0..crate::cpu_count() { + // TODO: Optimize: use global counter and percpu ack counters. + shootdown_tlb_ipi(Some(LogicalCpuId::new(id))); + } + } +} diff --git a/src/scheme/memory.rs b/src/scheme/memory.rs index 982d449489..40ec468ca4 100644 --- a/src/scheme/memory.rs +++ b/src/scheme/memory.rs @@ -84,6 +84,7 @@ impl MemoryScheme { } let page = addr_space.inner.write().mmap( + &addr_space, (map.address != 0).then_some(span.base), page_count, map.flags, @@ -131,10 +132,12 @@ impl MemoryScheme { } let page_count = NonZeroUsize::new(size.div_ceil(PAGE_SIZE)).ok_or(Error::new(EINVAL))?; - AddrSpace::current()? - .inner + let current_addrsp = AddrSpace::current()?; + + let base_page = current_addrsp.inner .write() .mmap_anywhere( + ¤t_addrsp, page_count, flags, |dst_page, mut page_flags, dst_mapper, dst_flusher| { @@ -169,8 +172,8 @@ impl MemoryScheme { dst_flusher, ) }, - ) - .map(|page| page.start_address().data()) + )?; + Ok(base_page.start_address().data()) } } impl KernelScheme for MemoryScheme { diff --git a/src/scheme/proc.rs b/src/scheme/proc.rs index be1a61c814..24b6f5a6ab 100644 --- a/src/scheme/proc.rs +++ b/src/scheme/proc.rs @@ -718,7 +718,7 @@ impl KernelScheme for ProcScheme { let requested_dst_base = (map.address != 0).then_some(requested_dst_page); let mut src_addr_space = addrspace.inner.write(); - let mut dst_addr_space = dst_addr_space.inner.write(); + let mut dst_addrsp_guard = dst_addr_space.inner.write(); let src_page_count = NonZeroUsize::new(src_span.count).ok_or(Error::new(EINVAL))?; @@ -727,8 +727,9 @@ impl KernelScheme for ProcScheme { // TODO: Validate flags let result_base = if consume { AddrSpace::r#move( - &mut *dst_addr_space, - Some(&mut *src_addr_space), + &*dst_addr_space, + &mut *dst_addrsp_guard, + Some((&addrspace, &mut *src_addr_space)), src_span, requested_dst_base, src_page_count.get(), @@ -736,7 +737,8 @@ impl KernelScheme for ProcScheme { &mut notify_files, )? } else { - dst_addr_space.mmap( + dst_addrsp_guard.mmap( + &dst_addr_space, requested_dst_base, src_page_count, map.flags, @@ -1052,18 +1054,14 @@ impl KernelScheme for ProcScheme { crate::syscall::validate_region(next()??, next()??)?; let unpin = false; - addrspace - .inner.write() - .munmap(PageSpan::new(page, page_count), unpin)?; + addrspace.munmap(PageSpan::new(page, page_count), unpin)?; } ADDRSPACE_OP_MPROTECT => { let (page, page_count) = crate::syscall::validate_region(next()??, next()??)?; let flags = MapFlags::from_bits(next()??).ok_or(Error::new(EINVAL))?; - addrspace - .inner.write() - .mprotect(PageSpan::new(page, page_count), flags)?; + addrspace.mprotect(PageSpan::new(page, page_count), flags)?; } _ => return Err(Error::new(EINVAL)), } @@ -1444,7 +1442,7 @@ impl KernelScheme for ProcScheme { addrspace: AddrSpaceWrapper::new()?, }, b"exclusive" => Operation::AddrSpace { - addrspace: addrspace.inner.write().try_clone()?, + addrspace: addrspace.try_clone()?, }, b"mmap-min-addr" => Operation::MmapMinAddr(Arc::clone(addrspace)), diff --git a/src/scheme/user.rs b/src/scheme/user.rs index 451501793f..0c45d9a841 100644 --- a/src/scheme/user.rs +++ b/src/scheme/user.rs @@ -243,6 +243,7 @@ impl UserInner { let is_pinned = true; let dst_page = dst_addr_space.inner.write().mmap_anywhere( + &dst_addr_space, ONE, PROT_READ, |dst_page, flags, mapper, flusher| { @@ -375,6 +376,7 @@ impl UserInner { } dst_space.mmap( + &dst_space_lock, Some(free_span.base), ONE, map_flags | MAP_FIXED_NOREPLACE, @@ -413,7 +415,9 @@ impl UserInner { .expect("split must succeed"); if let Some(middle_page_count) = NonZeroUsize::new(middle_page_count) { + let cur_space_lock_clone = Arc::clone(&cur_space_lock); dst_space.mmap( + &cur_space_lock, Some(first_middle_dst_page), middle_page_count, map_flags | MAP_FIXED_NOREPLACE, @@ -433,8 +437,8 @@ impl UserInner { let is_pinned_userscheme_borrow = true; Ok(Grant::borrow( - Arc::clone(&cur_space_lock), - &mut *cur_space_lock.inner.write(), + Arc::clone(&cur_space_lock_clone), + &mut *cur_space_lock_clone.inner.write(), first_middle_src_page, dst_page, middle_page_count.get(), @@ -473,6 +477,7 @@ impl UserInner { } dst_space.mmap( + &dst_space_lock, Some(tail_dst_page), ONE, map_flags | MAP_FIXED_NOREPLACE, @@ -851,6 +856,7 @@ impl UserInner { let page_count_nz = NonZeroUsize::new(page_count).expect("already validated map.size != 0"); let mut notify_files = Vec::new(); let dst_base = dst_addr_space.inner.write().mmap( + &dst_addr_space, dst_base, page_count_nz, map.flags, @@ -861,6 +867,7 @@ impl UserInner { flags, file_ref, src, + &dst_addr_space, mapper, flusher, ) @@ -935,10 +942,7 @@ impl CaptureGuard { let (first_page, page_count, _offset) = page_range_containing(self.base, self.len); let unpin = true; - space - .inner - .write() - .munmap(PageSpan::new(first_page, page_count), unpin)?; + space.munmap(PageSpan::new(first_page, page_count), unpin)?; result } diff --git a/src/syscall/fs.rs b/src/syscall/fs.rs index 19a8841abb..0b4e8b4b1d 100644 --- a/src/syscall/fs.rs +++ b/src/syscall/fs.rs @@ -433,7 +433,7 @@ pub fn funmap(virtual_address: usize, length: usize) -> Result { let span = PageSpan::validate_nonempty(VirtualAddress::new(virtual_address), length_aligned) .ok_or(Error::new(EINVAL))?; let unpin = false; - let notify = addr_space.inner.write().munmap(span, unpin)?; + let notify = addr_space.munmap(span, unpin)?; for map in notify { let _ = map.unmap(); @@ -483,6 +483,7 @@ pub fn mremap( let mut guard = addr_space.inner.write(); let base = AddrSpace::r#move( + &addr_space, &mut *guard, None, src_span, diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index e16971c1c8..f65a39b9a8 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -200,7 +200,7 @@ pub fn syscall( SYS_GETGID => getgid(), SYS_GETNS => getns(), SYS_GETUID => getuid(), - SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)), + SYS_MPROTECT => mprotect(b, c, MapFlags::from_bits_truncate(d)).map(|()| 0), SYS_MKNS => mkns(UserSlice::ro( b, c.checked_mul(core::mem::size_of::<[usize; 2]>()) diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 641dba424c..2c0c5ce2b4 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -254,17 +254,13 @@ pub fn kill(pid: ContextId, sig: usize) -> Result { } } -pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result { +pub fn mprotect(address: usize, size: usize, flags: MapFlags) -> Result<()> { // println!("mprotect {:#X}, {}, {:#X}", address, size, flags); let span = PageSpan::validate_nonempty(VirtualAddress::new(address), size) .ok_or(Error::new(EINVAL))?; - AddrSpace::current()? - .inner - .write() - .mprotect(span, flags) - .map(|()| 0) + AddrSpace::current()?.mprotect(span, flags) } pub fn setpgid(pid: ContextId, pgid: ContextId) -> Result { @@ -591,7 +587,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! { let page_count = NonZeroUsize::new(bootstrap.page_count) .expect("bootstrap contained no pages!"); - let _base_page = addr_space.inner.write().mmap(Some(base), page_count, flags, &mut Vec::new(), |page, flags, mapper, flusher| { + let _base_page = addr_space.inner.write().mmap(&addr_space, Some(base), page_count, flags, &mut Vec::new(), |page, flags, mapper, flusher| { let shared = false; Ok(Grant::zeroed(PageSpan::new(page, bootstrap.page_count), flags, mapper, flusher, shared)?) });