From 79381249a2265462e5a6080f06cbfeb17d4cb47a Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Mon, 26 Feb 2024 23:47:36 +0100 Subject: [PATCH] Implement a TLB shootdown draft. --- Cargo.lock | 109 ++++++++++++------------- Cargo.toml | 2 +- src/arch/x86_64/alternative.rs | 1 + src/arch/x86_64/gdt.rs | 13 +-- src/arch/x86_64/interrupt/exception.rs | 28 +++++-- src/arch/x86_64/mod.rs | 8 ++ src/arch/x86_shared/idt.rs | 2 +- src/context/arch/x86_64.rs | 7 +- src/context/memory.rs | 4 + src/memory/mod.rs | 1 + src/percpu.rs | 9 +- src/syscall/process.rs | 2 +- 12 files changed, 113 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74337ae449..e6a15e4a24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f" dependencies = [ "cfg-if", "once_cell", @@ -22,9 +22,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bit_field" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" [[package]] name = "bitfield" @@ -40,21 +40,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" [[package]] name = "cfg-if" @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown", @@ -110,7 +110,7 @@ name = "kernel" version = "0.4.1" dependencies = [ "bitfield", - "bitflags 1.3.2", + "bitflags 2.4.2", "byteorder", "cc", "fdt", @@ -147,14 +147,14 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "549ce1740e46b291953c4340adcd74c59bcf4308f4cac050fd33ba91b7168f4a" dependencies = [ - "spinning_top 0.2.4", + "spinning_top 0.2.5", ] [[package]] name = "lock_api" -version = "0.4.8" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -162,18 +162,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "once_cell" @@ -183,9 +180,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "paste" -version = "1.0.9" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "plain" @@ -195,27 +192,27 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] [[package]] name = "raw-cpuid" -version = "10.5.0" +version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ "bitflags 1.3.2", ] @@ -230,7 +227,7 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7" name = "redox_syscall" version = "0.4.1" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", ] [[package]] @@ -245,15 +242,15 @@ checksum = "56a596b5718bf5e059d59a30af12f7f462a152de147aa462b70892849ee18704" [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scroll" @@ -263,18 +260,18 @@ checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", @@ -283,9 +280,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -321,9 +318,9 @@ dependencies = [ [[package]] name = "spinning_top" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75adad84ee84b521fb2cca2d4fd0f1dab1d8d026bda3c5bea4ca63b5f9f9293c" +checksum = "5b9eb1a2f4c41445a3a0ff9abc5221c5fcd28e1f13cd7c0397706f9ac938ddb0" dependencies = [ "lock_api", ] @@ -339,9 +336,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c" dependencies = [ "proc-macro2", "quote", @@ -362,9 +359,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" dependencies = [ "serde", ] @@ -384,9 +381,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "version_check" @@ -396,9 +393,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "winnow" -version = "0.5.15" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -416,18 +413,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 59fa0b5a1b..9a60a8163e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ rustc-cfg = "0.3.0" toml = "0.7" [dependencies] -bitflags = "1.2.1" +bitflags = "2" bitfield = "0.13.2" hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] } linked_list_allocator = "0.9.0" diff --git a/src/arch/x86_64/alternative.rs b/src/arch/x86_64/alternative.rs index 496f5189f0..bb4576f40d 100644 --- a/src/arch/x86_64/alternative.rs +++ b/src/arch/x86_64/alternative.rs @@ -222,6 +222,7 @@ unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) { } bitflags! { + #[derive(Clone, Copy, Debug)] pub struct KcpuFeatures: usize { const SMAP = 1; const FSGSBASE = 2; diff --git a/src/arch/x86_64/gdt.rs b/src/arch/x86_64/gdt.rs index f611d351fe..53a672665b 100644 --- a/src/arch/x86_64/gdt.rs +++ b/src/arch/x86_64/gdt.rs @@ -1,6 +1,6 @@ //! Global descriptor table -use core::{convert::TryInto, mem, sync::atomic::AtomicUsize}; +use core::{convert::TryInto, mem, sync::atomic::AtomicUsize, cell::Cell}; use crate::{ cpu_set::LogicalCpuId, @@ -140,17 +140,17 @@ pub unsafe fn pcr() -> *mut ProcessorControlRegion { } #[cfg(feature = "pti")] -pub unsafe fn set_tss_stack(stack: usize) { +pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) { use super::pti::{PTI_CONTEXT_STACK, PTI_CPU_STACK}; - core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]) + core::ptr::addr_of_mut!((*pcr).tss.rsp[0]) .write_unaligned((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64); PTI_CONTEXT_STACK = stack; } #[cfg(not(feature = "pti"))] -pub unsafe fn set_tss_stack(stack: usize) { +pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) { // TODO: If this increases performance, read gs:[offset] directly - core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write_unaligned(stack as u64); + core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64); } // Initialize startup GDT @@ -231,7 +231,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0); // Set the stack pointer to use when coming back from userspace. - set_tss_stack(stack_offset); + set_tss_stack(pcr, stack_offset); // Load the task register task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0)); @@ -240,6 +240,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) { cpu_id, switch_internals: Default::default(), nmi_flags_lock: AtomicUsize::new(0), + current_addrspace: Cell::new(core::ptr::null_mut()), #[cfg(feature = "profiling")] profiling: None, diff --git a/src/arch/x86_64/interrupt/exception.rs b/src/arch/x86_64/interrupt/exception.rs index 7162b3277b..2e8d3f363e 100644 --- a/src/arch/x86_64/interrupt/exception.rs +++ b/src/arch/x86_64/interrupt/exception.rs @@ -1,5 +1,8 @@ +use core::sync::atomic::Ordering; + use x86::irq::PageFaultError; +use crate::percpu::{PercpuBlock, NmiReasons}; use crate::{ interrupt::stack_trace, interrupt_error, interrupt_stack, memory::GenericPfFlags, paging::VirtualAddress, ptrace, syscall::flag::*, @@ -40,13 +43,26 @@ interrupt_stack!(debug, @paranoid, |stack| { }); interrupt_stack!(non_maskable, @paranoid, |stack| { - #[cfg(feature = "profiling")] - crate::profiling::nmi_handler(stack); + let percpu = PercpuBlock::current(); - #[cfg(not(feature = "profiling"))] - { - println!("Non-maskable interrupt"); - stack.dump(); + let reasons = NmiReasons::from_bits_retain(percpu.nmi_flags_lock.load(Ordering::Relaxed)); + + if let Some(reason) = reasons.iter().next() { + if reason == NmiReasons::TLB_SHOOTDOWN && let Some(addrsp) = percpu.current_addrspace.get().as_ref() { + core::arch::asm!("mov rax, cr3; mov cr3, rax", out("rax") _); + addrsp.tlb_ack.fetch_add(1, Ordering::Relaxed); + } + + percpu.nmi_flags_lock.fetch_and(!reason.bits(), Ordering::Relaxed); + } else { + #[cfg(feature = "profiling")] + crate::profiling::nmi_handler(stack); + + #[cfg(not(feature = "profiling"))] + { + println!("Non-maskable interrupt"); + stack.dump(); + } } }); diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index ee5fa059c7..8decee877f 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -1,4 +1,5 @@ pub use crate::arch::x86_shared::*; +use crate::cpu_set::LogicalCpuId; pub mod alternative; @@ -69,3 +70,10 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) - pub use arch_copy_to_user as arch_copy_from_user; pub use alternative::kfx_size; + +pub fn send_tlb_nmi(target: LogicalCpuId) { + // TODO: Logical to APIC ID translation + unsafe { + device::local_apic::LOCAL_APIC.ipi_nmi(target.get()); + } +} diff --git a/src/arch/x86_shared/idt.rs b/src/arch/x86_shared/idt.rs index 2be1c12781..2c65c19dda 100644 --- a/src/arch/x86_shared/idt.rs +++ b/src/arch/x86_shared/idt.rs @@ -367,7 +367,7 @@ impl IdtEntry { } pub fn set_flags(&mut self, flags: IdtFlags) { - self.attribute = flags.bits; + self.attribute = flags.bits(); } pub fn set_ist(&mut self, ist: u8) { diff --git a/src/context/arch/x86_64.rs b/src/context/arch/x86_64.rs index 3ee8b84539..5a999f7619 100644 --- a/src/context/arch/x86_64.rs +++ b/src/context/arch/x86_64.rs @@ -143,9 +143,12 @@ pub unsafe fn empty_cr3() -> rmm::PhysicalAddress { /// Switch to the next context by restoring its stack and registers pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { + let pcr = crate::gdt::pcr(); + if let Some(ref stack) = next.kstack { - crate::gdt::set_tss_stack(stack.as_ptr() as usize + stack.len()); + crate::gdt::set_tss_stack(pcr, stack.as_ptr() as usize + stack.len()); } + let percpu_block = &*core::ptr::addr_of!((*pcr).percpu); core::arch::asm!( alternative2!( @@ -228,6 +231,8 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) { // to dyn or slice fat pointers), and NonNull optimization exists, map_or will hopefully be // optimized down to checking prev and next pointers, as next cannot be null. Some(ref next_space) => { + percpu_block.current_addrspace.set(Arc::as_ptr(next_space)); + if prev .addr_space .as_ref() diff --git a/src/context/memory.rs b/src/context/memory.rs index 5a40f30663..8f0bf75cd0 100644 --- a/src/context/memory.rs +++ b/src/context/memory.rs @@ -2521,6 +2521,10 @@ impl<'guard, 'addrsp> Flusher<'guard, 'addrsp> { let mut affected_cpu_count = 0; for cpu_id in self.active_cpus.iter_mut() { + if cpu_id == crate::cpu_id() { + continue; + } + crate::percpu::shootdown_tlb_ipi(Some(cpu_id)); affected_cpu_count += 1; } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 2d2b1ac705..f77da33a40 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -204,6 +204,7 @@ const RC_SHARED_NOT_COW: usize = 1 << (usize::BITS - 1); // userspace data page, or kernel heap page). This could be done only when debug assertions are // enabled. bitflags::bitflags! { + #[derive(Debug)] pub struct FrameFlags: usize { const NONE = 0; } diff --git a/src/percpu.rs b/src/percpu.rs index fd2684c285..0ddc18af0e 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -1,5 +1,7 @@ +use core::cell::Cell; use core::sync::atomic::{AtomicUsize, AtomicPtr, Ordering}; +use crate::context::memory::AddrSpaceWrapper; use crate::cpu_set::MAX_CPU_COUNT; use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId}; @@ -18,6 +20,8 @@ pub struct PercpuBlock { // flag to be cleared if it was already set. pub nmi_flags_lock: AtomicUsize, + pub current_addrspace: Cell<*const AddrSpaceWrapper>, + // TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it // first to avoid cache invalidation. #[cfg(feature = "profiling")] @@ -30,7 +34,8 @@ static ALL_PERCPU_BLOCKS: [AtomicPtr; MAX_CPU_COUNT as usize] = [NU // PercpuBlock::current() is implemented somewhere in the arch-specific modules bitflags::bitflags! { - struct NmiReasons: usize { + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + pub struct NmiReasons: usize { const TLB_SHOOTDOWN = 1; // TODO: Profiling code wakes all CPUs up, so use a global and percpu ack counter for that. @@ -57,6 +62,8 @@ pub fn shootdown_tlb_ipi(target: Option) { core::hint::spin_loop(); } } + + crate::arch::send_tlb_nmi(target); } else { for id in 0..crate::cpu_count() { // TODO: Optimize: use global counter and percpu ack counters. diff --git a/src/syscall/process.rs b/src/syscall/process.rs index 2c0c5ce2b4..5027dcc185 100644 --- a/src/syscall/process.rs +++ b/src/syscall/process.rs @@ -582,7 +582,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! { ); let base = Page::containing_address(VirtualAddress::new(0)); - let flags = MapFlags::PROT_EXEC | MapFlags::PROT_READ | MapFlags::PROT_WRITE; + let flags = MapFlags::MAP_FIXED_NOREPLACE | MapFlags::PROT_EXEC | MapFlags::PROT_READ | MapFlags::PROT_WRITE; let page_count = NonZeroUsize::new(bootstrap.page_count) .expect("bootstrap contained no pages!");