Implement a TLB shootdown draft.
This commit is contained in:
4lDO2
Generated
+53
-56
@@ -4,9 +4,9 @@ version = 3
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.6"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a"
|
||||
checksum = "d713b3834d76b85304d4d525563c1276e2e30dc97cc67bfb4585a4a29fc2c89f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
@@ -22,9 +22,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "bit_field"
|
||||
version = "0.10.1"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
|
||||
checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61"
|
||||
|
||||
[[package]]
|
||||
name = "bitfield"
|
||||
@@ -40,21 +40,21 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.4.1"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
|
||||
checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.4.3"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.73"
|
||||
version = "1.0.88"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
|
||||
checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
@@ -97,9 +97,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.0.0"
|
||||
version = "2.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d"
|
||||
checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown",
|
||||
@@ -110,7 +110,7 @@ name = "kernel"
|
||||
version = "0.4.1"
|
||||
dependencies = [
|
||||
"bitfield",
|
||||
"bitflags 1.3.2",
|
||||
"bitflags 2.4.2",
|
||||
"byteorder",
|
||||
"cc",
|
||||
"fdt",
|
||||
@@ -147,14 +147,14 @@ version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "549ce1740e46b291953c4340adcd74c59bcf4308f4cac050fd33ba91b7168f4a"
|
||||
dependencies = [
|
||||
"spinning_top 0.2.4",
|
||||
"spinning_top 0.2.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.8"
|
||||
version = "0.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390"
|
||||
checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
@@ -162,18 +162,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.6.3"
|
||||
version = "2.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c"
|
||||
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
@@ -183,9 +180,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.9"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1"
|
||||
checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
|
||||
|
||||
[[package]]
|
||||
name = "plain"
|
||||
@@ -195,27 +192,27 @@ checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.66"
|
||||
version = "1.0.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
|
||||
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.33"
|
||||
version = "1.0.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
|
||||
checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "10.5.0"
|
||||
version = "10.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6"
|
||||
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
]
|
||||
@@ -230,7 +227,7 @@ checksum = "64072665120942deff5fd5425d6c1811b854f4939e7f1c01ce755f64432bbea7"
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
dependencies = [
|
||||
"bitflags 2.4.1",
|
||||
"bitflags 2.4.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -245,15 +242,15 @@ checksum = "56a596b5718bf5e059d59a30af12f7f462a152de147aa462b70892849ee18704"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.21"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
|
||||
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scroll"
|
||||
@@ -263,18 +260,18 @@ checksum = "fda28d4b4830b807a8b43f7b0e6b5df875311b3e7621d84577188c175b6ec1ec"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.188"
|
||||
version = "1.0.197"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
|
||||
checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.188"
|
||||
version = "1.0.197"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
|
||||
checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -283,9 +280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "0.6.3"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
|
||||
checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -321,9 +318,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "spinning_top"
|
||||
version = "0.2.4"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75adad84ee84b521fb2cca2d4fd0f1dab1d8d026bda3c5bea4ca63b5f9f9293c"
|
||||
checksum = "5b9eb1a2f4c41445a3a0ff9abc5221c5fcd28e1f13cd7c0397706f9ac938ddb0"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
@@ -339,9 +336,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.32"
|
||||
version = "2.0.51"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
|
||||
checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -362,9 +359,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.6.3"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
|
||||
checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -384,9 +381,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.11"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
@@ -396,9 +393,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "0.5.15"
|
||||
version = "0.5.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc"
|
||||
checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
@@ -416,18 +413,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.30"
|
||||
version = "0.7.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7"
|
||||
checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.7.30"
|
||||
version = "0.7.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba"
|
||||
checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ rustc-cfg = "0.3.0"
|
||||
toml = "0.7"
|
||||
|
||||
[dependencies]
|
||||
bitflags = "1.2.1"
|
||||
bitflags = "2"
|
||||
bitfield = "0.13.2"
|
||||
hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] }
|
||||
linked_list_allocator = "0.9.0"
|
||||
|
||||
@@ -222,6 +222,7 @@ unsafe fn overwrite(relocs: &[AltReloc], enable: KcpuFeatures) {
|
||||
}
|
||||
|
||||
bitflags! {
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct KcpuFeatures: usize {
|
||||
const SMAP = 1;
|
||||
const FSGSBASE = 2;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Global descriptor table
|
||||
|
||||
use core::{convert::TryInto, mem, sync::atomic::AtomicUsize};
|
||||
use core::{convert::TryInto, mem, sync::atomic::AtomicUsize, cell::Cell};
|
||||
|
||||
use crate::{
|
||||
cpu_set::LogicalCpuId,
|
||||
@@ -140,17 +140,17 @@ pub unsafe fn pcr() -> *mut ProcessorControlRegion {
|
||||
}
|
||||
|
||||
#[cfg(feature = "pti")]
|
||||
pub unsafe fn set_tss_stack(stack: usize) {
|
||||
pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
|
||||
use super::pti::{PTI_CONTEXT_STACK, PTI_CPU_STACK};
|
||||
core::ptr::addr_of_mut!((*pcr()).tss.rsp[0])
|
||||
core::ptr::addr_of_mut!((*pcr).tss.rsp[0])
|
||||
.write_unaligned((PTI_CPU_STACK.as_ptr() as usize + PTI_CPU_STACK.len()) as u64);
|
||||
PTI_CONTEXT_STACK = stack;
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "pti"))]
|
||||
pub unsafe fn set_tss_stack(stack: usize) {
|
||||
pub unsafe fn set_tss_stack(pcr: *mut ProcessorControlRegion, stack: usize) {
|
||||
// TODO: If this increases performance, read gs:[offset] directly
|
||||
core::ptr::addr_of_mut!((*pcr()).tss.rsp[0]).write_unaligned(stack as u64);
|
||||
core::ptr::addr_of_mut!((*pcr).tss.rsp[0]).write_unaligned(stack as u64);
|
||||
}
|
||||
|
||||
// Initialize startup GDT
|
||||
@@ -231,7 +231,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
|
||||
x86::msr::wrmsr(x86::msr::IA32_FS_BASE, 0);
|
||||
|
||||
// Set the stack pointer to use when coming back from userspace.
|
||||
set_tss_stack(stack_offset);
|
||||
set_tss_stack(pcr, stack_offset);
|
||||
|
||||
// Load the task register
|
||||
task::load_tr(SegmentSelector::new(GDT_TSS as u16, Ring::Ring0));
|
||||
@@ -240,6 +240,7 @@ pub unsafe fn init_paging(stack_offset: usize, cpu_id: LogicalCpuId) {
|
||||
cpu_id,
|
||||
switch_internals: Default::default(),
|
||||
nmi_flags_lock: AtomicUsize::new(0),
|
||||
current_addrspace: Cell::new(core::ptr::null_mut()),
|
||||
|
||||
#[cfg(feature = "profiling")]
|
||||
profiling: None,
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
use x86::irq::PageFaultError;
|
||||
|
||||
use crate::percpu::{PercpuBlock, NmiReasons};
|
||||
use crate::{
|
||||
interrupt::stack_trace, interrupt_error, interrupt_stack, memory::GenericPfFlags,
|
||||
paging::VirtualAddress, ptrace, syscall::flag::*,
|
||||
@@ -40,13 +43,26 @@ interrupt_stack!(debug, @paranoid, |stack| {
|
||||
});
|
||||
|
||||
interrupt_stack!(non_maskable, @paranoid, |stack| {
|
||||
#[cfg(feature = "profiling")]
|
||||
crate::profiling::nmi_handler(stack);
|
||||
let percpu = PercpuBlock::current();
|
||||
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
{
|
||||
println!("Non-maskable interrupt");
|
||||
stack.dump();
|
||||
let reasons = NmiReasons::from_bits_retain(percpu.nmi_flags_lock.load(Ordering::Relaxed));
|
||||
|
||||
if let Some(reason) = reasons.iter().next() {
|
||||
if reason == NmiReasons::TLB_SHOOTDOWN && let Some(addrsp) = percpu.current_addrspace.get().as_ref() {
|
||||
core::arch::asm!("mov rax, cr3; mov cr3, rax", out("rax") _);
|
||||
addrsp.tlb_ack.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
percpu.nmi_flags_lock.fetch_and(!reason.bits(), Ordering::Relaxed);
|
||||
} else {
|
||||
#[cfg(feature = "profiling")]
|
||||
crate::profiling::nmi_handler(stack);
|
||||
|
||||
#[cfg(not(feature = "profiling"))]
|
||||
{
|
||||
println!("Non-maskable interrupt");
|
||||
stack.dump();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pub use crate::arch::x86_shared::*;
|
||||
use crate::cpu_set::LogicalCpuId;
|
||||
|
||||
pub mod alternative;
|
||||
|
||||
@@ -69,3 +70,10 @@ pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -
|
||||
pub use arch_copy_to_user as arch_copy_from_user;
|
||||
|
||||
pub use alternative::kfx_size;
|
||||
|
||||
pub fn send_tlb_nmi(target: LogicalCpuId) {
|
||||
// TODO: Logical to APIC ID translation
|
||||
unsafe {
|
||||
device::local_apic::LOCAL_APIC.ipi_nmi(target.get());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,7 +367,7 @@ impl IdtEntry {
|
||||
}
|
||||
|
||||
pub fn set_flags(&mut self, flags: IdtFlags) {
|
||||
self.attribute = flags.bits;
|
||||
self.attribute = flags.bits();
|
||||
}
|
||||
|
||||
pub fn set_ist(&mut self, ist: u8) {
|
||||
|
||||
@@ -143,9 +143,12 @@ pub unsafe fn empty_cr3() -> rmm::PhysicalAddress {
|
||||
|
||||
/// Switch to the next context by restoring its stack and registers
|
||||
pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
|
||||
let pcr = crate::gdt::pcr();
|
||||
|
||||
if let Some(ref stack) = next.kstack {
|
||||
crate::gdt::set_tss_stack(stack.as_ptr() as usize + stack.len());
|
||||
crate::gdt::set_tss_stack(pcr, stack.as_ptr() as usize + stack.len());
|
||||
}
|
||||
let percpu_block = &*core::ptr::addr_of!((*pcr).percpu);
|
||||
|
||||
core::arch::asm!(
|
||||
alternative2!(
|
||||
@@ -228,6 +231,8 @@ pub unsafe fn switch_to(prev: &mut super::Context, next: &mut super::Context) {
|
||||
// to dyn or slice fat pointers), and NonNull optimization exists, map_or will hopefully be
|
||||
// optimized down to checking prev and next pointers, as next cannot be null.
|
||||
Some(ref next_space) => {
|
||||
percpu_block.current_addrspace.set(Arc::as_ptr(next_space));
|
||||
|
||||
if prev
|
||||
.addr_space
|
||||
.as_ref()
|
||||
|
||||
@@ -2521,6 +2521,10 @@ impl<'guard, 'addrsp> Flusher<'guard, 'addrsp> {
|
||||
let mut affected_cpu_count = 0;
|
||||
|
||||
for cpu_id in self.active_cpus.iter_mut() {
|
||||
if cpu_id == crate::cpu_id() {
|
||||
continue;
|
||||
}
|
||||
|
||||
crate::percpu::shootdown_tlb_ipi(Some(cpu_id));
|
||||
affected_cpu_count += 1;
|
||||
}
|
||||
|
||||
@@ -204,6 +204,7 @@ const RC_SHARED_NOT_COW: usize = 1 << (usize::BITS - 1);
|
||||
// userspace data page, or kernel heap page). This could be done only when debug assertions are
|
||||
// enabled.
|
||||
bitflags::bitflags! {
|
||||
#[derive(Debug)]
|
||||
pub struct FrameFlags: usize {
|
||||
const NONE = 0;
|
||||
}
|
||||
|
||||
+8
-1
@@ -1,5 +1,7 @@
|
||||
use core::cell::Cell;
|
||||
use core::sync::atomic::{AtomicUsize, AtomicPtr, Ordering};
|
||||
|
||||
use crate::context::memory::AddrSpaceWrapper;
|
||||
use crate::cpu_set::MAX_CPU_COUNT;
|
||||
use crate::{context::switch::ContextSwitchPercpu, cpu_set::LogicalCpuId};
|
||||
|
||||
@@ -18,6 +20,8 @@ pub struct PercpuBlock {
|
||||
// flag to be cleared if it was already set.
|
||||
pub nmi_flags_lock: AtomicUsize,
|
||||
|
||||
pub current_addrspace: Cell<*const AddrSpaceWrapper>,
|
||||
|
||||
// TODO: Put mailbox queues here, e.g. for TLB shootdown? Just be sure to 128-byte align it
|
||||
// first to avoid cache invalidation.
|
||||
#[cfg(feature = "profiling")]
|
||||
@@ -30,7 +34,8 @@ static ALL_PERCPU_BLOCKS: [AtomicPtr<PercpuBlock>; MAX_CPU_COUNT as usize] = [NU
|
||||
// PercpuBlock::current() is implemented somewhere in the arch-specific modules
|
||||
|
||||
bitflags::bitflags! {
|
||||
struct NmiReasons: usize {
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct NmiReasons: usize {
|
||||
const TLB_SHOOTDOWN = 1;
|
||||
|
||||
// TODO: Profiling code wakes all CPUs up, so use a global and percpu ack counter for that.
|
||||
@@ -57,6 +62,8 @@ pub fn shootdown_tlb_ipi(target: Option<LogicalCpuId>) {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
crate::arch::send_tlb_nmi(target);
|
||||
} else {
|
||||
for id in 0..crate::cpu_count() {
|
||||
// TODO: Optimize: use global counter and percpu ack counters.
|
||||
|
||||
@@ -582,7 +582,7 @@ pub unsafe fn usermode_bootstrap(bootstrap: &Bootstrap) -> ! {
|
||||
);
|
||||
|
||||
let base = Page::containing_address(VirtualAddress::new(0));
|
||||
let flags = MapFlags::PROT_EXEC | MapFlags::PROT_READ | MapFlags::PROT_WRITE;
|
||||
let flags = MapFlags::MAP_FIXED_NOREPLACE | MapFlags::PROT_EXEC | MapFlags::PROT_READ | MapFlags::PROT_WRITE;
|
||||
|
||||
let page_count = NonZeroUsize::new(bootstrap.page_count)
|
||||
.expect("bootstrap contained no pages!");
|
||||
|
||||
Reference in New Issue
Block a user