diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index baa7905b08..1d3e83c3d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,8 @@ variables: GIT_SUBMODULE_STRATEGY: recursive before_script: - rustup component add rust-src + - apt-get install nasm + - rustup component add rust-src stages: - build @@ -15,14 +16,14 @@ stages: build:x86_64: stage: build script: - mkdir -p target/x86_64 - make ARCH=x86_64 BUILD=target/x86_64 + - mkdir -p target/x86_64 + - make ARCH=x86_64 BUILD=target/x86_64 build:i686: stage: build script: - mkdir -p target/i686 - make ARCH=i686 BUILD=target/i686 + - mkdir -p target/i686 + - make ARCH=i686 BUILD=target/i686 build:aarch64: stage: build diff --git a/Cargo.toml b/Cargo.toml index 0c86bd99c8..04001f2dee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ raw-cpuid = "10.2.0" x86 = { version = "0.47.0", default-features = false } [features] -default = ["acpi", "multi_core", "graphical_debug", "serial_debug", "self_modifying"] +default = ["acpi", "multi_core", "graphical_debug", "serial_debug", "self_modifying", "x86_kvm_pv"] # Activates some limited code-overwriting optimizations, based on CPU features. self_modifying = [] @@ -58,6 +58,7 @@ qemu_debug = [] serial_debug = [] system76_ec_debug = [] slab = ["slab_allocator"] +x86_kvm_pv = [] debugger = ["syscall_debug"] syscall_debug = [] diff --git a/src/arch/aarch64/device/mod.rs b/src/arch/aarch64/device/mod.rs index 577116ed04..0286da9460 100644 --- a/src/arch/aarch64/device/mod.rs +++ b/src/arch/aarch64/device/mod.rs @@ -54,3 +54,5 @@ pub unsafe fn io_mmap(addr: usize, io_size: usize) { .flush(); } } +#[derive(Default)] +pub struct ArchPercpuMisc; diff --git a/src/arch/aarch64/time.rs b/src/arch/aarch64/time.rs index 60809b40df..8bf457db66 100644 --- a/src/arch/aarch64/time.rs +++ b/src/arch/aarch64/time.rs @@ -1,4 +1,4 @@ -pub fn counter() -> u128 { +pub fn monotonic_absolute() -> u128 { //TODO: aarch64 generic timer counter - 0 + *crate::time::OFFSET.lock() } diff --git a/src/arch/x86_64/interrupt/handler.rs b/src/arch/x86_64/interrupt/handler.rs index 74a96887b0..9b9ed9f7c3 100644 --- a/src/arch/x86_64/interrupt/handler.rs +++ b/src/arch/x86_64/interrupt/handler.rs @@ -260,9 +260,9 @@ macro_rules! swapgs_iff_ring3_fast { // Check whether the last two bits RSP+8 (code segment) are equal to zero. test QWORD PTR [rsp + 8], 0x3 // Skip the SWAPGS instruction if CS & 0b11 == 0b00. - jz 1f + jz 2f swapgs - 1: + 2: " }; } @@ -271,9 +271,9 @@ macro_rules! swapgs_iff_ring3_fast_errorcode { () => { " test QWORD PTR [rsp + 16], 0x3 - jz 1f + jz 2f swapgs - 1: + 2: " }; } @@ -336,9 +336,9 @@ macro_rules! conditional_swapgs_paranoid { " cmp rdx, rdi sete bl - je 1f + je 2f swapgs - 1: + 2: ", ) } } @@ -346,9 +346,9 @@ macro_rules! conditional_swapgs_back_paranoid { () => { " test bl, bl - jnz 1f + jnz 2f swapgs - 1: + 2: " }; } diff --git a/src/arch/x86_64/interrupt/syscall.rs b/src/arch/x86_64/interrupt/syscall.rs index b542aaabf3..644d5875ca 100644 --- a/src/arch/x86_64/interrupt/syscall.rs +++ b/src/arch/x86_64/interrupt/syscall.rs @@ -149,7 +149,7 @@ pub unsafe extern "C" fn syscall_instruction() { "test BYTE PTR [rsp + 17], 1;", // If set, return using IRETQ instead. - "jnz 1f;", + "jnz 2f;", // Otherwise, continue with the fast sysretq. @@ -180,7 +180,7 @@ pub unsafe extern "C" fn syscall_instruction() { // IRETQ fallback: " .p2align 4 -1: +2: xor rcx, rcx xor r11, r11 iretq diff --git a/src/arch/x86_shared/device/mod.rs b/src/arch/x86_shared/device/mod.rs index 065cf40b1f..dd26ddac70 100644 --- a/src/arch/x86_shared/device/mod.rs +++ b/src/arch/x86_shared/device/mod.rs @@ -10,6 +10,9 @@ pub mod serial; #[cfg(feature = "system76_ec_debug")] pub mod system76_ec; +#[cfg(feature = "x86_kvm_pv")] +pub mod tsc; + use crate::paging::KernelMapper; pub unsafe fn init() { @@ -43,6 +46,12 @@ unsafe fn init_hpet() -> bool { pub unsafe fn init_noncore() { log::info!("Initializing system timer"); + + #[cfg(feature = "x86_kvm_pv")] + if tsc::init() { + log::info!("TSC used as system clock source"); + } + if init_hpet() { log::info!("HPET used as system timer"); } else { @@ -59,4 +68,13 @@ pub unsafe fn init_noncore() { pub unsafe fn init_ap() { local_apic::init_ap(); + + #[cfg(feature = "x86_kvm_pv")] + tsc::init(); +} + +#[derive(Default)] +pub struct ArchPercpuMisc { + #[cfg(feature = "x86_kvm_pv")] + pub tsc_info: tsc::TscPercpu, } diff --git a/src/arch/x86_shared/device/tsc.rs b/src/arch/x86_shared/device/tsc.rs new file mode 100644 index 0000000000..602dd17aaf --- /dev/null +++ b/src/arch/x86_shared/device/tsc.rs @@ -0,0 +1,159 @@ +use core::{cell::Cell, ptr::addr_of}; + +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::__cpuid; + +#[cfg(target_arch = "x86")] +use core::arch::x86::__cpuid; + +use rmm::Arch; +use spin::Once; + +use crate::{memory::allocate_frame, percpu::PercpuBlock}; + +struct KvmSupport { + max_leaf: u32, + supp_feats: KvmFeatureBits, +} +bitflags! { + // https://www.kernel.org/doc/html/latest/virt/kvm/x86/cpuid.html + #[derive(Debug)] + struct KvmFeatureBits: u32 { + const CLOCKSOURCE = 1 << 0; + const CLOCKSOURCE2 = 1 << 3; + const CLOCKSOURCE_STABLE = 1 << 24; + } +} + +// https://www.kernel.org/doc/html/v5.9/virt/kvm/msr.html +#[repr(packed)] +#[derive(Clone, Copy, Debug)] +struct PvclockVcpuTimeInfo { + version: u32, + pad: u32, + tsc_timestamp: u64, + system_time: u64, + tsc_to_system_mul: u32, + tsc_shift: i8, + flags: u8, + _pad: [u8; 2], +} + +const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b564d01; +const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b564d00; + +static KVM_SUPPORT: Once> = Once::new(); + +pub struct TscPercpu { + vcpu_page: Cell<*const PvclockVcpuTimeInfo>, + prev: Cell, +} +impl Default for TscPercpu { + fn default() -> Self { + Self { + vcpu_page: Cell::new(core::ptr::null()), + prev: Cell::new(0), + } + } +} + +pub fn monotonic_absolute() -> Option { + let inf = &PercpuBlock::current().misc_arch_info.tsc_info; + let ptr = inf.vcpu_page.get(); + if ptr.is_null() { + return None; + } + loop { + unsafe { + let cur_version = addr_of!((*ptr).version).read_volatile(); + if cur_version & 1 == 1 { + continue; + } + let elapsed_ticks = x86::time::rdtsc() + .checked_sub(addr_of!((*ptr).tsc_timestamp).read_volatile()) + .unwrap(); + let tsc_shift = addr_of!((*ptr).tsc_shift).read_volatile(); + let elapsed = if tsc_shift >= 0 { + elapsed_ticks.checked_shl(tsc_shift as u32).unwrap() + } else { + elapsed_ticks.checked_shr((-tsc_shift) as u32).unwrap() + }; + let system_time = addr_of!((*ptr).system_time).read_volatile(); + let tsc_to_system_mul = addr_of!((*ptr).tsc_to_system_mul).read_volatile(); + let new_version = addr_of!((*ptr).version).read_volatile(); + if new_version != cur_version || new_version & 1 == 1 { + continue; + } + let delta = (u128::from(elapsed) * u128::from(tsc_to_system_mul)) >> 32; + let time = u128::from(system_time) + delta; + let prev = inf.prev.replace(time); + if prev > time { + // TODO + log::error!("TSC wraparound ({prev} > {time})"); + return None; + } + assert!(prev <= time); + return Some(time); + } + } +} + +pub unsafe fn init() -> bool { + let cpuid = crate::cpuid::cpuid(); + if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) { + return false; + } + + let kvm_support = KVM_SUPPORT.call_once(|| { + let res = unsafe { __cpuid(0x4000_0000) }; + if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] { + return None; + } + let max_leaf = res.eax; + if max_leaf < 0x4000_0001 { + return None; + } + let res = unsafe { __cpuid(0x4000_0001) }; + + let supp_feats = KvmFeatureBits::from_bits_retain(res.eax); + + log::info!("Detected KVM paravirtualization support, features {supp_feats:?}"); + + Some(KvmSupport { + max_leaf, + supp_feats, + }) + }); + + if let Some(kvm_support) = kvm_support + && kvm_support + .supp_feats + .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE) + { + let frame = allocate_frame().expect("failed to allocate timer page"); + x86::msr::wrmsr( + MSR_KVM_SYSTEM_TIME_NEW, + (frame.start_address().data() as u64) | 1, + ); + let ptr = crate::paging::RmmA::phys_to_virt(frame.start_address()).data() + as *const PvclockVcpuTimeInfo; + PercpuBlock::current() + .misc_arch_info + .tsc_info + .vcpu_page + .set(ptr); + + /*let tsc_ghz = loop { + let val1 = ptr.read_volatile(); + let val2 = ptr.read_volatile(); + if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version { + continue; + } + let val1 + break tsc_hz / 1_000_000_000; + };*/ + true + } else { + false + } +} diff --git a/src/arch/x86_shared/time.rs b/src/arch/x86_shared/time.rs index d610050a09..3a02bf5935 100644 --- a/src/arch/x86_shared/time.rs +++ b/src/arch/x86_shared/time.rs @@ -2,7 +2,17 @@ use super::device::hpet; use super::device::pit; -pub fn counter() -> u128 { +pub fn monotonic_absolute() -> u128 { + // The paravirtualized TSC is already guaranteed to be monotonic, and thus doesn't need to be + // readjusted. + #[cfg(feature = "x86_kvm_pv")] + if let Some(ns) = super::device::tsc::monotonic_absolute() { + return ns; + } + + *crate::time::OFFSET.lock() + hpet_or_pit() +} +fn hpet_or_pit() -> u128 { #[cfg(feature = "acpi")] if let Some(ref hpet) = *crate::acpi::ACPI_TABLE.hpet.read() { //TODO: handle rollover? @@ -32,7 +42,6 @@ pub fn counter() -> u128 { // Calculate nanoseconds since last interrupt return (elapsed as u128 * period_fs as u128) / 1_000_000; } - // Read ticks since last interrupt let elapsed = unsafe { pit::read() }; // Calculate nanoseconds since last interrupt diff --git a/src/percpu.rs b/src/percpu.rs index b5971df7c5..34e8c832f4 100644 --- a/src/percpu.rs +++ b/src/percpu.rs @@ -39,6 +39,8 @@ pub struct PercpuBlock { #[cfg(feature = "syscall_debug")] pub syscall_debug_info: Cell, + + pub misc_arch_info: crate::device::ArchPercpuMisc, } const NULL: AtomicPtr = AtomicPtr::new(core::ptr::null_mut()); @@ -158,6 +160,8 @@ impl PercpuBlock { #[cfg(feature = "profiling")] profiling: None, + + misc_arch_info: Default::default(), } } } diff --git a/src/time.rs b/src/time.rs index 3d82c3903d..5a233967a7 100644 --- a/src/time.rs +++ b/src/time.rs @@ -2,13 +2,14 @@ use spin::Mutex; pub const NANOS_PER_SEC: u128 = 1_000_000_000; -/// Kernel start time, measured in (seconds, nanoseconds) since Unix epoch +// TODO: seqlock? +/// Kernel start time, measured in nanoseconds since Unix epoch pub static START: Mutex = Mutex::new(0); -/// Kernel up time, measured in (seconds, nanoseconds) since `START_TIME` +/// Kernel up time, measured in nanoseconds since `START_TIME` pub static OFFSET: Mutex = Mutex::new(0); pub fn monotonic() -> u128 { - *OFFSET.lock() + crate::arch::time::counter() + crate::arch::time::monotonic_absolute() } pub fn realtime() -> u128 {