Implement paravirtualized KVM TSC support

.gitlab-ci.yml

@@ -4,7 +4,8 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
 
 before_script:
-  rustup component add rust-src
+  - apt-get install nasm
+  - rustup component add rust-src
 
 stages:
   - build
@@ -15,14 +16,14 @@ stages:
 build:x86_64:
   stage: build
   script:
-    mkdir -p target/x86_64
-    make ARCH=x86_64 BUILD=target/x86_64
+    - mkdir -p target/x86_64
+    - make ARCH=x86_64 BUILD=target/x86_64
 
 build:i686:
   stage: build
   script:
-    mkdir -p target/i686
-    make ARCH=i686 BUILD=target/i686
+    - mkdir -p target/i686
+    - make ARCH=i686 BUILD=target/i686
 
 build:aarch64:
   stage: build

Cargo.toml

@@ -42,7 +42,7 @@ raw-cpuid = "10.2.0"
 x86 = { version = "0.47.0", default-features = false }
 
 [features]
-default = ["acpi", "multi_core", "graphical_debug", "serial_debug", "self_modifying"]
+default = ["acpi", "multi_core", "graphical_debug", "serial_debug", "self_modifying", "x86_kvm_pv"]
 
 # Activates some limited code-overwriting optimizations, based on CPU features.
 self_modifying = []
@@ -58,6 +58,7 @@ qemu_debug = []
 serial_debug = []
 system76_ec_debug = []
 slab = ["slab_allocator"]
+x86_kvm_pv = []
 
 debugger = ["syscall_debug"]
 syscall_debug = []

src/arch/aarch64/device/mod.rs

@@ -54,3 +54,5 @@ pub unsafe fn io_mmap(addr: usize, io_size: usize) {
             .flush();
     }
 }
+#[derive(Default)]
+pub struct ArchPercpuMisc;

src/arch/aarch64/time.rs

@@ -1,4 +1,4 @@
-pub fn counter() -> u128 {
+pub fn monotonic_absolute() -> u128 {
     //TODO: aarch64 generic timer counter
-    0
+    *crate::time::OFFSET.lock()
 }

src/arch/x86_64/interrupt/handler.rs

@@ -260,9 +260,9 @@ macro_rules! swapgs_iff_ring3_fast {
         // Check whether the last two bits RSP+8 (code segment) are equal to zero.
         test QWORD PTR [rsp + 8], 0x3
         // Skip the SWAPGS instruction if CS & 0b11 == 0b00.
-        jz 1f
+        jz 2f
         swapgs
-        1:
+        2:
     "
     };
 }
@@ -271,9 +271,9 @@ macro_rules! swapgs_iff_ring3_fast_errorcode {
     () => {
         "
         test QWORD PTR [rsp + 16], 0x3
-        jz 1f
+        jz 2f
         swapgs
-        1:
+        2:
     "
     };
 }
@@ -336,9 +336,9 @@ macro_rules! conditional_swapgs_paranoid {
         "
         cmp rdx, rdi
         sete bl
-        je 1f
+        je 2f
         swapgs
-        1:
+        2:
         ",
     ) }
 }
@@ -346,9 +346,9 @@ macro_rules! conditional_swapgs_back_paranoid {
     () => {
         "
         test bl, bl
-        jnz 1f
+        jnz 2f
         swapgs
-        1:
+        2:
     "
     };
 }

src/arch/x86_64/interrupt/syscall.rs

@@ -149,7 +149,7 @@ pub unsafe extern "C" fn syscall_instruction() {
 
     "test BYTE PTR [rsp + 17], 1;",
     // If set, return using IRETQ instead.
-    "jnz 1f;",
+    "jnz 2f;",
 
     // Otherwise, continue with the fast sysretq.
 
@@ -180,7 +180,7 @@ pub unsafe extern "C" fn syscall_instruction() {
     // IRETQ fallback:
     "
     .p2align 4
-1:
+2:
     xor rcx, rcx
     xor r11, r11
     iretq

src/arch/x86_shared/device/mod.rs

@@ -10,6 +10,9 @@ pub mod serial;
 #[cfg(feature = "system76_ec_debug")]
 pub mod system76_ec;
 
+#[cfg(feature = "x86_kvm_pv")]
+pub mod tsc;
+
 use crate::paging::KernelMapper;
 
 pub unsafe fn init() {
@@ -43,6 +46,12 @@ unsafe fn init_hpet() -> bool {
 
 pub unsafe fn init_noncore() {
     log::info!("Initializing system timer");
+
+    #[cfg(feature = "x86_kvm_pv")]
+    if tsc::init() {
+        log::info!("TSC used as system clock source");
+    }
+
     if init_hpet() {
         log::info!("HPET used as system timer");
     } else {
@@ -59,4 +68,13 @@ pub unsafe fn init_noncore() {
 
 pub unsafe fn init_ap() {
     local_apic::init_ap();
+
+    #[cfg(feature = "x86_kvm_pv")]
+    tsc::init();
+}
+
+#[derive(Default)]
+pub struct ArchPercpuMisc {
+    #[cfg(feature = "x86_kvm_pv")]
+    pub tsc_info: tsc::TscPercpu,
 }

src/arch/x86_shared/device/tsc.rs

@@ -0,0 +1,159 @@
+use core::{cell::Cell, ptr::addr_of};
+
+#[cfg(target_arch = "x86_64")]
+use core::arch::x86_64::__cpuid;
+
+#[cfg(target_arch = "x86")]
+use core::arch::x86::__cpuid;
+
+use rmm::Arch;
+use spin::Once;
+
+use crate::{memory::allocate_frame, percpu::PercpuBlock};
+
+struct KvmSupport {
+    max_leaf: u32,
+    supp_feats: KvmFeatureBits,
+}
+bitflags! {
+    // https://www.kernel.org/doc/html/latest/virt/kvm/x86/cpuid.html
+    #[derive(Debug)]
+    struct KvmFeatureBits: u32 {
+        const CLOCKSOURCE = 1 << 0;
+        const CLOCKSOURCE2 = 1 << 3;
+        const CLOCKSOURCE_STABLE = 1 << 24;
+    }
+}
+
+// https://www.kernel.org/doc/html/v5.9/virt/kvm/msr.html
+#[repr(packed)]
+#[derive(Clone, Copy, Debug)]
+struct PvclockVcpuTimeInfo {
+    version: u32,
+    pad: u32,
+    tsc_timestamp: u64,
+    system_time: u64,
+    tsc_to_system_mul: u32,
+    tsc_shift: i8,
+    flags: u8,
+    _pad: [u8; 2],
+}
+
+const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b564d01;
+const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b564d00;
+
+static KVM_SUPPORT: Once<Option<KvmSupport>> = Once::new();
+
+pub struct TscPercpu {
+    vcpu_page: Cell<*const PvclockVcpuTimeInfo>,
+    prev: Cell<u128>,
+}
+impl Default for TscPercpu {
+    fn default() -> Self {
+        Self {
+            vcpu_page: Cell::new(core::ptr::null()),
+            prev: Cell::new(0),
+        }
+    }
+}
+
+pub fn monotonic_absolute() -> Option<u128> {
+    let inf = &PercpuBlock::current().misc_arch_info.tsc_info;
+    let ptr = inf.vcpu_page.get();
+    if ptr.is_null() {
+        return None;
+    }
+    loop {
+        unsafe {
+            let cur_version = addr_of!((*ptr).version).read_volatile();
+            if cur_version & 1 == 1 {
+                continue;
+            }
+            let elapsed_ticks = x86::time::rdtsc()
+                .checked_sub(addr_of!((*ptr).tsc_timestamp).read_volatile())
+                .unwrap();
+            let tsc_shift = addr_of!((*ptr).tsc_shift).read_volatile();
+            let elapsed = if tsc_shift >= 0 {
+                elapsed_ticks.checked_shl(tsc_shift as u32).unwrap()
+            } else {
+                elapsed_ticks.checked_shr((-tsc_shift) as u32).unwrap()
+            };
+            let system_time = addr_of!((*ptr).system_time).read_volatile();
+            let tsc_to_system_mul = addr_of!((*ptr).tsc_to_system_mul).read_volatile();
+            let new_version = addr_of!((*ptr).version).read_volatile();
+            if new_version != cur_version || new_version & 1 == 1 {
+                continue;
+            }
+            let delta = (u128::from(elapsed) * u128::from(tsc_to_system_mul)) >> 32;
+            let time = u128::from(system_time) + delta;
+            let prev = inf.prev.replace(time);
+            if prev > time {
+                // TODO
+                log::error!("TSC wraparound ({prev} > {time})");
+                return None;
+            }
+            assert!(prev <= time);
+            return Some(time);
+        }
+    }
+}
+
+pub unsafe fn init() -> bool {
+    let cpuid = crate::cpuid::cpuid();
+    if !cpuid.get_feature_info().map_or(false, |f| f.has_tsc()) {
+        return false;
+    }
+
+    let kvm_support = KVM_SUPPORT.call_once(|| {
+        let res = unsafe { __cpuid(0x4000_0000) };
+        if [res.ebx, res.ecx, res.edx].map(u32::to_le_bytes) != [*b"KVMK", *b"VMKV", *b"M\0\0\0"] {
+            return None;
+        }
+        let max_leaf = res.eax;
+        if max_leaf < 0x4000_0001 {
+            return None;
+        }
+        let res = unsafe { __cpuid(0x4000_0001) };
+
+        let supp_feats = KvmFeatureBits::from_bits_retain(res.eax);
+
+        log::info!("Detected KVM paravirtualization support, features {supp_feats:?}");
+
+        Some(KvmSupport {
+            max_leaf,
+            supp_feats,
+        })
+    });
+
+    if let Some(kvm_support) = kvm_support
+        && kvm_support
+            .supp_feats
+            .contains(KvmFeatureBits::CLOCKSOURCE2 | KvmFeatureBits::CLOCKSOURCE_STABLE)
+    {
+        let frame = allocate_frame().expect("failed to allocate timer page");
+        x86::msr::wrmsr(
+            MSR_KVM_SYSTEM_TIME_NEW,
+            (frame.start_address().data() as u64) | 1,
+        );
+        let ptr = crate::paging::RmmA::phys_to_virt(frame.start_address()).data()
+            as *const PvclockVcpuTimeInfo;
+        PercpuBlock::current()
+            .misc_arch_info
+            .tsc_info
+            .vcpu_page
+            .set(ptr);
+
+        /*let tsc_ghz = loop {
+            let val1 = ptr.read_volatile();
+            let val2 = ptr.read_volatile();
+            if val1.version & 1 == 1 || val2.version & 1 == 1 || val1.version != val2.version {
+                continue;
+            }
+            let val1
+            break tsc_hz / 1_000_000_000;
+        };*/
+        true
+    } else {
+        false
+    }
+}

src/arch/x86_shared/time.rs

@@ -2,7 +2,17 @@
 use super::device::hpet;
 use super::device::pit;
 
-pub fn counter() -> u128 {
+pub fn monotonic_absolute() -> u128 {
+    // The paravirtualized TSC is already guaranteed to be monotonic, and thus doesn't need to be
+    // readjusted.
+    #[cfg(feature = "x86_kvm_pv")]
+    if let Some(ns) = super::device::tsc::monotonic_absolute() {
+        return ns;
+    }
+
+    *crate::time::OFFSET.lock() + hpet_or_pit()
+}
+fn hpet_or_pit() -> u128 {
     #[cfg(feature = "acpi")]
     if let Some(ref hpet) = *crate::acpi::ACPI_TABLE.hpet.read() {
         //TODO: handle rollover?
@@ -32,7 +42,6 @@ pub fn counter() -> u128 {
         // Calculate nanoseconds since last interrupt
         return (elapsed as u128 * period_fs as u128) / 1_000_000;
     }
-
     // Read ticks since last interrupt
     let elapsed = unsafe { pit::read() };
     // Calculate nanoseconds since last interrupt

src/percpu.rs

@@ -39,6 +39,8 @@ pub struct PercpuBlock {
 
     #[cfg(feature = "syscall_debug")]
     pub syscall_debug_info: Cell<SyscallDebugInfo>,
+
+    pub misc_arch_info: crate::device::ArchPercpuMisc,
 }
 
 const NULL: AtomicPtr<PercpuBlock> = AtomicPtr::new(core::ptr::null_mut());
@@ -158,6 +160,8 @@ impl PercpuBlock {
 
             #[cfg(feature = "profiling")]
             profiling: None,
+
+            misc_arch_info: Default::default(),
         }
     }
 }

src/time.rs

@@ -2,13 +2,14 @@ use spin::Mutex;
 
 pub const NANOS_PER_SEC: u128 = 1_000_000_000;
 
-/// Kernel start time, measured in (seconds, nanoseconds) since Unix epoch
+// TODO: seqlock?
+/// Kernel start time, measured in nanoseconds since Unix epoch
 pub static START: Mutex<u128> = Mutex::new(0);
-/// Kernel up time, measured in (seconds, nanoseconds) since `START_TIME`
+/// Kernel up time, measured in nanoseconds since `START_TIME`
 pub static OFFSET: Mutex<u128> = Mutex::new(0);
 
 pub fn monotonic() -> u128 {
-    *OFFSET.lock() + crate::arch::time::counter()
+    crate::arch::time::monotonic_absolute()
 }
 
 pub fn realtime() -> u128 {