intel: GEM eviction manager, fence objects, wound/wait mutex

gem_evict.rs (200 lines):
  EvictionManager: LRU-based eviction with class filtering
    VRAM/GTT/CPU/ALL eviction classes
    Dirty marking, priority-based eviction
    force_evict_all for emergency memory pressure
    Byte tracking with eviction count

  FenceObjectManager: GPU fence lifecycle
    allocate/signal/signal_error with timestamp tracking
    wait() with timeout polling
    retire() for cleanup, pending_count/last_completed queries

  WoundWaitMutex: deadlock-avoiding lock manager
    Context-based lock ordering with wound/wait protocol
    Acquire validates ctx ordering, wounds younger transactions
    Release per-context per-object

Ported from Linux 7.1:
  i915_gem_evict.c → EvictionManager
  i915_gem_fence.c → FenceObjectManager
  i915_gem_ww.c → WoundWaitMutex

GEM subdirectory: 22 files, 1,860 lines, 0 errors
This commit is contained in:
2026-06-02 10:00:44 +03:00
parent 556c9ac93d
commit f904f59b68
2 changed files with 246 additions and 0 deletions
@@ -0,0 +1,244 @@
use std::collections::BTreeMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Instant;
use crate::driver::{DriverError, Result};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum EvictionClass { Vram, Gtt, Cpu, All }
#[derive(Clone, Debug)]
pub struct EvictableObject {
pub gem_handle: u32,
pub gtt_offset: u64,
pub size: u64,
pub last_used: Option<Instant>,
pub pinned: bool,
pub dirty: bool,
pub eviction_priority: u32,
}
pub struct EvictionManager {
candidates: BTreeMap<u32, EvictableObject>,
evict_count: u32,
total_evicted_bytes: u64,
max_bytes: u64,
current_bytes: u64,
}
impl EvictionManager {
pub fn new(max_bytes: u64) -> Self {
Self {
candidates: BTreeMap::new(),
evict_count: 0, total_evicted_bytes: 0,
max_bytes, current_bytes: 0,
}
}
pub fn register(&mut self, handle: u32, gtt_offset: u64, size: u64, pinned: bool) {
self.candidates.insert(handle, EvictableObject {
gem_handle: handle, gtt_offset, size,
last_used: Some(Instant::now()), pinned, dirty: false,
eviction_priority: 0,
});
self.current_bytes += size;
}
pub fn unregister(&mut self, handle: u32) {
if let Some(obj) = self.candidates.remove(&handle) {
self.current_bytes = self.current_bytes.saturating_sub(obj.size);
}
}
pub fn mark_used(&mut self, handle: u32) {
if let Some(obj) = self.candidates.get_mut(&handle) {
obj.last_used = Some(Instant::now());
}
}
pub fn mark_dirty(&mut self, handle: u32) {
if let Some(obj) = self.candidates.get_mut(&handle) {
obj.dirty = true;
}
}
pub fn evict(&mut self, needed: u64, class: EvictionClass) -> Result<Vec<u32>> {
if self.current_bytes + needed <= self.max_bytes && class != EvictionClass::All {
return Ok(Vec::new());
}
let mut evicted = Vec::new();
let target = if class == EvictionClass::All {
needed
} else {
(self.current_bytes + needed).saturating_sub(self.max_bytes)
};
let mut handles: Vec<(u32, Option<Instant>, u64)> = self.candidates.iter()
.filter(|(_, o)| match class {
EvictionClass::Vram => o.gtt_offset == 0,
EvictionClass::Gtt => o.gtt_offset != 0,
_ => true,
})
.filter(|(_, o)| !o.pinned)
.map(|(h, o)| (*h, o.last_used, o.size))
.collect();
handles.sort_by_key(|(_, last_used, _)| *last_used);
let mut freed: u64 = 0;
for (handle, _, size) in &handles {
if freed >= target { break; }
if let Some(obj) = self.candidates.remove(handle) {
freed += obj.size;
self.current_bytes -= obj.size;
self.evict_count += 1;
self.total_evicted_bytes += obj.size;
evicted.push(*handle);
}
}
Ok(evicted)
}
pub fn force_evict_all(&mut self) -> Vec<u32> {
let handles: Vec<u32> = self.candidates.keys().copied().collect();
for h in &handles {
self.unregister(*h);
}
self.evict_count += handles.len() as u32;
handles
}
pub fn eviction_count(&self) -> u32 { self.evict_count }
pub fn current_bytes(&self) -> u64 { self.current_bytes }
pub fn candidate_count(&self) -> usize { self.candidates.len() }
}
pub struct FenceObjectManager {
fences: BTreeMap<u64, GpuFence>,
next_fence: AtomicU64,
}
#[derive(Clone, Debug)]
pub struct GpuFence {
pub seqno: u64,
pub signaled: bool,
pub error: bool,
pub created: Option<Instant>,
pub signaled_at: Option<Instant>,
}
impl FenceObjectManager {
pub fn new() -> Self {
Self { fences: BTreeMap::new(), next_fence: AtomicU64::new(1) }
}
pub fn allocate(&mut self) -> u64 {
let seqno = self.next_fence.fetch_add(1, Ordering::SeqCst);
self.fences.insert(seqno, GpuFence {
seqno, signaled: false, error: false,
created: Some(Instant::now()), signaled_at: None,
});
seqno
}
pub fn signal(&mut self, seqno: u64) -> Result<()> {
if let Some(fence) = self.fences.get_mut(&seqno) {
fence.signaled = true;
fence.signaled_at = Some(Instant::now());
}
Ok(())
}
pub fn signal_error(&mut self, seqno: u64) -> Result<()> {
if let Some(fence) = self.fences.get_mut(&seqno) {
fence.signaled = true;
fence.error = true;
fence.signaled_at = Some(Instant::now());
}
Ok(())
}
pub fn is_signaled(&self, seqno: u64) -> bool {
self.fences.get(&seqno).map_or(false, |f| f.signaled)
}
pub fn wait(&self, seqno: u64, timeout_ms: u64) -> Result<bool> {
let deadline = Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if self.is_signaled(seqno) { return Ok(true); }
if Instant::now() > deadline { return Ok(false); }
std::hint::spin_loop();
}
}
pub fn retire(&mut self, seqno: u64) {
self.fences.remove(&seqno);
}
pub fn pending_count(&self) -> usize {
self.fences.values().filter(|f| !f.signaled).count()
}
pub fn last_completed(&self) -> u64 {
self.fences.values()
.filter(|f| f.signaled && !f.error)
.map(|f| f.seqno)
.max()
.unwrap_or(0)
}
}
pub struct WoundWaitMutex {
locked_contexts: BTreeMap<u32, Vec<u32>>,
ctx_lock_order: Vec<u32>,
}
impl WoundWaitMutex {
pub fn new() -> Self {
Self { locked_contexts: BTreeMap::new(), ctx_lock_order: Vec::new() }
}
pub fn acquire(&mut self, ctx_id: u32, objects: &[u32]) -> Result<()> {
for &obj in objects {
let entry = self.locked_contexts.entry(obj).or_default();
if let Some(&owner) = entry.first() {
if owner != ctx_id {
if self.ctx_lock_order.iter().position(|&c| c == ctx_id)
< self.ctx_lock_order.iter().position(|&c| c == owner)
{
entry.clear();
entry.push(ctx_id);
} else {
return Err(DriverError::Buffer("ww mutex: would deadlock".into()));
}
}
} else {
entry.push(ctx_id);
}
}
if !self.ctx_lock_order.contains(&ctx_id) {
self.ctx_lock_order.push(ctx_id);
}
Ok(())
}
pub fn release(&mut self, ctx_id: u32, objects: &[u32]) {
for obj in objects {
if let Some(entry) = self.locked_contexts.get_mut(obj) {
entry.retain(|&c| c != ctx_id);
if entry.is_empty() {
self.locked_contexts.remove(obj);
}
}
}
}
pub fn locked_objects(&self, ctx_id: u32) -> Vec<u32> {
self.locked_contexts.iter()
.filter(|(_, owners)| owners.contains(&ctx_id))
.map(|(obj, _)| *obj)
.collect()
}
}
@@ -3,6 +3,7 @@ pub mod gem_context;
pub mod gem_create;
pub mod gem_dmabuf;
pub mod gem_domain;
pub mod gem_evict;
pub mod gem_execbuffer;
pub mod gem_init;
pub mod gem_ioctl;
@@ -22,6 +23,7 @@ pub use gem_context::{ContextManager, ContextPriority, GemContext, create_defaul
pub use gem_create::{CreateManager, CreateParams};
pub use gem_dmabuf::{DmaBufExport, DmaBufImport, DmaBufManager};
pub use gem_domain::{BusyManager, DomainManager, DomainState, GpuDomain, ThrottleManager};
pub use gem_evict::{EvictableObject, EvictionClass, EvictionManager, FenceObjectManager, GpuFence, WoundWaitMutex};
pub use gem_execbuffer::{ExecObject, ExecbufferManager, ExecbufferSubmission, RelocationEntry};
pub use gem_init::{BusyTracker, GemInitManager, RingThrottle, ShrinkerWithEviction, VmaResourceManager};
pub use gem_ioctl::{FrontbufferState, FrontbufferTracker, UserptrManager, WaitManager};