intel: GEM eviction manager, fence objects, wound/wait mutex
gem_evict.rs (200 lines):
EvictionManager: LRU-based eviction with class filtering
VRAM/GTT/CPU/ALL eviction classes
Dirty marking, priority-based eviction
force_evict_all for emergency memory pressure
Byte tracking with eviction count
FenceObjectManager: GPU fence lifecycle
allocate/signal/signal_error with timestamp tracking
wait() with timeout polling
retire() for cleanup, pending_count/last_completed queries
WoundWaitMutex: deadlock-avoiding lock manager
Context-based lock ordering with wound/wait protocol
Acquire validates ctx ordering, wounds younger transactions
Release per-context per-object
Ported from Linux 7.1:
i915_gem_evict.c → EvictionManager
i915_gem_fence.c → FenceObjectManager
i915_gem_ww.c → WoundWaitMutex
GEM subdirectory: 22 files, 1,860 lines, 0 errors
This commit is contained in:
@@ -0,0 +1,244 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::driver::{DriverError, Result};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum EvictionClass { Vram, Gtt, Cpu, All }
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct EvictableObject {
|
||||
pub gem_handle: u32,
|
||||
pub gtt_offset: u64,
|
||||
pub size: u64,
|
||||
pub last_used: Option<Instant>,
|
||||
pub pinned: bool,
|
||||
pub dirty: bool,
|
||||
pub eviction_priority: u32,
|
||||
}
|
||||
|
||||
pub struct EvictionManager {
|
||||
candidates: BTreeMap<u32, EvictableObject>,
|
||||
evict_count: u32,
|
||||
total_evicted_bytes: u64,
|
||||
max_bytes: u64,
|
||||
current_bytes: u64,
|
||||
}
|
||||
|
||||
impl EvictionManager {
|
||||
pub fn new(max_bytes: u64) -> Self {
|
||||
Self {
|
||||
candidates: BTreeMap::new(),
|
||||
evict_count: 0, total_evicted_bytes: 0,
|
||||
max_bytes, current_bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register(&mut self, handle: u32, gtt_offset: u64, size: u64, pinned: bool) {
|
||||
self.candidates.insert(handle, EvictableObject {
|
||||
gem_handle: handle, gtt_offset, size,
|
||||
last_used: Some(Instant::now()), pinned, dirty: false,
|
||||
eviction_priority: 0,
|
||||
});
|
||||
self.current_bytes += size;
|
||||
}
|
||||
|
||||
pub fn unregister(&mut self, handle: u32) {
|
||||
if let Some(obj) = self.candidates.remove(&handle) {
|
||||
self.current_bytes = self.current_bytes.saturating_sub(obj.size);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark_used(&mut self, handle: u32) {
|
||||
if let Some(obj) = self.candidates.get_mut(&handle) {
|
||||
obj.last_used = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark_dirty(&mut self, handle: u32) {
|
||||
if let Some(obj) = self.candidates.get_mut(&handle) {
|
||||
obj.dirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn evict(&mut self, needed: u64, class: EvictionClass) -> Result<Vec<u32>> {
|
||||
if self.current_bytes + needed <= self.max_bytes && class != EvictionClass::All {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut evicted = Vec::new();
|
||||
let target = if class == EvictionClass::All {
|
||||
needed
|
||||
} else {
|
||||
(self.current_bytes + needed).saturating_sub(self.max_bytes)
|
||||
};
|
||||
|
||||
let mut handles: Vec<(u32, Option<Instant>, u64)> = self.candidates.iter()
|
||||
.filter(|(_, o)| match class {
|
||||
EvictionClass::Vram => o.gtt_offset == 0,
|
||||
EvictionClass::Gtt => o.gtt_offset != 0,
|
||||
_ => true,
|
||||
})
|
||||
.filter(|(_, o)| !o.pinned)
|
||||
.map(|(h, o)| (*h, o.last_used, o.size))
|
||||
.collect();
|
||||
|
||||
handles.sort_by_key(|(_, last_used, _)| *last_used);
|
||||
|
||||
let mut freed: u64 = 0;
|
||||
for (handle, _, size) in &handles {
|
||||
if freed >= target { break; }
|
||||
if let Some(obj) = self.candidates.remove(handle) {
|
||||
freed += obj.size;
|
||||
self.current_bytes -= obj.size;
|
||||
self.evict_count += 1;
|
||||
self.total_evicted_bytes += obj.size;
|
||||
evicted.push(*handle);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(evicted)
|
||||
}
|
||||
|
||||
pub fn force_evict_all(&mut self) -> Vec<u32> {
|
||||
let handles: Vec<u32> = self.candidates.keys().copied().collect();
|
||||
for h in &handles {
|
||||
self.unregister(*h);
|
||||
}
|
||||
self.evict_count += handles.len() as u32;
|
||||
handles
|
||||
}
|
||||
|
||||
pub fn eviction_count(&self) -> u32 { self.evict_count }
|
||||
pub fn current_bytes(&self) -> u64 { self.current_bytes }
|
||||
pub fn candidate_count(&self) -> usize { self.candidates.len() }
|
||||
}
|
||||
|
||||
pub struct FenceObjectManager {
|
||||
fences: BTreeMap<u64, GpuFence>,
|
||||
next_fence: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GpuFence {
|
||||
pub seqno: u64,
|
||||
pub signaled: bool,
|
||||
pub error: bool,
|
||||
pub created: Option<Instant>,
|
||||
pub signaled_at: Option<Instant>,
|
||||
}
|
||||
|
||||
impl FenceObjectManager {
|
||||
pub fn new() -> Self {
|
||||
Self { fences: BTreeMap::new(), next_fence: AtomicU64::new(1) }
|
||||
}
|
||||
|
||||
pub fn allocate(&mut self) -> u64 {
|
||||
let seqno = self.next_fence.fetch_add(1, Ordering::SeqCst);
|
||||
self.fences.insert(seqno, GpuFence {
|
||||
seqno, signaled: false, error: false,
|
||||
created: Some(Instant::now()), signaled_at: None,
|
||||
});
|
||||
seqno
|
||||
}
|
||||
|
||||
pub fn signal(&mut self, seqno: u64) -> Result<()> {
|
||||
if let Some(fence) = self.fences.get_mut(&seqno) {
|
||||
fence.signaled = true;
|
||||
fence.signaled_at = Some(Instant::now());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn signal_error(&mut self, seqno: u64) -> Result<()> {
|
||||
if let Some(fence) = self.fences.get_mut(&seqno) {
|
||||
fence.signaled = true;
|
||||
fence.error = true;
|
||||
fence.signaled_at = Some(Instant::now());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_signaled(&self, seqno: u64) -> bool {
|
||||
self.fences.get(&seqno).map_or(false, |f| f.signaled)
|
||||
}
|
||||
|
||||
pub fn wait(&self, seqno: u64, timeout_ms: u64) -> Result<bool> {
|
||||
let deadline = Instant::now() + std::time::Duration::from_millis(timeout_ms);
|
||||
loop {
|
||||
if self.is_signaled(seqno) { return Ok(true); }
|
||||
if Instant::now() > deadline { return Ok(false); }
|
||||
std::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn retire(&mut self, seqno: u64) {
|
||||
self.fences.remove(&seqno);
|
||||
}
|
||||
|
||||
pub fn pending_count(&self) -> usize {
|
||||
self.fences.values().filter(|f| !f.signaled).count()
|
||||
}
|
||||
|
||||
pub fn last_completed(&self) -> u64 {
|
||||
self.fences.values()
|
||||
.filter(|f| f.signaled && !f.error)
|
||||
.map(|f| f.seqno)
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct WoundWaitMutex {
|
||||
locked_contexts: BTreeMap<u32, Vec<u32>>,
|
||||
ctx_lock_order: Vec<u32>,
|
||||
}
|
||||
|
||||
impl WoundWaitMutex {
|
||||
pub fn new() -> Self {
|
||||
Self { locked_contexts: BTreeMap::new(), ctx_lock_order: Vec::new() }
|
||||
}
|
||||
|
||||
pub fn acquire(&mut self, ctx_id: u32, objects: &[u32]) -> Result<()> {
|
||||
for &obj in objects {
|
||||
let entry = self.locked_contexts.entry(obj).or_default();
|
||||
if let Some(&owner) = entry.first() {
|
||||
if owner != ctx_id {
|
||||
if self.ctx_lock_order.iter().position(|&c| c == ctx_id)
|
||||
< self.ctx_lock_order.iter().position(|&c| c == owner)
|
||||
{
|
||||
entry.clear();
|
||||
entry.push(ctx_id);
|
||||
} else {
|
||||
return Err(DriverError::Buffer("ww mutex: would deadlock".into()));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
entry.push(ctx_id);
|
||||
}
|
||||
}
|
||||
if !self.ctx_lock_order.contains(&ctx_id) {
|
||||
self.ctx_lock_order.push(ctx_id);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn release(&mut self, ctx_id: u32, objects: &[u32]) {
|
||||
for obj in objects {
|
||||
if let Some(entry) = self.locked_contexts.get_mut(obj) {
|
||||
entry.retain(|&c| c != ctx_id);
|
||||
if entry.is_empty() {
|
||||
self.locked_contexts.remove(obj);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn locked_objects(&self, ctx_id: u32) -> Vec<u32> {
|
||||
self.locked_contexts.iter()
|
||||
.filter(|(_, owners)| owners.contains(&ctx_id))
|
||||
.map(|(obj, _)| *obj)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ pub mod gem_context;
|
||||
pub mod gem_create;
|
||||
pub mod gem_dmabuf;
|
||||
pub mod gem_domain;
|
||||
pub mod gem_evict;
|
||||
pub mod gem_execbuffer;
|
||||
pub mod gem_init;
|
||||
pub mod gem_ioctl;
|
||||
@@ -22,6 +23,7 @@ pub use gem_context::{ContextManager, ContextPriority, GemContext, create_defaul
|
||||
pub use gem_create::{CreateManager, CreateParams};
|
||||
pub use gem_dmabuf::{DmaBufExport, DmaBufImport, DmaBufManager};
|
||||
pub use gem_domain::{BusyManager, DomainManager, DomainState, GpuDomain, ThrottleManager};
|
||||
pub use gem_evict::{EvictableObject, EvictionClass, EvictionManager, FenceObjectManager, GpuFence, WoundWaitMutex};
|
||||
pub use gem_execbuffer::{ExecObject, ExecbufferManager, ExecbufferSubmission, RelocationEntry};
|
||||
pub use gem_init::{BusyTracker, GemInitManager, RingThrottle, ShrinkerWithEviction, VmaResourceManager};
|
||||
pub use gem_ioctl::{FrontbufferState, FrontbufferTracker, UserptrManager, WaitManager};
|
||||
|
||||
Reference in New Issue
Block a user