Merge branch 'pcie' into 'master'

Add support for reading the extended PCIe configuration space

See merge request redox-os/drivers!60

Cargo.lock

@@ -837,9 +837,11 @@ dependencies = [
  "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)",
+ "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)",
  "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)",
  "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
  "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

pcid/Cargo.toml

@@ -16,8 +16,10 @@ bincode = "1.2"
 bitflags = "1"
 byteorder = "1.2"
 libc = "0.2"
+plain = "0.2"
 redox_syscall = { git = "https://gitlab.redox-os.org/redox-os/syscall.git" }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+smallvec = "1"
 thiserror = "1"
 toml = "0.5"

pcid/src/main.rs

@@ -1,10 +1,5 @@
 #![feature(asm)]
 
-extern crate bitflags;
-extern crate byteorder;
-extern crate syscall;
-extern crate toml;
-
 use std::fs::{File, metadata, read_dir};
 use std::io::prelude::*;
 use std::os::unix::io::{FromRawFd, RawFd};
@@ -15,12 +10,14 @@ use std::{env, io, i64, thread};
 use syscall::iopl;
 
 use crate::config::Config;
-use crate::pci::{Pci, PciBar, PciBus, PciClass, PciDev, PciFunc, PciHeader, PciHeaderError, PciHeaderType};
+use crate::pci::{CfgAccess, Pci, PciIter, PciBar, PciBus, PciClass, PciDev, PciFunc, PciHeader, PciHeaderError, PciHeaderType};
 use crate::pci::cap::Capability as PciCapability;
+use crate::pcie::Pcie;
 
 mod config;
 mod driver_interface;
 mod pci;
+mod pcie;
 
 pub struct DriverHandler {
     config: config::DriverConfig,
@@ -137,7 +134,13 @@ impl DriverHandler {
 
 pub struct State {
     threads: Mutex<Vec<thread::JoinHandle<()>>>,
-    pci: Pci,
+    pci: Arc<Pci>,
+    pcie: Option<Pcie>,
+}
+impl State {
+    fn preferred_cfg_access(&self) -> &dyn CfgAccess {
+        self.pcie.as_ref().map(|pcie| pcie as &dyn CfgAccess).unwrap_or(&*self.pci as &dyn CfgAccess)
+    }
 }
 
 fn handle_parsed_header(state: Arc<State>, config: &Config, bus_num: u8,
@@ -273,11 +276,11 @@ fn handle_parsed_header(state: Arc<State>, config: &Config, bus_num: u8,
 
                     let offset = 0x10 + (i as u8) * 4;
 
-                    let original = pci.read(bus_num, dev_num, func_num, offset);
-                    pci.write(bus_num, dev_num, func_num, offset, 0xFFFFFFFF);
+                    let original = pci.read(bus_num, dev_num, func_num, offset.into());
+                    pci.write(bus_num, dev_num, func_num, offset.into(), 0xFFFFFFFF);
 
-                    let new = pci.read(bus_num, dev_num, func_num, offset);
-                    pci.write(bus_num, dev_num, func_num, offset, original);
+                    let new = pci.read(bus_num, dev_num, func_num, offset.into());
+                    pci.write(bus_num, dev_num, func_num, offset.into(), original);
 
                     let masked = if new & 1 == 1 {
                         new & 0xFFFFFFFC
@@ -296,7 +299,7 @@ fn handle_parsed_header(state: Arc<State>, config: &Config, bus_num: u8,
 
             let capabilities = {
                 let bus = PciBus {
-                    pci,
+                    pci: state.preferred_cfg_access(),
                     num: bus_num,
                 };
                 let dev = PciDev {
@@ -444,18 +447,25 @@ fn main() {
         }
     }
 
+    let pci = Arc::new(Pci::new());
+
     let state = Arc::new(State {
-        pci: Pci::new(),
+        pci: Arc::clone(&pci),
+        pcie: match Pcie::new(Arc::clone(&pci)) {
+            Ok(pcie) => Some(pcie),
+            Err(error) => {
+                println!("Couldn't retrieve PCIe info, perhaps the kernel is not compiled with acpi? Using the PCI 3.0 configuration space instead. Error: {:?}", error);
+                None
+            }
+        },
         threads: Mutex::new(Vec::new()),
     });
 
-    let pci = &state.pci;
-
-    unsafe { iopl(3).unwrap() };
+    let pci = state.preferred_cfg_access();
 
     print!("PCI BS/DV/FN VEND:DEVI CL.SC.IN.RV\n");
 
-    'bus: for bus in pci.buses() {
+    'bus: for bus in PciIter::new(pci) {
         'dev: for dev in bus.devs() {
             for func in dev.funcs() {
                 let func_num = func.num;

pcid/src/pci/bus.rs

@@ -1,7 +1,7 @@
-use super::{Pci, PciDev};
+use super::{Pci, PciDev, CfgAccess};
 
 pub struct PciBus<'pci> {
-    pub pci: &'pci Pci,
+    pub pci: &'pci dyn CfgAccess,
     pub num: u8
 }
 
@@ -10,10 +10,10 @@ impl<'pci> PciBus<'pci> {
         PciBusIter::new(self)
     }
 
-    pub unsafe fn read(&self, dev: u8, func: u8, offset: u8) -> u32 {
+    pub unsafe fn read(&self, dev: u8, func: u8, offset: u16) -> u32 {
         self.pci.read(self.num, dev, func, offset)
     }
-    pub unsafe fn write(&self, dev: u8, func: u8, offset: u8, value: u32) {
+    pub unsafe fn write(&self, dev: u8, func: u8, offset: u16, value: u32) {
         self.pci.write(self.num, dev, func, offset, value)
     }
 }
@@ -26,7 +26,7 @@ pub struct PciBusIter<'pci> {
 impl<'pci> PciBusIter<'pci> {
     pub fn new(bus: &'pci PciBus<'pci>) -> Self {
         PciBusIter {
-            bus: bus,
+            bus,
             num: 0
         }
     }

pcid/src/pci/cap.rs

@@ -25,7 +25,7 @@ where
 
             if self.offset == 0 { return None };
 
-            let first_dword = dbg!(self.reader.read_u32(dbg!(self.offset)));
+            let first_dword = dbg!(self.reader.read_u32(dbg!(u16::from(self.offset))));
             let next = ((first_dword >> 8) & 0xFF) as u8;
 
             let offset = self.offset;
@@ -135,9 +135,9 @@ impl Capability {
     }
     unsafe fn parse_msix<R: ConfigReader>(reader: &R, offset: u8) -> Self {
         Self::MsiX(MsixCapability {
-            a: reader.read_u32(offset),
-            b: reader.read_u32(offset + 4),
-            c: reader.read_u32(offset + 8),
+            a: reader.read_u32(u16::from(offset)),
+            b: reader.read_u32(u16::from(offset + 4)),
+            c: reader.read_u32(u16::from(offset + 8)),
         })
     }
     unsafe fn parse_pcie<R: ConfigReader>(reader: &R, offset: u8) -> Self {
@@ -147,7 +147,7 @@ impl Capability {
     unsafe fn parse<R: ConfigReader>(reader: &R, offset: u8) -> Self {
         assert_eq!(offset & 0xF8, offset, "capability must be dword aligned");
 
-        let dword = reader.read_u32(offset);
+        let dword = reader.read_u32(u16::from(offset));
         let capability_id = (dword & 0xFF) as u8;
 
         if capability_id == CapabilityId::Msi as u8 {

pcid/src/pci/dev.rs

@@ -10,10 +10,10 @@ impl<'pci> PciDev<'pci> {
         PciDevIter::new(self)
     }
 
-    pub unsafe fn read(&self, func: u8, offset: u8) -> u32 {
+    pub unsafe fn read(&self, func: u8, offset: u16) -> u32 {
         self.bus.read(self.num, func, offset)
     }
-    pub unsafe fn write(&self, func: u8, offset: u8, value: u32) {
+    pub unsafe fn write(&self, func: u8, offset: u16, value: u32) {
         self.bus.write(self.num, func, offset, value);
     }
 }

pcid/src/pci/func.rs

@@ -2,11 +2,8 @@ use byteorder::{LittleEndian, ByteOrder};
 
 use super::PciDev;
 
-// TODO: PCI Express Configuration Space, which uses a flat memory buffer, rather than IN/OUT
-// instructions.
-
 pub trait ConfigReader {
-    unsafe fn read_range(&self, offset: u8, len: u8) -> Vec<u8> {
+    unsafe fn read_range(&self, offset: u16, len: u16) -> Vec<u8> {
         assert!(len > 3 && len % 4 == 0);
         let mut ret = Vec::with_capacity(len as usize);
         let results = (offset..offset + len).step_by(4).fold(Vec::new(), |mut acc, offset| {
@@ -19,9 +16,9 @@ pub trait ConfigReader {
         ret
     }
 
-    unsafe fn read_u32(&self, offset: u8) -> u32;
+    unsafe fn read_u32(&self, offset: u16) -> u32;
 
-    unsafe fn read_u8(&self, offset: u8) -> u8 {
+    unsafe fn read_u8(&self, offset: u16) -> u8 {
         let dword_offset = (offset / 4) * 4;
         let dword = self.read_u32(dword_offset);
 
@@ -30,7 +27,7 @@ pub trait ConfigReader {
     }
 }
 pub trait ConfigWriter {
-    unsafe fn write_u32(&self, offset: u8, value: u32);
+    unsafe fn write_u32(&self, offset: u16, value: u32);
 }
 
 pub struct PciFunc<'pci> {
@@ -39,12 +36,12 @@ pub struct PciFunc<'pci> {
 }
 
 impl<'pci> ConfigReader for PciFunc<'pci> {
-    unsafe fn read_u32(&self, offset: u8) -> u32 {
+    unsafe fn read_u32(&self, offset: u16) -> u32 {
         self.dev.read(self.num, offset)
     }
 }
 impl<'pci> ConfigWriter for PciFunc<'pci> {
-    unsafe fn write_u32(&self, offset: u8, value: u32) {
+    unsafe fn write_u32(&self, offset: u16, value: u32) {
         self.dev.write(self.num, offset, value);
     }
 }

pcid/src/pci/mod.rs

@@ -1,4 +1,5 @@
-use std::sync::Mutex;
+use std::convert::TryFrom;
+use std::sync::{Mutex, Once};
 
 pub use self::bar::PciBar;
 pub use self::bus::{PciBus, PciBusIter};
@@ -16,14 +17,24 @@ mod func;
 pub mod header;
 pub mod msi;
 
+pub trait CfgAccess {
+    unsafe fn read_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32;
+    unsafe fn read(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32;
+
+    unsafe fn write_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32);
+    unsafe fn write(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32);
+}
+
 pub struct Pci {
     lock: Mutex<()>,
+    iopl_once: Once,
 }
 
 impl Pci {
     pub fn new() -> Self {
         Self {
             lock: Mutex::new(()),
+            iopl_once: Once::new(),
         }
     }
 
@@ -31,9 +42,33 @@ impl Pci {
         PciIter::new(self)
     }
 
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    pub unsafe fn read_nolock(&self, bus: u8, dev: u8, func: u8, offset: u8) -> u32 {
-        let address = 0x80000000 | ((bus as u32) << 16) | ((dev as u32) << 11) | ((func as u32) << 8) | ((offset as u32) & 0xFC);
+    fn set_iopl() {
+        // make sure that pcid is not granted io port permission unless pcie memory-mapped
+        // configuration space is not available.
+        println!("PCI: couldn't find or access PCIe extended configuration, and thus falling back to PCI 3.0 io ports");
+        unsafe { syscall::iopl(3).expect("pcid: failed to set iopl to 3"); }
+    }
+    fn address(bus: u8, dev: u8, func: u8, offset: u8) -> u32 {
+        // TODO: Find the part of pcid that uses an unaligned offset!
+        //
+        // assert_eq!(offset & 0xFC, offset, "pci offset is not aligned");
+        //
+        let offset = offset & 0xFC;
+
+        assert_eq!(dev & 0x1F, dev, "pci device larger than 5 bits");
+        assert_eq!(func & 0x7, func, "pci func larger than 3 bits");
+
+        0x80000000 | (u32::from(bus) << 16) | (u32::from(dev) << 11) | (u32::from(func) << 8) | u32::from(offset)
+    }
+}
+#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
+impl CfgAccess for Pci {
+    unsafe fn read_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32 {
+        self.iopl_once.call_once(Self::set_iopl);
+
+        let offset = u8::try_from(offset).expect("offset too large for PCI 3.0 configuration space");
+        let address = Self::address(bus, dev, func, offset);
+
         let value: u32;
         asm!("mov dx, 0xCF8
               out dx, eax
@@ -43,15 +78,17 @@ impl Pci {
         value
     }
 
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    pub unsafe fn read(&self, bus: u8, dev: u8, func: u8, offset: u8) -> u32 {
+    unsafe fn read(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32 {
         let _guard = self.lock.lock().unwrap();
         self.read_nolock(bus, dev, func, offset)
     }
 
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    pub unsafe fn write_nolock(&self, bus: u8, dev: u8, func: u8, offset: u8, value: u32) {
-        let address = 0x80000000 | ((bus as u32) << 16) | ((dev as u32) << 11) | ((func as u32) << 8) | ((offset as u32) & 0xFC);
+    unsafe fn write_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32) {
+        self.iopl_once.call_once(Self::set_iopl);
+
+        let offset = u8::try_from(offset).expect("offset too large for PCI 3.0 configuration space");
+        let address = Self::address(bus, dev, func, offset);
+
         asm!("mov dx, 0xCF8
               out dx, eax"
              : : "{eax}"(address) : "dx" : "intel", "volatile");
@@ -59,22 +96,21 @@ impl Pci {
               out dx, eax"
              : : "{eax}"(value) : "dx" : "intel", "volatile");
     }
-    #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-    pub unsafe fn write(&self, bus: u8, dev: u8, func: u8, offset: u8, value: u32) {
+    unsafe fn write(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32) {
         let _guard = self.lock.lock().unwrap();
         self.write_nolock(bus, dev, func, offset, value)
     }
 }
 
 pub struct PciIter<'pci> {
-    pci: &'pci Pci,
+    pci: &'pci dyn CfgAccess,
     num: u32
 }
 
 impl<'pci> PciIter<'pci> {
-    pub fn new(pci: &'pci Pci) -> Self {
+    pub fn new(pci: &'pci dyn CfgAccess) -> Self {
         PciIter {
-            pci: pci,
+            pci,
             num: 0
         }
     }

pcid/src/pci/msi.rs

@@ -19,7 +19,7 @@ impl MsiCapability {
     pub const MC_MSI_ENABLED_BIT: u16 = 1;
 
     pub unsafe fn parse<R: ConfigReader>(reader: &R, offset: u8) -> Self {
-        let dword = reader.read_u32(offset);
+        let dword = reader.read_u32(u16::from(offset));
 
         let message_control = (dword >> 16) as u16;
 
@@ -27,34 +27,34 @@ impl MsiCapability {
             if message_control & Self::MC_64_BIT_ADDR_BIT != 0 {
                 Self::_64BitAddressWithPvm {
                     message_control: dword,
-                    message_address_lo: reader.read_u32(offset + 4),
-                    message_address_hi: reader.read_u32(offset + 8),
-                    message_data: reader.read_u32(offset + 12),
-                    mask_bits: reader.read_u32(offset + 16),
-                    pending_bits: reader.read_u32(offset + 20),
+                    message_address_lo: reader.read_u32(u16::from(offset + 4)),
+                    message_address_hi: reader.read_u32(u16::from(offset + 8)),
+                    message_data: reader.read_u32(u16::from(offset + 12)),
+                    mask_bits: reader.read_u32(u16::from(offset + 16)),
+                    pending_bits: reader.read_u32(u16::from(offset + 20)),
                 }
             } else {
                 Self::_32BitAddressWithPvm {
                     message_control: dword,
-                    message_address: reader.read_u32(offset + 4),
-                    message_data: reader.read_u32(offset + 8),
-                    mask_bits: reader.read_u32(offset + 12),
-                    pending_bits: reader.read_u32(offset + 16),
+                    message_address: reader.read_u32(u16::from(offset + 4)),
+                    message_data: reader.read_u32(u16::from(offset + 8)),
+                    mask_bits: reader.read_u32(u16::from(offset + 12)),
+                    pending_bits: reader.read_u32(u16::from(offset + 16)),
                 }
             }
         } else {
             if message_control & Self::MC_64_BIT_ADDR_BIT != 0 {
                 Self::_64BitAddress {
                     message_control: dword,
-                    message_address_lo: reader.read_u32(offset + 4),
-                    message_address_hi: reader.read_u32(offset + 8),
-                    message_data: reader.read_u32(offset + 12),
+                    message_address_lo: reader.read_u32(u16::from(offset + 4)),
+                    message_address_hi: reader.read_u32(u16::from(offset + 8)),
+                    message_data: reader.read_u32(u16::from(offset + 12)),
                 }
             } else {
                 Self::_32BitAddress {
                     message_control: dword,
-                    message_address: reader.read_u32(offset + 4),
-                    message_data: reader.read_u32(offset + 8),
+                    message_address: reader.read_u32(u16::from(offset + 4)),
+                    message_data: reader.read_u32(u16::from(offset + 8)),
                 }
             }
         }
@@ -81,7 +81,7 @@ impl MsiCapability {
         }
     }
     pub unsafe fn write_message_control<W: ConfigWriter>(&mut self, writer: &W, offset: u8) {
-        writer.write_u32(offset, self.message_control_raw());
+        writer.write_u32(u16::from(offset), self.message_control_raw());
     }
     pub fn is_pvt_capable(&self) -> bool {
         self.message_control() & Self::MC_PVT_CAPABLE_BIT != 0
@@ -238,17 +238,17 @@ impl MsixCapability {
     /// Write the first DWORD into configuration space (containing the partially modifiable Message
     /// Control field).
     pub unsafe fn write_a<W: ConfigWriter>(&self, writer: &W, offset: u8) {
-        writer.write_u32(offset, self.a)
+        writer.write_u32(u16::from(offset), self.a)
     }
     /// Write the second DWORD into configuration space (containing the modifiable table
     /// offset and the readonly table BIR).
     pub unsafe fn write_b<W: ConfigWriter>(&self, writer: &W, offset: u8) {
-        writer.write_u32(offset + 4, self.a)
+        writer.write_u32(u16::from(offset + 4), self.a)
     }
     /// Write the third DWORD into configuration space (containing the modifiable pending bit array
     /// offset, and the readonly PBA BIR).
     pub unsafe fn write_c<W: ConfigWriter>(&self, writer: &W, offset: u8) {
-        writer.write_u32(offset + 8, self.a)
+        writer.write_u32(u16::from(offset + 8), self.a)
     }
     /// Write this capability structure back to configuration space.
     pub unsafe fn write_all<W: ConfigWriter>(&self, writer: &W, offset: u8) {

pcid/src/pcie/mod.rs

@@ -0,0 +1,214 @@
+use std::{fmt, fs, io, mem, ptr, slice};
+use std::collections::BTreeMap;
+use std::sync::{Arc, Mutex};
+
+use syscall::flag::PhysmapFlags;
+use syscall::io::Dma;
+
+use smallvec::SmallVec;
+
+use crate::pci::{CfgAccess, Pci, PciIter};
+
+pub const MCFG_NAME: [u8; 4] = *b"MCFG";
+
+#[repr(packed)]
+#[derive(Clone, Copy)]
+pub struct Mcfg {
+    // base sdt fields
+    name: [u8; 4],
+    length: u32,
+    revision: u8,
+    checksum: u8,
+    oem_id: [u8; 6],
+    oem_table_id: [u8; 8],
+    oem_revision: u32,
+    creator_id: [u8; 4],
+    creator_revision: u32,
+    _rsvd: [u8; 8],
+
+    base_addrs: [PcieAlloc; 0],
+}
+unsafe impl plain::Plain for Mcfg {}
+
+/// The "Memory Mapped Enhanced Configuration Space Base Address Allocation Structure" (yes, it's
+/// called that).
+
+#[repr(packed)]
+#[derive(Clone, Copy, Debug)]
+pub struct PcieAlloc {
+    pub base_addr: u64,
+    pub seg_group_num: u16,
+    pub start_bus: u8,
+    pub end_bus: u8,
+    _rsvd: [u8; 4],
+}
+unsafe impl plain::Plain for PcieAlloc {}
+
+impl Mcfg {
+    pub fn base_addr_structs(&self) -> &[PcieAlloc] {
+        let total_length = mem::size_of::<Self>();
+        let len = total_length - 44;
+        // safe because the length cannot be changed arbitrarily
+        unsafe { slice::from_raw_parts(&self.base_addrs as *const PcieAlloc, len / mem::size_of::<PcieAlloc>()) }
+    }
+}
+impl fmt::Debug for Mcfg {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.debug_struct("Mcfg")
+            .field("name", &"MCFG")
+            .field("length", &self.length)
+            .field("revision", &self.revision)
+            .field("checksum", &self.checksum)
+            .field("oem_id", &self.oem_id)
+            .field("oem_table_id", &self.oem_table_id)
+            .field("oem_revision", &self.oem_revision)
+            .field("creator_revision", &self.creator_revision)
+            .field("creator_id", &self.creator_id)
+            .field("base_addrs", &self.base_addr_structs())
+            .finish()
+    }
+}
+
+pub struct Mcfgs {
+    tables: SmallVec<[Vec<u8>; 2]>,
+}
+
+impl Mcfgs {
+    pub fn tables<'a>(&'a self) -> impl Iterator<Item = &'a Mcfg> + 'a {
+        self.tables.iter().filter_map(|bytes| {
+            let mcfg = plain::from_bytes::<Mcfg>(bytes).ok()?;
+            if mcfg.length as usize > bytes.len() {
+                return None;
+            }
+            Some(mcfg)
+        })
+    }
+    pub fn allocs<'a>(&'a self) -> impl Iterator<Item = &'a PcieAlloc> + 'a {
+        self.tables().map(|table| table.base_addr_structs().iter()).flatten()
+    }
+
+    pub fn fetch() -> io::Result<Self> {
+        let table_dir = fs::read_dir("acpi:tables")?;
+
+        let tables = table_dir.map(|table_direntry| -> io::Result<Option<_>> {
+            let table_direntry = table_direntry?;
+            let table_path = table_direntry.path();
+
+            let table_filename = match table_path.file_name() {
+                Some(n) => n.to_str().ok_or(io::Error::new(io::ErrorKind::InvalidData, "Non-UTF-8 ACPI table filename"))?,
+                None => return Ok(None),
+            };
+
+            if table_filename.starts_with("MCFG") {
+                Ok(Some(fs::read(table_path)?))
+            } else {
+                Ok(None)
+            }
+        }).filter_map(|result_option| result_option.transpose()).collect::<Result<SmallVec<_>, _>>()?;
+
+        Ok(Self {
+            tables,
+        })
+    }
+    pub fn table_and_alloc_at_bus(&self, bus: u8) -> Option<(&Mcfg, &PcieAlloc)> {
+        self.tables().find_map(|table| {
+            Some((table, table.base_addr_structs().iter().find(|addr_struct| {
+                (addr_struct.start_bus..addr_struct.end_bus).contains(&bus)
+            })?))
+        })
+    }
+    pub fn at_bus(&self, bus: u8) -> Option<&PcieAlloc> {
+        self.table_and_alloc_at_bus(bus).map(|(_, alloc)| alloc)
+    }
+}
+
+impl fmt::Debug for Mcfgs {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        struct Tables<'a>(&'a Mcfgs);
+        impl<'a> fmt::Debug for Tables<'a> {
+            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+                f.debug_list().entries(self.0.tables()).finish()
+            }
+        }
+
+        f.debug_tuple("Mcfgs").field(&Tables(self)).finish()
+    }
+}
+
+pub struct Pcie {
+    lock: Mutex<()>,
+    mcfgs: Mcfgs,
+    maps: Mutex<BTreeMap<(u8, u8, u8), *mut u32>>,
+    fallback: Arc<Pci>,
+}
+unsafe impl Send for Pcie {}
+unsafe impl Sync for Pcie {}
+
+impl Pcie {
+    pub fn new(fallback: Arc<Pci>) -> io::Result<Self> {
+        let mcfgs = Mcfgs::fetch()?;
+
+        Ok(Self {
+            lock: Mutex::new(()),
+            mcfgs,
+            maps: Mutex::new(BTreeMap::new()),
+            fallback,
+        })
+    }
+    fn addr_offset_in_bytes(starting_bus: u8, bus: u8, dev: u8, func: u8, offset: u16) -> usize {
+        assert_eq!(offset & 0xFFFC, offset, "pcie offset not dword-aligned");
+        assert_eq!(offset & 0x0FFF, offset, "pcie offset larger than 4095");
+        assert_eq!(dev & 0x1F, dev, "pcie dev number larger than 5 bits");
+        assert_eq!(func & 0x7, func, "pcie func number larger than 3 bits");
+
+        (((bus - starting_bus) as usize) << 20) | ((dev as usize) << 15) | ((func as usize) << 12) | (offset as usize)
+    }
+    fn addr_offset_in_dwords(starting_bus: u8, bus: u8, dev: u8, func: u8, offset: u16) -> usize {
+        Self::addr_offset_in_bytes(starting_bus, bus, dev, func, offset) / mem::size_of::<u32>()
+    }
+    unsafe fn with_pointer<T, F: FnOnce(Option<&mut u32>) -> T>(&self, bus: u8, dev: u8, func: u8, offset: u16, f: F) -> T {
+        let (base_address_phys, starting_bus) = match self.mcfgs.at_bus(bus) {
+            Some(t) => (t.base_addr, t.start_bus),
+            None => return f(None),
+        };
+        let mut maps_lock = self.maps.lock().unwrap();
+        let virt_pointer = maps_lock.entry((bus, dev, func)).or_insert_with(|| {
+            syscall::physmap(base_address_phys as usize + Self::addr_offset_in_bytes(starting_bus, bus, dev, func, 0), 4096, PhysmapFlags::PHYSMAP_NO_CACHE | PhysmapFlags::PHYSMAP_WRITE).unwrap_or_else(|error| panic!("failed to physmap pcie configuration space for {:2x}:{:2x}.{:2x}: {:?}", bus, dev, func, error)) as *mut u32
+        });
+        f(Some(&mut *virt_pointer.offset((offset as usize / mem::size_of::<u32>()) as isize)))
+    }
+    pub fn buses<'pcie>(&'pcie self) -> PciIter<'pcie> {
+        PciIter::new(self)
+    }
+}
+
+impl CfgAccess for Pcie {
+    unsafe fn read_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32 {
+        self.with_pointer(bus, dev, func, offset, |pointer| match pointer {
+            Some(address) => ptr::read_volatile::<u32>(address),
+            None => self.fallback.read(bus, dev, func, offset),
+        })
+    }
+    unsafe fn read(&self, bus: u8, dev: u8, func: u8, offset: u16) -> u32 {
+        let _guard = self.lock.lock().unwrap();
+        self.read_nolock(bus, dev, func, offset)
+    }
+    unsafe fn write_nolock(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32) {
+        self.with_pointer(bus, dev, func, offset, |pointer| match pointer {
+            Some(address) => ptr::write_volatile::<u32>(address, value),
+            None => { self.fallback.read(bus, dev, func, offset); }
+        });
+    }
+    unsafe fn write(&self, bus: u8, dev: u8, func: u8, offset: u16, value: u32) {
+        let _guard = self.lock.lock().unwrap();
+        self.write_nolock(bus, dev, func, offset, value);
+    }
+}
+
+impl Drop for Pcie {
+    fn drop(&mut self) {
+        for address in self.maps.lock().unwrap().values().copied() {
+            let _ = unsafe { syscall::physfree(address as usize, 4096) };
+        }
+    }
+}