From 443b160b6d67dd67d8fd36c9c34fc5d12a3631e8 Mon Sep 17 00:00:00 2001 From: 4lDO2 <4lDO2@protonmail.com> Date: Thu, 26 Mar 2020 18:18:00 +0100 Subject: [PATCH] Add internal locking to the Xhci struct. --- Cargo.lock | 172 +++++++++++++++ pcid/src/driver_interface.rs | 23 +- pcid/src/main.rs | 23 +- pcid/src/pci/msi.rs | 15 +- xhcid/Cargo.toml | 3 + xhcid/src/main.rs | 141 +++++++----- xhcid/src/xhci/capability.rs | 20 ++ xhcid/src/xhci/command.rs | 46 ---- xhcid/src/xhci/context.rs | 44 +++- xhcid/src/xhci/event.rs | 7 + xhcid/src/xhci/executor.rs | 1 - xhcid/src/xhci/irq_reactor.rs | 395 ++++++++++++++++++++++++++++++++++ xhcid/src/xhci/mod.rs | 279 ++++++++++++------------ xhcid/src/xhci/ring.rs | 53 ++++- xhcid/src/xhci/scheme.rs | 383 ++++++++++++++++---------------- xhcid/src/xhci/trb.rs | 71 +++++- 16 files changed, 1221 insertions(+), 455 deletions(-) delete mode 100644 xhcid/src/xhci/command.rs delete mode 100644 xhcid/src/xhci/executor.rs create mode 100644 xhcid/src/xhci/irq_reactor.rs diff --git a/Cargo.lock b/Cargo.lock index 43d3e114a3..2eeb19c3db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,6 +141,15 @@ name = "cfg-if" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "chashmap" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "clap" version = "2.33.0" @@ -259,6 +268,88 @@ name = "futures" version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "futures" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-executor 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-channel" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-executor" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-io" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "futures-sink" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-task" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "futures-util" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-macro 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.14 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "gpt" version = "0.6.3" @@ -419,6 +510,11 @@ name = "maybe-uninit" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "memoffset" version = "0.5.3" @@ -627,6 +723,14 @@ dependencies = [ "sdl2 0.32.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "owning_ref" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "owning_ref" version = "0.4.0" @@ -635,6 +739,15 @@ dependencies = [ "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "parking_lot" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parking_lot" version = "0.7.1" @@ -644,6 +757,17 @@ dependencies = [ "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "parking_lot_core" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "parking_lot_core" version = "0.4.0" @@ -716,11 +840,26 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pin-utils" +version = "0.1.0-alpha.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "plain" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "proc-macro-hack" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro-nested" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "proc-macro2" version = "1.0.8" @@ -747,6 +886,18 @@ dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand" version = "0.6.5" @@ -1527,6 +1678,9 @@ name = "xhcid" version = "0.1.0" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "chashmap 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "pcid 0.1.0", "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1558,6 +1712,7 @@ dependencies = [ "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum chashmap 2.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ff41a3c2c1e39921b9003de14bf0439c7b63a9039637c291e1a64925d8ddfa45" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum crc 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" @@ -1571,6 +1726,15 @@ dependencies = [ "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" +"checksum futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5c329ae8753502fb44ae4fc2b622fa2a94652c41e795143765ba0927f92ab780" +"checksum futures-channel 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c77d04ce8edd9cb903932b608268b3fffec4163dc053b3b402bf47eac1f1a8" +"checksum futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f25592f769825e89b92358db00d26f965761e094951ac44d3663ef25b7ac464a" +"checksum futures-executor 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f674f3e1bcb15b37284a90cedf55afdba482ab061c407a9c0ebbd0f3109741ba" +"checksum futures-io 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "a638959aa96152c7a4cddf50fcb1e3fede0583b27157c26e67d6f99904090dc6" +"checksum futures-macro 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9a5081aa3de1f7542a794a397cde100ed903b0630152d0973479018fd85423a7" +"checksum futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3466821b4bc114d95b087b850a724c6f83115e929bc88f1fa98a3304a944c8a6" +"checksum futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" +"checksum futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" "checksum gpt 0.6.3 (git+https://gitlab.redox-os.org/redox-os/gpt)" = "" "checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" @@ -1589,6 +1753,7 @@ dependencies = [ "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum memoffset 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75189eb85871ea5c2e2c15abbdd541185f63b408415e5051f5cac122d8c774b9" "checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" "checksum mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)" = "" @@ -1607,16 +1772,23 @@ dependencies = [ "checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" "checksum orbclient 0.3.27 (registry+https://github.com/rust-lang/crates.io-index)" = "f8b18f57ab94fbd058e30aa57f712ec423c0bb7403f8493a6c58eef0c36d9402" +"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" "checksum owning_ref 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "49a4b8ea2179e6a2e27411d3bca09ca6dd630821cf6894c6c7c8467a8ee7ef13" +"checksum parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "149d8f5b97f3c1133e3cfcd8886449959e856b557ff281e292b733d7c69e005e" "checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" +"checksum parking_lot_core 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "4db1a8ccf734a7bce794cc19b3df06ed87ab2f3907036b693c68f56b4d4537fa" "checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" "checksum partitionlib 0.1.0 (git+https://gitlab.redox-os.org/redox-os/partitionlib.git)" = "" "checksum pbr 1.0.2 (git+https://github.com/a8m/pb)" = "" "checksum pbr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4403eb718d70c03ee279e51737782902c68cca01e870a33b6a2f9dfb50b9cd83" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" "checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +"checksum proc-macro-hack 0.5.14 (registry+https://github.com/rust-lang/crates.io-index)" = "fcfdefadc3d57ca21cf17990a28ef4c0f7c61383a28cb7604cf4a18e6ede1420" +"checksum proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" "checksum proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" "checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" "checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" diff --git a/pcid/src/driver_interface.rs b/pcid/src/driver_interface.rs index e2f5b68d8c..28c06ef02a 100644 --- a/pcid/src/driver_interface.rs +++ b/pcid/src/driver_interface.rs @@ -10,21 +10,42 @@ use thiserror::Error; pub use crate::pci::PciBar; pub use crate::pci::msi; +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[repr(u8)] +pub enum LegacyInterruptPin { + /// INTa# + IntA = 1, + /// INTb# + IntB = 2, + /// INTc# + IntC = 3, + /// INTd# + IntD = 4, +} + #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub struct PciFunction { /// Number of PCI bus pub bus_num: u8, + /// Number of PCI device pub dev_num: u8, + /// Number of PCI function pub func_num: u8, + /// PCI Base Address Registers pub bars: [PciBar; 6], + /// BAR sizes pub bar_sizes: [u32; 6], + /// Legacy IRQ line - // TODO: Stop using legacy IRQ lines, and physical pins, but MSI/MSI-X instead. pub legacy_interrupt_line: u8, + + /// Legacy interrupt pin (INTx#), none if INTx# interrupts aren't supported at all. + pub legacy_interrupt_pin: Option, + /// Vendor ID pub venid: u16, /// Device ID diff --git a/pcid/src/main.rs b/pcid/src/main.rs index 42f28f29bf..63637cb50b 100644 --- a/pcid/src/main.rs +++ b/pcid/src/main.rs @@ -75,7 +75,10 @@ impl DriverHandler { None => return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(feature)), }; unsafe { - with_pci_func_raw(&self.state.pci, self.bus_num, self.dev_num, self.func_num, |func| capability.set_enabled(func, offset, true)); + with_pci_func_raw(&self.state.pci, self.bus_num, self.dev_num, self.func_num, |func| { + capability.set_enabled(true); + capability.write_message_control(func, offset); + }); } PcidClientResponse::FeatureEnabled(feature) } @@ -255,6 +258,8 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, pci.write(bus_num, dev_num, func_num, 0x3C, data); } + let interrupt_pin = unsafe { pci.read(bus_num, dev_num, func_num, 0x3B) }; + // Find BAR sizes let mut bars = [PciBar::None; 6]; let mut bar_sizes = [0; 6]; @@ -308,6 +313,21 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, }; println!("PCI DEVICE CAPABILITIES for {}: {:?}", args.iter().map(|string| string.as_ref()).nth(0).unwrap_or("[unknown]"), capabilities); + use driver_interface::LegacyInterruptPin; + + let legacy_interrupt_pin = match interrupt_pin { + 0 => None, + 1 => Some(LegacyInterruptPin::IntA), + 2 => Some(LegacyInterruptPin::IntB), + 3 => Some(LegacyInterruptPin::IntC), + 4 => Some(LegacyInterruptPin::IntD), + + other => { + println!("pcid: invalid interrupt pin: {}", other); + None + } + }; + let func = driver_interface::PciFunction { bars, bar_sizes, @@ -316,6 +336,7 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, func_num, devid: header.device_id(), legacy_interrupt_line: irq, + legacy_interrupt_pin, venid: header.vendor_id(), }; diff --git a/pcid/src/pci/msi.rs b/pcid/src/pci/msi.rs index 8b420501aa..d4400257f4 100644 --- a/pcid/src/pci/msi.rs +++ b/pcid/src/pci/msi.rs @@ -68,11 +68,10 @@ impl MsiCapability { pub fn message_control(&self) -> u16 { (self.message_control_raw() >> 16) as u16 } - pub unsafe fn set_message_control(&mut self, writer: &W, offset: u8, value: u16) { + pub fn set_message_control(&mut self, value: u16) { let mut new_message_control = self.message_control_raw(); new_message_control &= 0x0000_FFFF; new_message_control |= u32::from(value) << 16; - writer.write_u32(offset, new_message_control); match self { Self::_32BitAddress { ref mut message_control, .. } @@ -81,6 +80,9 @@ impl MsiCapability { | Self::_64BitAddressWithPvm { ref mut message_control, .. } => *message_control = new_message_control, } } + pub unsafe fn write_message_control(&mut self, writer: &W, offset: u8) { + writer.write_u32(offset, self.message_control_raw()); + } pub fn is_pvt_capable(&self) -> bool { self.message_control() & Self::MC_PVT_CAPABLE_BIT != 0 } @@ -90,10 +92,10 @@ impl MsiCapability { pub fn enabled(&self) -> bool { self.message_control() & Self::MC_MSI_ENABLED_BIT != 0 } - pub unsafe fn set_enabled(&mut self, writer: &W, offset: u8, enabled: bool) { + pub fn set_enabled(&mut self, enabled: bool) { let mut new_message_control = self.message_control() & (!Self::MC_MSI_ENABLED_BIT); new_message_control |= u16::from(enabled); - self.set_message_control(writer, offset, new_message_control) + self.set_message_control(new_message_control); } pub fn multi_message_capable(&self) -> u8 { ((self.message_control() & Self::MC_MULTI_MESSAGE_MASK) >> Self::MC_MULTI_MESSAGE_SHIFT) as u8 @@ -101,6 +103,11 @@ impl MsiCapability { pub fn multi_message_enabled(&self) -> u8 { ((self.message_control() & Self::MC_MULTI_MESSAGE_ENABLE_MASK) >> Self::MC_MULTI_MESSAGE_ENABLE_SHIFT) as u8 } + pub fn set_multi_message_enabled(&mut self, log_mme: u8) { + let mut new_message_control = self.message_control() & (!Self::MC_MULTI_MESSAGE_ENABLE_MASK); + new_message_control |= (u16::from(log_mme) << Self::MC_MULTI_MESSAGE_ENABLE_SHIFT); + self.set_message_control(new_message_control); + } } impl MsixCapability { diff --git a/xhcid/Cargo.toml b/xhcid/Cargo.toml index 0ef92d5fd7..e46a79a13b 100644 --- a/xhcid/Cargo.toml +++ b/xhcid/Cargo.toml @@ -13,6 +13,9 @@ path = "src/lib.rs" [dependencies] bitflags = "1" +chashmap = "2.2.2" +crossbeam-channel = "0.4" +futures = "0.3" plain = "0.2" lazy_static = "1.4" spin = "0.4" diff --git a/xhcid/src/main.rs b/xhcid/src/main.rs index a415b5cc24..1a3e35e03b 100644 --- a/xhcid/src/main.rs +++ b/xhcid/src/main.rs @@ -8,13 +8,14 @@ use pcid_interface::{PcidServerHandle, PciFeature, PciFeatureInfo}; use pcid_interface::msi::{MsiCapability, MsixCapability, MsixTableEntry}; use event::{Event, EventQueue}; -use std::cell::RefCell; use std::convert::TryInto; use std::fs::{self, File}; +use std::future::Future; use std::io::{self, Read, Write}; use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use std::pin::Pin; use std::ptr::NonNull; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use std::env; use syscall::data::Packet; use syscall::error::EWOULDBLOCK; @@ -22,7 +23,7 @@ use syscall::flag::{CloneFlags, PHYSMAP_NO_CACHE, PHYSMAP_WRITE}; use syscall::scheme::SchemeMut; use syscall::io::Io; -use crate::xhci::Xhci; +use crate::xhci::{InterruptMethod, Xhci}; mod driver_interface; mod usb; @@ -71,6 +72,10 @@ fn allocate_interrupt_vector() -> io::Result> { Ok(None) } +async fn handle_packet(hci: Arc, packet: Packet) -> Packet { + todo!() +} + fn main() { // Daemonize if unsafe { syscall::clone(CloneFlags::empty()).unwrap() } != 0 { @@ -103,15 +108,26 @@ fn main() { dbg!(has_msi, msi_enabled); dbg!(has_msix, msix_enabled); - if has_msi && !msi_enabled { + if has_msi && !msi_enabled && !has_msix { + pcid_handle.enable_feature(PciFeature::Msi).expect("xhcid: failed to enable MSI"); + println!("Enabled MSI"); msi_enabled = true; } if has_msix && !msix_enabled { + pcid_handle.enable_feature(PciFeature::MsiX).expect("xhcid: failed to enable MSI-X"); + println!("Enabled MSI-X"); msix_enabled = true; } - let (mut irq_file, msix_info) = if msi_enabled && !msix_enabled { - todo!("only msi-x is currently implemented") + let (mut irq_file, interrupt_method) = if msi_enabled && !msix_enabled { + let mut capability = match pcid_handle.feature_info(PciFeature::MsiX).expect("xhcid: failed to retrieve the MSI capability structure from pcid") { + PciFeatureInfo::Msi(s) => s, + PciFeatureInfo::MsiX(_) => panic!(), + }; + // use one vector + capability.set_multi_message_enabled(0); + + todo!("msi (msix is implemented though)") } else if msix_enabled { let capability = match pcid_handle.feature_info(PciFeature::MsiX).expect("xhcid: failed to retrieve the MSI-X capability structure from pcid") { PciFeatureInfo::Msi(_) => panic!(), @@ -160,17 +176,19 @@ fn main() { let (vector, interrupt_handle) = allocate_interrupt_vector().expect("xhcid: failed to allocate interrupt vector").expect("xhcid: no interrupt vectors left"); let msg_data = x86_64_msix::message_data_edge_triggered(DeliveryMode::Fixed, vector); - dbg!(vector, destination_id); - table_entry_pointer.addr_lo.write(addr); table_entry_pointer.addr_hi.write(0); table_entry_pointer.msg_data.write(msg_data); table_entry_pointer.vec_ctl.writef(MsixTableEntry::VEC_CTL_MASK_BIT, false); - (interrupt_handle, Some(info)) + (Some(interrupt_handle), InterruptMethod::MsiX(info)) } + } else if pci_config.func.legacy_interrupt_pin.is_some() { + // legacy INTx# interrupt pins. + (Some(File::open(format!("irq:{}", irq)).expect("xhcid: failed to open legacy IRQ file")), InterruptMethod::Intx) } else { - (File::open(format!("irq:{}", irq)).expect("xhcid: failed to open legacy IRQ file"), None) + // no interrupts at all + (None, InterruptMethod::Polling) }; std::thread::sleep(std::time::Duration::from_millis(300)); @@ -187,27 +205,25 @@ fn main() { let socket_fd = syscall::open( format!(":usb/{}", name), - syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK, + syscall::O_RDWR | syscall::O_CREAT, ) .expect("xhcid: failed to create usb scheme"); - let socket = Arc::new(RefCell::new(unsafe { + let socket = Arc::new(Mutex::new(unsafe { File::from_raw_fd(socket_fd as RawFd) })); - { - let hci = Arc::new(RefCell::new( - Xhci::new(name, address, msi_enabled, msix_enabled, msix_info, pcid_handle).expect("xhcid: failed to allocate device"), - )); + let hci = Arc::new(Xhci::new(name, address, interrupt_method, irq_file.as_mut(), pcid_handle).expect("xhcid: failed to allocate device")); + hci.probe().expect("xhcid: failed to probe"); - hci.borrow_mut().probe().expect("xhcid: failed to probe"); + let mut event_queue = + EventQueue::<()>::new().expect("xhcid: failed to create event queue"); - let mut event_queue = - EventQueue::<()>::new().expect("xhcid: failed to create event queue"); + syscall::setrens(0, 0).expect("xhcid: failed to enter null namespace"); - syscall::setrens(0, 0).expect("xhcid: failed to enter null namespace"); - - let todo = Arc::new(RefCell::new(Vec::::new())); + let todo = Arc::new(Mutex::new(Vec::::new())); + let todo_futures = Arc::new(Mutex::new(Vec:: + Send + Sync + 'static>>>::new())); + if let Some(irq_file) = irq_file { let hci_irq = hci.clone(); let socket_irq = socket.clone(); let todo_irq = todo.clone(); @@ -216,21 +232,24 @@ fn main() { let mut irq = [0; 8]; irq_file.read(&mut irq)?; - if hci_irq.borrow_mut().received_irq() { - hci_irq.borrow_mut().on_irq(); + let hci = hci_irq.lock().unwrap(); + let socket = socket_irq.lock().unwrap(); + let todo = todo_irq.lock().unwrap(); + + if hci.received_irq() { + hci.on_irq(); irq_file.write(&mut irq)?; - let mut todo = todo_irq.borrow_mut(); let mut i = 0; while i < todo.len() { let a = todo[i].a; - hci_irq.borrow_mut().handle(&mut todo[i]); + hci.handle(&mut todo[i]); if todo[i].a == (-EWOULDBLOCK) as usize { todo[i].a = a; i += 1; } else { - socket_irq.borrow_mut().write(&mut todo[i])?; + socket.write(&todo[i])?; todo.remove(i); } } @@ -239,39 +258,43 @@ fn main() { Ok(None) }) .expect("xhcid: failed to catch events on IRQ file"); - - let socket_fd = socket.borrow().as_raw_fd(); - let socket_packet = socket.clone(); - event_queue - .add(socket_fd, move |_| -> io::Result> { - loop { - let mut packet = Packet::default(); - match socket_packet.borrow_mut().read(&mut packet) { - Ok(0) => break, - Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, - Ok(_) => (), - Err(err) => return Err(err), - } - - let a = packet.a; - hci.borrow_mut().handle(&mut packet); - if packet.a == (-EWOULDBLOCK) as usize { - packet.a = a; - todo.borrow_mut().push(packet); - } else { - socket_packet.borrow_mut().write(&mut packet)?; - } - } - Ok(None) - }) - .expect("xhcid: failed to catch events on scheme file"); - - event_queue - .trigger_all(Event { fd: 0, flags: 0 }) - .expect("xhcid: failed to trigger events"); - - event_queue.run().expect("xhcid: failed to handle events"); } + + let socket_fd = socket.lock().unwrap().as_raw_fd(); + let socket_packet = socket.clone(); + event_queue + .add(socket_fd, move |_| -> io::Result> { + let mut socket = socket_packet.lock().unwrap(); + let mut hci = hci.lock().unwrap(); + let mut todo = todo.lock().unwrap(); + + loop { + let mut packet = Packet::default(); + match socket.read(&mut packet) { + Ok(0) => break, + Ok(_) => (), + Err(err) => return Err(err), + } + + let a = packet.a; + hci.handle(&mut packet); + if packet.a == (-EWOULDBLOCK) as usize { + packet.a = a; + todo.push(packet); + } else { + socket.write(&packet)?; + } + } + Ok(None) + }) + .expect("xhcid: failed to catch events on scheme file"); + + event_queue + .trigger_all(Event { fd: 0, flags: 0 }) + .expect("xhcid: failed to trigger events"); + + event_queue.run().expect("xhcid: failed to handle events"); + unsafe { let _ = syscall::physunmap(address); } diff --git a/xhcid/src/xhci/capability.rs b/xhcid/src/xhci/capability.rs index 59e841b24e..98750bc12c 100644 --- a/xhcid/src/xhci/capability.rs +++ b/xhcid/src/xhci/capability.rs @@ -27,6 +27,13 @@ pub const HCS_PARAMS1_MAX_PORTS_SHIFT: u8 = 24; pub const HCS_PARAMS1_MAX_SLOTS_MASK: u32 = 0x0000_00FF; pub const HCS_PARAMS1_MAX_SLOTS_SHIFT: u8 = 0; +pub const HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_LO_MASK: u32 = 0xF800_0000; +pub const HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_LO_SHIFT: u8 = 27; +pub const HCS_PARAMS2_SPR_BIT: u32 = 1 << HCS_PARAMS2_SPR_SHIFT; +pub const HCS_PARAMS2_SPR_SHIFT: u8 = 26; +pub const HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_HI_MASK: u32 = 0x03E0_0000; +pub const HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_HI_SHIFT: u8 = 21; + impl CapabilityRegs { pub fn lec(&self) -> bool { self.hcc_params2.readf(HCC_PARAMS2_LEC_BIT) @@ -48,4 +55,17 @@ impl CapabilityRegs { pub fn ext_caps_ptr_in_dwords(&self) -> u16 { ((self.hcc_params1.read() & HCC_PARAMS1_XECP_MASK) >> HCC_PARAMS1_XECP_SHIFT) as u16 } + pub fn max_scratchpad_bufs_lo(&self) -> u8 { + ((self.hcs_params2.read() & HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_LO_MASK) >> HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_LO_SHIFT) + } + pub fn spr(&self) -> bool { + self.hcs_params2.readf(HCS_PARAMS2_SPR_BIT) + } + pub fn max_scratchpad_bufs_hi(&self) -> u8 { + ((self.hcs_params2.read() & HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_HI_MASK) >> HCS_PARAMS2_MAX_SCRATCHPAD_BUFS_HI_SHIFT) + } + pub fn max_scratchpad_bufs(&self) -> u16 { + u16::from(self.max_scratchpad_bufs_lo()) + | (u16::from(self.max_scratchpad_bufs_hi()) << 5) + } } diff --git a/xhcid/src/xhci/command.rs b/xhcid/src/xhci/command.rs deleted file mode 100644 index c82dc4311e..0000000000 --- a/xhcid/src/xhci/command.rs +++ /dev/null @@ -1,46 +0,0 @@ -use syscall::error::Result; - -use super::event::EventRing; -use super::ring::Ring; -use super::trb::Trb; - -pub struct CommandRing { - pub ring: Ring, - pub events: EventRing, -} - -impl CommandRing { - pub fn new() -> Result { - Ok(CommandRing { - ring: Ring::new(16, true)?, - events: EventRing::new()?, - }) - } - - pub fn crcr(&self) -> u64 { - self.ring.register() - } - - pub fn erdp(&self) -> u64 { - let address = self.events.ring.register(); - address & 0xFFFF_FFFF_FFFF_FFF0 - } - - pub fn erstba(&self) -> u64 { - self.events.ste.physical() as u64 - } - - pub fn next(&mut self) -> (&mut Trb, bool, &mut Trb) { - let cmd = self.ring.next(); - let event = self.events.next(); - (cmd.0, cmd.1, event) - } - - pub fn next_cmd(&mut self) -> (&mut Trb, bool) { - self.ring.next() - } - - pub fn next_event(&mut self) -> &mut Trb { - self.events.next() - } -} diff --git a/xhcid/src/xhci/context.rs b/xhcid/src/xhci/context.rs index 74944da174..f3eb970737 100644 --- a/xhcid/src/xhci/context.rs +++ b/xhcid/src/xhci/context.rs @@ -75,7 +75,7 @@ impl InputContext { pub struct DeviceContextList { pub dcbaa: Dma<[u64; 256]>, - pub contexts: Vec>, + pub contexts: Box<[Dma]>, } impl DeviceContextList { @@ -91,8 +91,8 @@ impl DeviceContextList { } Ok(DeviceContextList { - dcbaa: dcbaa, - contexts: contexts, + dcbaa, + contexts: contexts.into_boxed_slice(), }) } @@ -159,3 +159,41 @@ impl StreamContextArray { self.contexts.physical() as u64 } } + +#[repr(packed)] +#[derive(Clone, Debug)] +pub struct ScratchpadBufferEntry { + pub value: Mmio, +} +impl ScratchpadBufferEntry { + pub fn set_addr(&mut self, addr: u64) { + self.value.write(addr); + } +} + +pub struct ScratchpadBufferArray { + pub entries: Dma<[ScratchpadBufferEntry]>, + pub pages: Vec, +} +impl ScratchpadBufferArray { + pub fn new(entries: u16) -> Result { + let mut entries = unsafe { Dma::zeroed_unsized(entries as usize)? }; + + let pages = entries.iter_mut().map(|entry| { + // TODO: When a better memory allocation API arrives (like the `mem:` scheme which I think is + // being worked on), no assumptions about the page size always being 4k will have to be + // made. + let pointer = syscall::physalloc(4096); + assert_eq!(pointer & 0xFFFF_FFFF_FFFF_F000, pointer, "physically allocated pointer (physalloc) wasn't 4k page-aligned"); + entry.set_addr(pointer); + }); + + Ok(Self { + entries, + pages, + }) + } + pub fn register(&self) -> u64 { + self.entries.physical() + } +} diff --git a/xhcid/src/xhci/event.rs b/xhcid/src/xhci/event.rs index b8301b34be..b3e0af0e35 100644 --- a/xhcid/src/xhci/event.rs +++ b/xhcid/src/xhci/event.rs @@ -12,6 +12,7 @@ pub struct EventRingSte { _rsvd2: Mmio, } +// TODO: Use atomic operations, and perhaps an occasional lock for reallocating. pub struct EventRing { pub ste: Dma<[EventRingSte]>, pub ring: Ring, @@ -33,4 +34,10 @@ impl EventRing { pub fn next(&mut self) -> &mut Trb { self.ring.next().0 } + pub fn erdp(&self) -> u64 { + self.ring.register() & 0xFFFF_FFFF_FFFF_FFF0 + } + pub fn erstba(&self) -> u64 { + self.ste.physical() as u64 + } } diff --git a/xhcid/src/xhci/executor.rs b/xhcid/src/xhci/executor.rs deleted file mode 100644 index 01fe6603d8..0000000000 --- a/xhcid/src/xhci/executor.rs +++ /dev/null @@ -1 +0,0 @@ -use super::Xhci; diff --git a/xhcid/src/xhci/irq_reactor.rs b/xhcid/src/xhci/irq_reactor.rs new file mode 100644 index 0000000000..a8b777f37a --- /dev/null +++ b/xhcid/src/xhci/irq_reactor.rs @@ -0,0 +1,395 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::sync::atomic::{self, AtomicUsize}; +use std::{mem, task, thread}; + +use crossbeam_channel::{Sender, Receiver}; +use futures::Stream; + +use super::Xhci; +use super::ring::Ring; +use super::trb::{Trb, TrbCompletionCode, TrbType}; + +/// Short-term states (as in, they are removed when the waker is consumed, but probably pushed back +/// by the future unless it completed). +pub struct State { + waker: task::Waker, + kind: StateKind, + message: Arc>>, + is_isoch_or_vf: bool, +} + +pub struct NextEventTrb { + pub event_trb: Trb, + pub src_trb: Option, +} + +// TODO: Perhaps all of the transfer rings used by the xHC should be stored linearly, and then +// indexed using this struct instead. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct RingId { + pub slot: u8, + pub endpoint_num: u8, + pub stream_id: u16, +} + +/// The state specific to a TRB-type. Since some of the event TDs may asynchronously appear, for +/// example the Command Completion Event and the Transfer Event TDs, they have to be +/// distinguishable. Luckily, the xHC also gives us the actual (physical) pointer to the source +/// TRB, from the command ring, unless the event TD has one the completion codes Ring Underrun, +/// Ring Overrun, or VF Event Ring Full Error. When these errors are encountered, it simply +/// indicates that the commands causing the errors continue to be pending, and thus no information +/// is lost. +#[derive(Clone, Copy, Debug)] +pub enum StateKind { + CommandCompletion { phys_ptr: u64 }, + Transfer { phys_ptr: u64, ring_id: RingId }, + Other(TrbType), +} + +impl StateKind { + pub fn trb_type(&self) -> TrbType { + match self.kind { + Self::CommandCompletion { .. } => TrbType::CommandCompletion, + Self::Transfer { .. } => TrbType::Transfer, + Self::Other(ty) => ty, + } + } +} + + +pub struct IrqReactor { + hci: Arc, + current_count: Arc, + irq_file: Option, + receiver: Receiver, + + states: Vec, + + // TODO: Since the IRQ reactor is the only part of this driver that gets event TRBs, perhaps + // the event ring should be owned here? +} + +pub type NewPendingTrb = State; + +pub fn start_irq_reactor(hci: Arc, irq_file: Option) -> thread::JoinHandle<()> { + thread::spawn(move || { + IrqReactor::new(hci, irq_file).run() + }) +} + +impl IrqReactor { + pub fn new(hci: Arc, irq_file: Option) -> Self { + Self { + hci, + irq_file, + current_count: Arc::new(AtomicUsize::new()), + } + } + // TODO: Configure the amount of time to be awaited when no more work can be done. + fn pause(&self) { + std::thread::yield_now(); + } + fn run_polling(mut self) { + loop { + self.handle_requests(); + + let index = self.hci.primary_event_ring.lock().unwrap().next_index(); + + let mut trb; + + 'busy_waiting: loop { + trb = self.hci.primary_event_ring.lock().unwrap().trbs[index]; + + if trb.completion_code() == TrbCompletionCode::Invalid as u8 { + self.pause(); + continue 'busy_waiting; + } + } + self.acknowledge(trb); + self.update_erdp(); + } + } + fn run_with_irq_file(mut self) { + 'event_loop: loop { + self.handle_requests(); + + let irq_file = self.irq_file.as_mut().unwrap(); + + let mut buffer = [0u8; 8]; + let bytes_read = self.irq_file.read(&mut buffer).expect("Failed to read from irq scheme"); + if bytes_read < mem::size_of::() { + panic!("wrong number of bytes read from `irq:`: expected {}, got {}", mem::size_of::(), bytes_read); + } + + if !self.hci.received_irq() { + continue; + } + + let _ = self.irq_file.write(&buffer); + + // TODO: More event rings, probably even with different IRQs. + + let event_ring = self.hci.primary_event_ring.lock().unwrap(); + let trb = event_ring.next(); + + if trb.completion_code() == TrbCompletionCode::Invalid as u8 { + println!("xhci: Received interrupt, but no event was found in the event ring. Ignoring interrupt."); + continue 'event_loop; + } + + self.acknowledge(*trb); + trb.reserved(false); + + self.update_erdp(); + } + } + fn update_erdp(&self) { + let dequeue_pointer_and_dcs = self.hci.primary_event_ring.lock().unwrap().register(); + let dequeue_pointer = dequeue_pointer_and_dcs & 0xFFFF_FFFF_FFFF_FFFE; + assert_eq!(dequeue_pointer & 0xFFFF_FFFF_FFFF_FFF0, dequeue_pointer, "unaligned ERDP received from primary event ring"); + + self.xhci.run.lock().unwrap().ints[0].erdp.write(dequeue_pointer); + } + fn handle_requests(&mut self) { + self.states.extend(self.receiver.try_iter()); + } + fn acknowledge(&mut self, trb: Trb) { + let mut index = 0; + + loop { + match self.states[index].kind { + StateKind::CommandCompletion { phys_ptr } if trb.trb_type() == TrbType::CommandCompletion as u8 => if trb.completion_trb_pointer() == Some(phys_ptr) { + let state = self.states.remove(index).unwrap(); + + // Before waking, it's crucial that the command TRB that generated this event + // be fetched before removing this event TRB from the queue. + let command_trb = match self.hci.command_ring.lock().unwrap().ring.phys_addr_to_entry_mut(phys_ptr) { + Some(command_trb) => { + let t = command_trb.clone(); + command_trb.reserved(false); + t + }, + None => { + println!("The xHC supplied a pointer to a command TRB that was outside the known command ring bounds. Ignoring event TRB {:?}.", trb); + continue; + } + }; + + // TODO: Validate the command TRB. + *state.message.lock().unwrap() = Some(NextEventTrb { + src_trb: command_trb.clone(), + event_trb: trb, + }); + + state.waker.wake(); + } else if trb.completion_trb_pointer().is_none() { + println!("Command TRB somehow resulted in an error that only can be caused by transfer TRBs. Ignoring event TRB: {:?}.", trb); + continue; + } else { + // The event TRB simply didn't match the current future + continue; + } + + StateKind::Transfer { phys_ptr, ring_id } if trb.trb_type() == TrbType::Transfer as u8 => if let Some(src_trb) = self.xhc.lock().unwrap().get_transfer_trb(trb.transfer_event_trb_pointer(), ring_id) { + if trb.transfer_event_trb_pointer() == Some(phys_ptr) { + // Give the source transfer TRB together with the event TRB, to the future. + + let state = self.states.remove(index).unwrap(); + *state.message.lock().unwrap() = Some(NextEventTrb { + src_trb, + event_trb: trb, + }); + state.waker.wake(); + } else if trb.transfer_event_trb_pointer().is_none() { + // Ring Overrun, Ring Underrun, or Virtual Function Event Ring Full. + // + // These errors are caused when either an isoch transfer that shall write data, doesn't + // have any data since the ring is empty, or if an isoch receive is impossible due to a + // full ring. The Virtual Function Event Ring Full is only for Virtual Machine + // Managers, and since this isn't implemented yet, they are irrelevant. + // + // The best solution here is to differentiate between isoch transfers (and + // virtual function event rings when virtualization gets implemented), with + // regular commands and transfers, and send the error TRB to all of them, or + // possibly an error code wrapped in a Result. + self.acknowledge_failed_transfer_trbs(trb); + return; + } else { + // The event TRB simply didn't match the current future + continue; + } + } else { continue } + + StateKind::Other(trb_type) if trb_type as u8 == trb.trb_type() => { + let state = self.states.remove(index).unwrap(); + state.waker.wake(); + } + + _ => { + index += 1; + if index >= self.states.len() { + break; + } + continue; + } + } + } + } + pub fn acknowledge_failed_transfer_trbs(&mut self, trb: Trb) { + let mut index = 0; + + loop { + if ! self.states[index].is_isoch_or_vf { + index += 1; + if index >= self.states.len() { + break; + } + continue; + } + let state = self.states.remove(index).unwrap(); + *state.message.lock().unwrap() = Some(NextEventTrb { + event_trb: trb, + src_trb: None, + }); + state.waker.wake(); + } + } + pub fn run(mut self) { + if self.irq_file.is_some() { + self.run_with_irq_file(); + } else { + self.run_polling(); + } + } +} + +struct FutureState { + message: Arc>>, + is_isoch_or_vf: bool, + state_kind: StateKind, +} + +enum EventTrbFuture { + Pending { state: FutureState, sender: Sender, }, + Finished, +} + +impl Future for EventTrbFuture { + type Output = NextEventTrb; + + fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { + match self.get_mut() { + &mut Self::Pending { ref mut state, ref mut sender } => if let Some(message) = state.message.lock().unwrap().take() { + *self.get_mut() = Self::Finished; + + task::Poll::Ready(message) + } else { + sender.send(State { + message: Arc::clone(&state.message), + is_isoch_or_vf: state.is_isoch_or_vf, + state_kind: state.state_kind, + waker: context.waker().clone(), + }).expect("IRQ reactor thread unexpectedly stopped"); + + task::Poll::Pending + } + &mut Self::Finished => panic!("Polling finished EventTrbFuture again."), + } + } +} + +impl Xhci { + pub fn get_transfer_trb(&self, paddr: u64, id: RingId) -> Option { + self.with_ring(id, |ring| ring.phys_addr_to_entry(paddr)) + } + pub fn with_ring T>(&self, id: RingId, function: F) -> T { + use super::RingOrStreams; + + let slot_state = self.slot_states.get(&id.slot)?; + let endpoint_state = slot_state.endpoint_states.get(&id.endpoint_num)?; + + let ring_ref = match endpoint_state.transfer { + RingOrStreams::Ring(ref ring) => ring, + RingOrStreams::Streams(ref ctx_arr) => ctx_arr.rings.get(&id.stream_id)?, + }; + + function(ring_ref) + } + pub fn with_ring_mut T>(&self, id: RingId, function: F) -> T { + use super::RingOrStreams; + + let slot_state = self.slot_states.get(&id.slot)?; + let endpoint_state = slot_state.endpoint_states.get_mut(&id.endpoint_num)?; + + let ring_ref = match endpoint_state.transfer { + RingOrStreams::Ring(ref mut ring) => ring, + RingOrStreams::Streams(ref mut ctx_arr) => ctx_arr.rings.get_mut(&id.stream_id)?, + }; + + function(ring_ref) + } + pub fn next_transfer_event_trb(&self, ring_id: RingId, trb: &Trb) -> impl Future + Send + Sync + 'static { + if ! trb.is_transfer_trb() { + panic!("Invalid TRB type given to next_transfer_event_trb(): {} (TRB {:?}. Expected transfer TRB.", trb.trb_type(), trb) + } + + let is_isoch_or_vf = trb.trb_type() == TrbType::Isoch as u8; + + EventTrbFuture::Pending { + state: FutureState { + is_isoch_or_vf, + state_kind: StateKind::Transfer { + ring_id, + phys_ptr: self.with_ring(ring_id, |ring| ring.trb_phys_ptr(trb)/*.expect("Invalid TRB: transfer TRB wasn't in the ring specified. Only direct references to the TRBs of a ring can be used (ring address range: {:p}-{:p}).", ring.start_addr(), ring.end_addr())*/), + }, + message: Arc::new(Mutex::new(None)), + }, + sender: self.irq_reactor_sender.clone(), + } + } + pub fn next_command_completion_event_trb(&self, trb: &Trb) -> impl Future + Send + Sync + 'static { + if ! trb.is_command_trb() { + panic!("Invalid TRB type given to next_command_completion_event_trb(): {} (TRB {:?}. Expected command TRB.", trb.trb_type(), trb) + } + + let command_ring = self.cmd.lock().unwrap(); + let ring = &command_ring.ring; + + EventTrbFuture::Pending { + state: FutureState { + // This is only possible for transfers if they are isochronous, or for Force Event TRBs (virtualization). + is_isoch_or_vf: false, + state_kind: StateKind::CommandCompletion { + phys_ptr: ring.trb_phys_ptr(trb),//.expect("Invalid TRB: expected a command TRB within the address range of the command TRB ({:p} {:p}), found TRB {:?} at {:p}", ring.start_addr(), ring.end_addr(), trb, trb) + }, + message: Arc::new(Mutex::new(None)), + }, + sender: self.irq_reactor_sender.clone(), + } + } + pub fn next_misc_event_trb(&self, trb_type: TrbType) -> impl Future + Send + Sync + 'static { + let valid_trb_types = [ + TrbType::PortStatusChange as u8, + TrbType::BandwidthRequest as u8, + TrbType::Doorbell as u8, + TrbType::HostController as u8, + TrbType::DeviceNotification as u8, + TrbType::MfindexWrap as u8, + ]; + if ! valid_trb_types.contains(&trb_type) { + panic!("Invalid TRB type given to next_misc_event_trb(): {:?}. Only event TRB types that are neither transfer events or command completion events can be used.", trb_type) + } + EventTrbFuture::Pending { + state: FutureState { + is_isoch_or_vf: false, + state_kind: StateKind::Other(trb_type), + message: Arc::new(Mutex::new(None)), + }, + sender: self.irq_reactor_sender.clone(), + } + } +} diff --git a/xhcid/src/xhci/mod.rs b/xhcid/src/xhci/mod.rs index 813ec2728b..3970677053 100644 --- a/xhcid/src/xhci/mod.rs +++ b/xhcid/src/xhci/mod.rs @@ -1,10 +1,14 @@ use std::collections::BTreeMap; use std::convert::TryFrom; +use std::fs::File; +use std::future::Future; use std::pin::Pin; use std::ptr::NonNull; use std::sync::{atomic::AtomicBool, Arc, Mutex, Weak}; -use std::{mem, process, slice, sync::atomic, task}; +use std::{mem, process, slice, sync::atomic, task, thread}; +use chashmap::CHashMap; +use crossbeam_channel::Sender; use serde::Deserialize; use syscall::error::{Error, Result, EBADF, EBADMSG, ENOENT}; use syscall::io::{Dma, Io}; @@ -15,12 +19,11 @@ use pcid_interface::msi::{MsixTableEntry, MsixCapability}; use pcid_interface::{PcidServerHandle, PciFeature}; mod capability; -mod command; mod context; mod doorbell; mod event; -pub mod executor; mod extended; +pub mod irq_reactor; mod operational; mod port; mod ring; @@ -29,9 +32,10 @@ pub mod scheme; mod trb; use self::capability::CapabilityRegs; -use self::command::CommandRing; -use self::context::{DeviceContextList, InputContext, StreamContextArray}; +use self::context::{DeviceContextList, InputContext, ScratchpadBufferArray, StreamContextArray}; use self::doorbell::Doorbell; +use self::irq_reactor::NewPendingTrb; +use self::event::EventRing; use self::extended::{CapabilityId, ExtendedCapabilitiesIter, ProtocolSpeed, SupportedProtoCap}; use self::operational::OperationalRegs; use self::port::Port; @@ -43,6 +47,20 @@ use self::scheme::EndpIfState; use crate::driver_interface::*; +pub enum InterruptMethod { + /// No interrupts whatsoever; the driver will instead rely on polling event rings. + Polling, + + /// Legacy PCI INTx# interrupt pin. + Intx, + + /// Message signaled interrupts. + Msi, + + /// Extended message signaled interrupts. + MsiX(Mutex), +} + pub struct MsixInfo { pub virt_table_base: NonNull, pub virt_pba_base: NonNull, @@ -72,7 +90,7 @@ impl MsixInfo { struct Device<'a> { ring: &'a mut Ring, - cmd: &'a mut CommandRing, + cmd: &'a mut Ring, db: &'a mut Doorbell, int: &'a mut Interrupter, } @@ -104,6 +122,7 @@ impl<'a> Device<'a> { { let event = self.cmd.next_event(); + // TODO: Replace polling here as well. while event.data.read() == 0 { println!(" - Waiting for event"); } @@ -144,43 +163,43 @@ impl<'a> Device<'a> { } pub struct Xhci { - cap: &'static mut CapabilityRegs, - op: &'static mut OperationalRegs, - ports: &'static mut [Port], - dbs: &'static mut [Doorbell], - run: &'static mut RuntimeRegs, - dev_ctx: DeviceContextList, - cmd: CommandRing, + // immutable + cap: &'static CapabilityRegs, + // XXX: It would be really useful to be able to mutably access individual elements of a slice, + // without having to wrap every element in a lock (which wouldn't work since they're packed). + op: Mutex<&'static mut OperationalRegs>, + ports: Mutex<&'static mut [Port]>, + dbs: Mutex<&'static mut [Doorbell]>, + run: Mutex<&'static mut RuntimeRegs>, + cmd: Mutex, + primary_event_ring: Mutex, + + // immutable + dev_ctx: DeviceContextList, + scratchpad_buf_arr: Option, + + // used for the extended capabilities, and so far none of them are mutated, and thus no lock. base: *const u8, - handles: BTreeMap, + handles: CHashMap, next_handle: usize, - port_states: BTreeMap, + port_states: CHashMap, - // TODO: Is this the correct implementation? I mean, there will be a really limited number of - // IRQs, if not just one, and since we probably wont use a thread pool scheduler like those of - // async-std or tokio, one could possibly assume that the futures themselves won't have to push - // all the wakers. - // TODO: This should probably be a BTreeMap (or just a VecMap) of states for each IRQ number, - // if more than one are used. I'm not sure if the XHCI interrupters actually use different - // IRQs, but it would make sense in case the hub has both isochronous (which trigger interrupts - // reapeatedly with some time in between), bulk, control, etc. I might be wrong though... - irq_state: Arc, - - drivers: BTreeMap, + drivers: CHashMap, scheme_name: String, - msi: bool, - msix: bool, - msix_info: Option, - - pcid_handle: PcidServerHandle, + interrupt_method: InterruptMethod, + pcid_handle: Mutex, + irq_reactor: Option>, + irq_reactor_sender: Sender, } struct PortState { slot: u8, - input_context: Dma, + cfg_idx: Option, + if_idx: Option, + input_context: Mutex>, dev_desc: DevDesc, endpoint_states: BTreeMap, } @@ -204,7 +223,7 @@ impl EndpointState { } impl Xhci { - pub fn new(scheme_name: String, address: usize, msi: bool, msix: bool, msix_info: Option, handle: PcidServerHandle) -> Result { + pub fn new(scheme_name: String, address: usize, interrupt_method: InterruptMethod, pcid_handle: PcidServerHandle) -> Result { let cap = unsafe { &mut *(address as *mut CapabilityRegs) }; println!(" - CAP {:X}", address); @@ -265,21 +284,17 @@ impl Xhci { dbs, run, dev_ctx: DeviceContextList::new(max_slots)?, - cmd: CommandRing::new()?, + cmd: Ring::new(), + primary_event_ring: EventRing::new(), handles: BTreeMap::new(), next_handle: 0, port_states: BTreeMap::new(), - irq_state: Arc::new(IrqState { - triggered: AtomicBool::new(false), - wakers: Mutex::new(Vec::new()), - }), drivers: BTreeMap::new(), scheme_name, - msi, - msix, - msix_info, - pcid_handle: handle, + + interrupt_method, + pcid_handle: Mutex::new(pcid_handle), }; xhci.init(max_slots); @@ -290,87 +305,87 @@ impl Xhci { pub fn init(&mut self, max_slots: u8) { // Set enabled slots println!(" - Set enabled slots to {}", max_slots); - self.op.config.write(max_slots as u32); - println!(" - Enabled Slots: {}", self.op.config.read() & 0xFF); + self.op.get_mut().config.write(max_slots as u32); + println!(" - Enabled Slots: {}", self.op.get_mut().config.read() & 0xFF); // Set device context address array pointer let dcbaap = self.dev_ctx.dcbaap(); println!(" - Write DCBAAP: {:X}", dcbaap); - self.op.dcbaap.write(dcbaap as u64); + self.op.get_mut().dcbaap.write(dcbaap as u64); // Set command ring control register - let crcr = self.cmd.crcr(); + let crcr = self.cmd.get_mut().register(); + assert_eq!(crcr & 0xFFFF_FFFF_FFFF_FFC1, crcr, "unaligned CRCR"); println!(" - Write CRCR: {:X}", crcr); - self.op.crcr.write(crcr as u64); + self.op.get_mut().crcr.write(crcr as u64); // Set event ring segment table registers println!(" - Interrupter 0: {:X}", self.run.ints.as_ptr() as usize); { - self.run.ints[0].iman.writef(1, true); // clear interrupt pending if set earlier by the BIOS - - println!("IP={}", self.run.ints[0].iman.readf(1)); - - /*if self.msi { - self.pcid_handle.enable_feature(PciFeature::Msi).expect("xhcid: failed to enable MSI"); - println!("Enabled MSI"); - }*/ - if self.msix { - self.pcid_handle.enable_feature(PciFeature::MsiX).expect("xhcid: failed to enable MSI-X"); - println!("Enabled MSI-X"); - } - - dbg!(self.pcid_handle.feature_info(PciFeature::MsiX).unwrap()); - dbg!(self.pcid_handle.feature_info(PciFeature::Msi).unwrap()); + let int = &mut self.run.get_mut().ints[0]; let erstz = 1; println!(" - Write ERSTZ: {}", erstz); - self.run.ints[0].erstsz.write(erstz); + int.erstsz.write(erstz); - let erdp = self.cmd.erdp(); + let erdp = self.primary_event_ring.get_mut().erdp(); println!(" - Write ERDP: {:X}", erdp); - self.run.ints[0].erdp.write(erdp as u64 | (1 << 3)); + int.erdp.write(erdp as u64 | (1 << 3)); - let erstba = self.cmd.erstba(); + let erstba = self.primary_event_ring.get_mut().erstba(); println!(" - Write ERSTBA: {:X}", erstba); - self.run.ints[0].erstba.write(erstba as u64); + int.erstba.write(erstba as u64); println!(" - Write IMODC and IMODI: {} and {}", 0, 0); - self.run.ints[0].imod.write(0); + int.imod.write(0); println!(" - Enable interrupts"); - self.run.ints[0].iman.writef(1 << 1, true); + int.iman.writef(1 << 1, true); } - self.op.usb_cmd.writef(1 << 2, true); + self.op.get_mut().usb_cmd.writef(1 << 2, true); + + // Setup the scratchpad buffers that are required for the xHC to function. + self.setup_scratchpads(); // Set run/stop to 1 println!(" - Start"); - self.op.usb_cmd.writef(1, true); + self.op.get_mut().usb_cmd.writef(1, true); // Wait until controller is running println!(" - Wait for running"); - while self.op.usb_sts.readf(1) { + while self.op.get_mut().usb_sts.readf(1) { println!(" - Waiting for XHCI running"); } - println!("IP={}", self.run.ints[0].iman.readf(1)); + println!("IP={}", self.run.get_mut().ints[0].iman.readf(1)); // Ring command doorbell println!(" - Ring doorbell"); - self.dbs[0].write(0); + self.dbs.get_mut()[0].write(0); println!(" - XHCI initialized"); } + pub fn setup_scratchpads(&mut self) -> Result<()> { + let buf_count = self.cap.max_scratchpad_bufs(); + + if buf_count == 0 { + return; + } + self.scratchpad_buf_arr = Some(ScratchpadBufferArray::new(buf_count)?); + self.dev_ctx.dcbaa[0] = self.scratchpad_buf_arr.register(); + } + pub fn enable_port_slot(&mut self, slot_ty: u8) -> Result { assert_eq!(slot_ty & 0x1F, slot_ty); let cloned_event_trb = - self.execute_command("ENABLE_SLOT", |cmd, cycle| cmd.enable_slot(0, cycle))?; + self.execute_command("ENABLE_SLOT", |cmd, cycle| cmd.enable_slot(slot_ty, cycle))?; Ok(cloned_event_trb.event_slot()) } pub fn disable_port_slot(&mut self, slot: u8) -> Result<()> { - self.execute_command("DISABLE_SLOT", |cmd, cycle| cmd.disable_slot(0, cycle))?; + self.execute_command("DISABLE_SLOT", |cmd, cycle| cmd.disable_slot(slot, cycle))?; Ok(()) } @@ -378,11 +393,12 @@ impl Xhci { self.dev_ctx.contexts[slot].slot.state() } - pub fn probe(&mut self) -> Result<()> { + pub async fn probe(&self) -> Result<()> { println!("XHCI capabilities: {:?}", self.capabilities_iter().collect::>()); - for i in 0..self.ports.len() { + + for i in 0..self.ports.lock().unwrap().len() { let (data, state, speed, flags) = { - let port = &self.ports[i]; + let port = &self.ports.lock().unwrap()[i]; (port.read(), port.state(), port.speed(), port.flags()) }; println!( @@ -404,13 +420,13 @@ impl Xhci { println!(" - Slot {}", slot); let mut input = Dma::::zeroed()?; - let mut ring = self.address_device(&mut input, i, slot_ty, slot, speed)?; + let mut ring = self.address_device(&mut input, i, slot_ty, slot, speed).await?; let dev_desc = Self::get_dev_desc_raw( - &mut self.ports, - &mut self.run, - &mut self.cmd, - &mut self.dbs, + &mut *self.ports.lock().unwrap(), + &mut *self.run.lock().unwrap(), + &mut *self.cmd.lock().unwrap(), + &mut *self.dbs.lock().unwrap(), i, slot, &mut ring, @@ -420,8 +436,10 @@ impl Xhci { let mut port_state = PortState { slot, - input_context: input, + input_context: Mutex::new(input), dev_desc, + cfg_idx: None, + if_idx: None, endpoint_states: std::iter::once(( 0, EndpointState { @@ -433,7 +451,7 @@ impl Xhci { }; if self.cap.cic() { - self.op.set_cie(true); + self.op.lock().unwrap().set_cie(true); } /*match self.spawn_drivers(i, &mut port_state) { @@ -474,8 +492,8 @@ impl Xhci { Ok(()) } - pub fn address_device( - &mut self, + pub async fn address_device( + &self, input_context: &mut Dma, i: usize, slot_ty: u8, @@ -563,23 +581,45 @@ impl Xhci { let input_context_physical = input_context.physical(); - self.execute_command("ADDRESS_DEVICE", |trb, cycle| { + let (event_trb, _) = self.execute_command(|trb, cycle| { trb.address_device(slot, input_context_physical, false, cycle) - }) - .expect("ADDRESS_DEVICE failed"); + }).await; + + if event_trb.completion_code() != TrbCompletionCode::Success as u8 { + println!("Failed to address device at slot {} (port {})", slot, i); + } + Ok(ring) } + pub fn uses_msi(&self) -> bool { + if let InterruptMethod::Msi = self.interrupt_method { true } else { false } + } + pub fn uses_msix(&self) -> bool { + if let InterruptMethod::MsiX(_) = self.interrupt_method { true } else { false } + } + pub fn msix_info(&self) -> Option<&MsixInfo> { + match self.interrupt_method { + InterruptMethod::MsiX(ref info) => Some(info), + _ => None, + } + } + pub fn msix_info_mut(&mut self) -> Option<&mut MsixInfo> { + match self.interrupt_method { + InterruptMethod::MsiX(ref mut info) => Some(info), + _ => None, + } + } /// Checks whether an IRQ has been received from *this* device, in case of an interrupt. Always /// true when using MSI/MSI-X. pub fn received_irq(&mut self) -> bool { - if self.msi || self.msix { + if self.uses_msi() || self.uses_msix() { // Since using MSI and MSI-X implies having no IRQ sharing whatsoever, the IP bit // doesn't have to be touched. println!("Successfully received MSI/MSI-X interrupt, IP={}, EHB={}", self.run.ints[0].iman.readf(1), self.run.ints[0].erdp.readf(3)); - println!("MSI-X PB={}", self.msix_info.as_mut().unwrap().pba(0)); - let entry = self.msix_info.as_mut().unwrap().table_entry_pointer(0); + println!("MSI-X PB={}", self.msix_info_mut().unwrap().pba(0)); + let entry = self.msix_info_mut().unwrap().table_entry_pointer(0); println!("MSI-X entry (addr_lo, addr_hi, msg_data, vec_ctl: {:#0x} {:#0x} {:#0x} {:#0x}", entry.addr_lo.read(), entry.addr_hi.read(), entry.msg_data.read(), entry.vec_ctl.read()); true } else if self.run.ints[0].iman.readf(1) { @@ -595,17 +635,7 @@ impl Xhci { } } - /// Handle an IRQ event. pub fn on_irq(&mut self) { - // Wake all futures awaiting the IRQ. - for waker in self.irq_state.wakers.lock().unwrap().drain(..) { - waker.wake(); - } - } - pub(crate) fn irq(&self) -> IrqFuture { - IrqFuture { - state: IrqFutureState::Pending(Arc::downgrade(&self.irq_state)), - } } fn spawn_drivers(&mut self, port: usize, ps: &mut PortState) -> Result<()> { // TODO: There should probably be a way to select alternate interfaces, and not just the @@ -784,44 +814,3 @@ lazy_static! { toml::from_slice::(TOML).expect("Failed to parse internally embedded config file") }; } - -pub(crate) struct IrqFuture { - state: IrqFutureState, -} - -struct IrqState { - triggered: AtomicBool, - // TODO: Perhaps a channel? - wakers: Mutex>, -} - -enum IrqFutureState { - Pending(Weak), - Finished, -} - -impl std::future::Future for IrqFuture { - type Output = (); - - fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { - let this = self.get_mut(); - - match &mut this.state { - // TODO: Ordering? - IrqFutureState::Pending(state_weak) => { - let state = state_weak.upgrade().expect( - "IRQ futures keep getting polled even after the driver has been deinitialized", - ); - - if state.triggered.load(atomic::Ordering::SeqCst) { - this.state = IrqFutureState::Finished; - task::Poll::Ready(()) - } else { - state.wakers.lock().unwrap().push(context.waker().clone()); - task::Poll::Pending - } - } - IrqFutureState::Finished => panic!("polling finished future"), - } - } -} diff --git a/xhcid/src/xhci/ring.rs b/xhcid/src/xhci/ring.rs index c941c70dca..26d0433277 100644 --- a/xhcid/src/xhci/ring.rs +++ b/xhcid/src/xhci/ring.rs @@ -1,3 +1,5 @@ +use std::mem; + use syscall::error::Result; use syscall::io::Dma; @@ -26,7 +28,7 @@ impl Ring { addr as u64 | self.cycle as u64 } - pub fn next(&mut self) -> (&mut Trb, bool) { + pub fn next_index(&mut self) -> usize { let mut i; loop { i = self.i; @@ -45,7 +47,11 @@ impl Ring { break; } } + i + } + pub fn next(&mut self) -> (&mut Trb, bool) { + let i = self.next_index(); (&mut self.trbs[i], self.cycle) } /// Endless iterator that iterates through the ring items, over and over again. The iterator @@ -53,6 +59,51 @@ impl Ring { pub fn iter(&self) -> impl Iterator + '_ { Iter { ring: self, i: self.i } } + /// Takes a physical address and returns the index into this ring, that the index represents. + /// Returns `None` if the address is outside the bounds of this ring. + /// + /// # Panics + /// Panics if paddr is not a multiple of 16 bytes, i.e. the size of a TRB. + pub fn phys_addr_to_index(&self, paddr: u64) -> Option { + let base = self.trbs.physical(); + let offset = paddr.checked_sub(base)?; + + assert_eq!(offset % mem::size_of::(), 0, "unaligned TRB physical address"); + + let index = offset / mem::size_of::(); + + if index > self.trbs.len() { + return None; + } + + Some(index) + } + pub fn phys_addr_to_entry_ref(&self, paddr: u64) -> Option<&Trb> { + &self.trbs[self.phys_addr_to_index(paddr)] + } + pub fn phys_addr_to_entry_mut(&self, paddr: u64) -> Option<&mut Trb> { + &mut self.trbs[self.phys_addr_to_index(paddr)] + } + pub fn phys_addr_to_entry(&self, paddr: u64) -> Option { + self.trbs[self.phys_addr_to_index(paddr)].clone() + } + pub(crate) fn start_virt_addr(&self) -> *const Trb { + self.trbs.as_ptr() + } + pub(crate) fn end_virt_addr(&self) -> *const Trb { + unsafe { self.start_virt_addr().offset(self.trbs.len()) } + } + pub fn trb_phys_ptr(&self, trb: &Trb) -> u64 { + let trb_virt_pointer = trb as *const Trb; + let trbs_base_virt_pointer = self.trbs.as_ptr(); + + if trb_virt_pointer < trbs_base_virt_pointer || trb_virt_pointer > trbs_base_virt_pointer + self.trbs.len() * mem::size_of::() { + panic!("Gave a TRB outside of the ring, when retrieving its physical address in that ring. TRB: {:?} (at address {:p})", trb, trb); + } + let trbs_base_phys_ptr = self.trbs.physical(); + let trb_phys_ptr = trb_virt_pointer - trbs_base_phys_ptr; + trb_phys_ptr + } /* /// Endless mutable iterator that iterates through the ring items, over and over again. The /// iterator doesn't enqueue or dequeue anything, but the trbs are mutably borrowed. diff --git a/xhcid/src/xhci/scheme.rs b/xhcid/src/xhci/scheme.rs index 1abfc8b056..feaca29bcf 100644 --- a/xhcid/src/xhci/scheme.rs +++ b/xhcid/src/xhci/scheme.rs @@ -17,7 +17,6 @@ use syscall::{ use super::{port, usb}; use super::{Device, EndpointState, Xhci}; -use super::command::CommandRing; use super::context::{ InputContext, SlotState, StreamContext, StreamContextArray, StreamContextType, ENDPOINT_CONTEXT_STATUS_MASK, @@ -211,40 +210,38 @@ impl AnyDescriptor { } impl Xhci { - pub fn execute_command( - &mut self, - cmd_name: &str, - f: F, - ) -> Result { - self.run.ints[0].erdp.write(self.cmd.erdp() | (1 << 3)); - let (cmd, cycle, event) = self.cmd.next(); - + pub fn execute_command_noreply(&self, f: F) { + let (cmd, cycle) = self.cmd.lock().unwrap(); f(cmd, cycle); - println!("INTE={}", self.op.usb_cmd.readf(1 << 2)); - println!("IP={}", self.run.ints[0].iman.readf(1)); - self.dbs[0].write(0); - println!("IP={}", self.run.ints[0].iman.readf(1)); + self.dbs.lock().unwrap()[0].write(0); - while event.data.read() == 0/* || self.run.ints[0].iman.readf(1)*/ { - println!(" - {} Waiting for event", cmd_name); - } - println!("IP={}", self.run.ints[0].iman.readf(1)); + // TODO: It's still possible not to reset the TRB, right? + } - if event.completion_code() != TrbCompletionCode::Success as u8 - || event.trb_type() != TrbType::CommandCompletion as u8 - { - println!("{} failed with event TRB ({:#0x} {:#0x} {:#0x}) and command TRB ({:#0x} {:#0x} {:#0x})", cmd_name, event.data.read(), event.status.read(), event.control.read(), cmd.data.read(), cmd.status.read(), cmd.control.read()); - return Err(Error::new(EIO)); - } + pub async fn execute_command( + &self, + f: F, + ) -> (Trb, Trb) { + let next_event = { + let command_ring = self.cmd.lock().unwrap(); - let ret = event.clone(); + let (cmd, cycle) = command_ring.next(); + f(cmd, cycle); - cmd.reserved(false); - event.reserved(false); + // get the future here before awaiting, to destroy the lock before deadlock + self.next_command_completion_event_trb(&cmd) + }; - self.run.ints[0].erdp.write(self.cmd.erdp() | (1 << 3)); - Ok(ret) + self.dbs.lock().unwrap()[0].write(0); + + let trbs = next_event.await; + let event_trb = trbs.event_trb; + let command_trb = trbs.src_trb.expect("Command completion event TRBs shall always have a valid pointer to a valid source command TRB"); + + assert_eq!(command_trb.trb_type(), TrbType::CommandCompletion as u8, "The IRQ reactor (or the xHC) gave an invalid event TRB"); + + (event_trb, command_trb) } pub fn execute_control_transfer( &mut self, @@ -418,8 +415,16 @@ impl Xhci { ) } - fn reset_endpoint(&mut self, port_num: usize, endp_num: u8, tsp: bool) -> Result<()> { - let endp_num_xhc = Self::endp_num_to_dci(endp_num, self.endp_desc(port_num, endp_num)?); + async fn reset_endpoint(&mut self, port_num: usize, endp_num: u8, tsp: bool) -> Result<()> { + let port_state = self.port_states.get(&port_num).ok_or(Error::new(EBADFD))?; + + let (cfg_idx, if_idx) = match (port_state.cfg_idx, port_state.if_idx) { + (Some(c), Some(i)) => (c, i), + _ => return Err(Error::new(EIO)), + }; + + let endp_desc = port_state.dev_desc.config_descs[usize::from(cfg_idx)].interface_descs[usize::from(if_idx)].endpoints.get(usize::from(endp_num)).ok_or(Error::new(EBADFD))?; + let endp_num_xhc = Self::endp_num_to_dci(endp_num, endp_desc); let slot = self .port_states @@ -427,10 +432,10 @@ impl Xhci { .ok_or(Error::new(EBADF))? .slot; - self.execute_command("RESET_ENDPOINT", |trb, cycle| { - trb.reset_endpoint(slot, endp_num_xhc, tsp, cycle) - })?; - Ok(()) + let (event_trb, command_trb) = self.execute_command(|trb, cycle| { + trb.reset_endpoint(slot, endp_num_xhc, tsp, cycle); + }).await; + handle_event_trb("RESET_ENDPOINT", &event_trb, &command_trb) } fn endp_ctx_interval(speed_id: &ProtocolSpeed, endp_desc: &EndpDesc) -> u8 { @@ -507,62 +512,14 @@ impl Xhci { } } - fn port_state(&self, port: usize) -> Result<&super::PortState> { + fn port_state(&self, port: usize) -> Result> { self.port_states.get(&port).ok_or(Error::new(EBADF)) } - fn port_state_mut(&mut self, port: usize) -> Result<&mut super::PortState> { - self.port_states.get_mut(&port).ok_or(Error::new(EBADF)) - } - fn endpoint_state_mut(&mut self, port: usize, endp_num: u8) -> Result<&mut EndpointState> { - self.port_state_mut(port)? - .endpoint_states - .get_mut(&endp_num) - .ok_or(Error::new(EBADF)) - } - fn input_context(&mut self, port: usize) -> Result<&mut Dma> { - Ok(&mut self.port_state_mut(port)?.input_context) - } - fn endp_ctx( - &mut self, - port: usize, - endp_num_xhc: u8, - ) -> Result<&mut super::context::EndpointContext> { - Ok(self - .input_context(port)? - .device - .endpoints - .get_mut(endp_num_xhc as usize - 1) - .ok_or(Error::new(EIO))?) - } - fn dev_desc(&self, port: usize) -> Result<&DevDesc> { - Ok(&self.port_state(port)?.dev_desc) - } - fn config_descs(&self, port: usize) -> Result<&[ConfDesc]> { - Ok(&self.dev_desc(port)?.config_descs) - } - fn config_desc(&self, port: usize, desc: u8) -> Result<&ConfDesc> { - Ok(self - .config_descs(port)? - .get(usize::from(desc)) - .ok_or(Error::new(EBADF))?) - } - fn endp_descs(&self, port: usize, config_desc: u8, if_desc: u8) -> Result<&[EndpDesc]> { - Ok(&self - .port_state(port)? - .dev_desc - .config_descs - .get(usize::from(config_desc)) - .ok_or(Error::new(EIO))? - .interface_descs - .get(usize::from(if_desc)) - .ok_or(Error::new(EIO))? - .endpoints) - } - fn configure_endpoints(&mut self, port: usize, json_buf: &[u8]) -> Result<()> { + async fn configure_endpoints(&mut self, port: usize, json_buf: &[u8]) -> Result<()> { let mut req: ConfigureEndpointsReq = serde_json::from_slice(json_buf).or(Err(Error::new(EBADMSG)))?; - if (!self.cap.cic() || !self.op.cie()) + if (!self.cap.cic() || !self.op.lock().unwrap().cie()) && (req.config_desc != 0 || req.interface_desc != None || req.alternate_setting != None) { //return Err(Error::new(EOPNOTSUPP)); @@ -574,9 +531,11 @@ impl Xhci { return Err(Error::new(EBADMSG)); } - let (endp_desc_count, new_context_entries) = { - let endpoints = - self.endp_descs(port, req.config_desc, req.interface_desc.unwrap_or(0))?; + let (endp_desc_count, new_context_entries, configuration_value) = { + let port_state = self.port_states.get(&port).ok_or(Error::new(EBADFD))?; + let config_desc = port_state.dev_desc.config_descs.get(usize::from(req.config_desc)).ok_or(Error::new(EBADFD))?; + + let endpoints = config_desc.interface_descs.get(usize::from(req.interface_desc.unwrap_or(0))).ok_or(Error::new(EBADFD))?.endpoints; if endpoints.len() >= 31 { return Err(Error::new(EIO)); @@ -588,18 +547,20 @@ impl Xhci { Some(l) => Self::endp_num_to_dci(endpoints.len() as u8, l), None => 1, }) + 1, + config_desc.configuration_value, ) }; let lec = self.cap.lec(); let log_max_psa_size = self.cap.max_psa_size(); - let port_speed_id = self.ports[port].speed(); + let port_speed_id = self.ports.lock().unwrap()[port].speed(); let speed_id: &ProtocolSpeed = self .lookup_psiv(port as u8, port_speed_id) .ok_or(Error::new(EIO))?; { - let input_context = self.input_context(port)?; + let port_state = self.port_states.get(&port).ok_or(Error::new(EBADFD))?; + let input_context = port_state.input_context.lock().unwrap(); // Configure the slot context as well, which holds the last index of the endp descs. input_context.add_context.write(1); @@ -625,9 +586,9 @@ impl Xhci { for endp_idx in 0..endp_desc_count as u8 { let endp_num = endp_idx + 1; - let endpoints = - self.endp_descs(port, req.config_desc, req.interface_desc.unwrap_or(0))?; - let dev_desc = self.dev_desc(port)?; + let port_state = self.port_states.get(&port).ok_or(Error::new(EBADFD))?; + let dev_desc = &port_state.dev_desc; + let endpoints = &dev_desc.config_descs.get(usize::from(req.config_desc)).ok_or(Error::new(EBADFD))?.interface_descs.get(usize::from(req.interface_desc.unwrap_or(0))).ok_or(Error::new(EBADFD))?.endpoints; let endp_desc = endpoints.get(endp_idx as usize).ok_or(Error::new(EIO))?; let endp_num_xhc = Self::endp_num_to_dci(endp_num, endp_desc); @@ -685,7 +646,7 @@ impl Xhci { assert_eq!(max_error_count & 0x3, max_error_count); assert_ne!(ep_ty, 0); // 0 means invalid. - let port_state = self.port_state_mut(port)?; + let port_state = self.port_states.get_mut(&port).ok_or(Error::new(EBADFD))?; let ring_ptr = if usb_log_max_streams.is_some() { let mut array = StreamContextArray::new(1 << (primary_streams + 1))?; @@ -728,10 +689,11 @@ impl Xhci { }; assert_eq!(primary_streams & 0x1F, primary_streams); - let input_context = self.input_context(port)?; + let port_state = self.port_states.get_mut(&port).ok_or(Error::new(EBADFD))?; + let input_context = port_state.input_context.lock().unwrap(); input_context.add_context.writef(1 << endp_num_xhc, true); - let endp_ctx = self.endp_ctx(port, endp_num_xhc)?; + let endp_ctx = input_context.device.endpoints.get_mut(endp_num_xhc as usize - 1).ok_or(Error::new(EIO))?; endp_ctx.a.write( u32::from(mult) << 8 @@ -756,14 +718,17 @@ impl Xhci { .write(u32::from(avg_trb_len) | (u32::from(max_esit_payload_lo) << 16)); } - let slot = self.port_state(port)?.slot; - let input_context_physical = self.input_context(port)?.physical(); - self.execute_command("CONFIGURE_ENDPOINT", |trb, cycle| { + let port_state = self.port_states.get(&port).ok_or(Error::new(EBADFD))?; + let slot = port_state.slot; + let input_context_physical = port_state.input_context.lock().unwrap().physical(); + + let (event_trb, command_trb) = self.execute_command(|trb, cycle| { trb.configure_endpoint(slot, input_context_physical, cycle) - })?; + }).await; + + handle_event_trb("CONFIGURE_ENDPOINT", &event_trb, &command_trb)?; // Tell the device about this configuration. - let configuration_value = self.config_desc(port, req.config_desc)?.configuration_value; self.set_configuration(port, configuration_value)?; if let (Some(interface_num), Some(alternate_setting)) = @@ -819,12 +784,6 @@ impl Xhci { unreachable!() } } - fn endp_desc(&self, port_num: usize, endp_num: u8) -> Result<&EndpDesc> { - Ok(self - .endp_descs(port_num, 0, 0)? - .get(usize::from(endp_num) - 1) - .ok_or(Error::new(EBADF))?) - } fn endp_doorbell(endp_num: u8, desc: &EndpDesc, stream_id: u16) -> u32 { let db_target = Self::endp_num_to_dci(endp_num, desc); let db_task_id: u16 = stream_id; @@ -969,11 +928,16 @@ impl Xhci { .port_states .get_mut(&port_id) .ok_or(Error::new(ENOENT))?; + let ports = self.ports.lock().unwrap(); + let run = self.run.lock().unwrap(); + let cmd = self.cmd.lock().unwrap(); + let dbs = self.dbs.lock().unwrap(); + Self::get_dev_desc_raw( - &mut self.ports, - &mut self.run, - &mut self.cmd, - &mut self.dbs, + &mut *ports, + &mut *run, + &mut *cmd, + &mut *dbs, port_id, st.slot, st.endpoint_states @@ -986,7 +950,7 @@ impl Xhci { pub(crate) fn get_dev_desc_raw( ports: &mut [port::Port], run: &mut RuntimeRegs, - cmd: &mut CommandRing, + cmd: &mut Ring, dbs: &mut [Doorbell], port_id: usize, slot: u8, @@ -999,7 +963,7 @@ impl Xhci { // TODO: Should the descriptors be stored in PortState? - run.ints[0].erdp.write(cmd.erdp() | (1 << 3)); + run.ints[0].erdp.write(cmd.register() | (1 << 3)); let mut dev = Device { ring, @@ -1148,11 +1112,6 @@ impl Xhci { setup: usb::Setup, transfer_kind: TransferKind, ) -> Result<()> { - // TODO: This json format might be too high level, but is useful for debugging, - // but when actual device-specific drivers are written, a binary format would - // be better. Maybe something simple like bincode could be used, if a custom binary struct - // is too much overkill. - self.execute_control_transfer( port_num, setup, @@ -1242,7 +1201,8 @@ impl Xhci { PortReqState::WaitingForDeviceBytes(_, _) => return Err(Error::new(EBADF)), PortReqState::Tmp | PortReqState::TmpSetup(_) => unreachable!(), }; - match self.handles.get_mut(&fd).ok_or(Error::new(EBADF))? { + let guard = self.handles.get_mut(&fd).ok_or(Error::new(EBADF))?; + match &mut *guard { Handle::PortReq(_, ref mut state) => *state = st, _ => unreachable!(), } @@ -1272,7 +1232,9 @@ impl Xhci { } PortReqState::Tmp | PortReqState::TmpSetup(_) => unreachable!(), }; - match self.handles.get_mut(&fd).ok_or(Error::new(EBADF))? { + + let guard = self.handles.get_mut(&fd).ok_or(Error::new(EBADF))?; + match &mut *guard { Handle::PortReq(_, ref mut state) => *state = st, _ => unreachable!(), } @@ -1306,8 +1268,9 @@ impl SchemeMut for Xhci { if flags & O_DIRECTORY != 0 || flags & O_STAT != 0 { let mut contents = Vec::new(); - for (index, _) in self - .ports + let ports_guard = self.ports.lock().unwrap(); + + for (index, _) in ports_guard .iter() .enumerate() .filter(|(_, port)| port.flags().contains(port::PortFlags::PORT_CCS)) @@ -1455,34 +1418,36 @@ impl SchemeMut for Xhci { } fn fstat(&mut self, id: usize, stat: &mut Stat) -> Result { - match self.handles.get(&id).ok_or(Error::new(EBADF))? { - Handle::TopLevel(_, ref buf) - | Handle::Port(_, _, ref buf) - | Handle::Endpoints(_, _, ref buf) => { + let guard = self.handles.get(&id).ok_or(Error::new(EBADF))?; + + match &*guard { + &Handle::TopLevel(_, ref buf) + | &Handle::Port(_, _, ref buf) + | &Handle::Endpoints(_, _, ref buf) => { stat.st_mode = MODE_DIR; stat.st_size = buf.len() as u64; } - Handle::PortDesc(_, _, ref buf) => { + &Handle::PortDesc(_, _, ref buf) => { stat.st_mode = MODE_FILE; stat.st_size = buf.len() as u64; } - Handle::PortReq(_, PortReqState::WaitingForDeviceBytes(ref buf, _)) - | Handle::PortReq(_, PortReqState::WaitingForHostBytes(ref buf, _)) => { + &Handle::PortReq(_, PortReqState::WaitingForDeviceBytes(ref buf, _)) + | &Handle::PortReq(_, PortReqState::WaitingForHostBytes(ref buf, _)) => { stat.st_mode = MODE_CHR; stat.st_size = buf.len() as u64; } - Handle::PortReq(_, PortReqState::Tmp) - | Handle::PortReq(_, PortReqState::TmpSetup(_)) => unreachable!(), + &Handle::PortReq(_, PortReqState::Tmp) + | &Handle::PortReq(_, PortReqState::TmpSetup(_)) => unreachable!(), - Handle::PortState(_, _) | Handle::PortReq(_, _) => stat.st_mode = MODE_CHR, - Handle::Endpoint(_, _, st) => match st { - EndpointHandleTy::Ctl | EndpointHandleTy::Data => stat.st_mode = MODE_CHR, - EndpointHandleTy::Root(_, ref buf) => { + &Handle::PortState(_, _) | &Handle::PortReq(_, _) => stat.st_mode = MODE_CHR, + &Handle::Endpoint(_, _, ref st) => match st { + &EndpointHandleTy::Ctl | &EndpointHandleTy::Data => stat.st_mode = MODE_CHR, + &EndpointHandleTy::Root(_, ref buf) => { stat.st_mode = MODE_DIR; stat.st_size = buf.len() as u64; } }, - Handle::ConfigureEndpoints(_) => { + &Handle::ConfigureEndpoints(_) => { stat.st_mode = MODE_CHR | 0o200; // write only } } @@ -1490,21 +1455,22 @@ impl SchemeMut for Xhci { } fn fpath(&mut self, fd: usize, buffer: &mut [u8]) -> Result { - // XXX: write!() should return the length instead of (). - let mut src = Vec::::new(); - match self.handles.get(&fd).ok_or(Error::new(EBADF))? { - Handle::TopLevel(_, _) => write!(src, "/").unwrap(), - Handle::Port(port_num, _, _) => write!(src, "/port{}/", port_num).unwrap(), - Handle::PortDesc(port_num, _, _) => { - write!(src, "/port{}/descriptors", port_num).unwrap() + let cursor = io::Cursor::new(buffer); + + let guard = self.handles.get(&fd).ok_or(Error::new(EBADF))?; + match &*guard { + &Handle::TopLevel(_, _) => write!(cursor, "/").unwrap(), + &Handle::Port(port_num, _, _) => write!(cursor, "/port{}/", port_num).unwrap(), + &Handle::PortDesc(port_num, _, _) => { + write!(cursor, "/port{}/descriptors", port_num).unwrap() } - Handle::PortState(port_num, _) => write!(src, "/port{}/state", port_num).unwrap(), - Handle::PortReq(port_num, _) => write!(src, "/port{}/request", port_num).unwrap(), - Handle::Endpoints(port_num, _, _) => { - write!(src, "/port{}/endpoints/", port_num).unwrap() + &Handle::PortState(port_num, _) => write!(cursor, "/port{}/state", port_num).unwrap(), + &Handle::PortReq(port_num, _) => write!(cursor, "/port{}/request", port_num).unwrap(), + &Handle::Endpoints(port_num, _, _) => { + write!(cursor, "/port{}/endpoints/", port_num).unwrap() } - Handle::Endpoint(port_num, endp_num, st) => write!( - src, + &Handle::Endpoint(port_num, endp_num, st) => write!( + cursor, "/port{}/endpoints/{}/{}", port_num, endp_num, @@ -1515,17 +1481,17 @@ impl SchemeMut for Xhci { } ) .unwrap(), - Handle::ConfigureEndpoints(port_num) => { - write!(src, "/port{}/configure", port_num).unwrap() + &Handle::ConfigureEndpoints(port_num) => { + write!(cursor, "/port{}/configure", port_num).unwrap() } } - let bytes_to_read = cmp::min(src.len(), buffer.len()); - buffer[..bytes_to_read].copy_from_slice(&src[..bytes_to_read]); - Ok(bytes_to_read) + let src_len = usize::try_from(cursor.seek(io::SeekFrom::End(0)).unwrap()).unwrap(); + Ok(src_len) } fn seek(&mut self, fd: usize, pos: usize, whence: usize) -> Result { - match self.handles.get_mut(&fd).ok_or(Error::new(EBADF))? { + let guard = self.handles.get_mut(&fd).ok_or(Error::new(EBADF))?; + match &mut *guard { // Directories, or fixed files Handle::TopLevel(ref mut offset, ref buf) | Handle::Port(_, ref mut offset, ref buf) @@ -1557,7 +1523,8 @@ impl SchemeMut for Xhci { } fn read(&mut self, fd: usize, buf: &mut [u8]) -> Result { - match self.handles.get_mut(&fd).ok_or(Error::new(EBADF))? { + let guard = self.handles.get_mut(&fd).ok_or(Error::new(EBADF))?; + match &mut *guard { Handle::TopLevel(ref mut offset, ref src_buf) | Handle::Port(_, ref mut offset, ref src_buf) | Handle::PortDesc(_, ref mut offset, ref src_buf) @@ -1609,7 +1576,8 @@ impl SchemeMut for Xhci { } } fn write(&mut self, fd: usize, buf: &[u8]) -> Result { - match self.handles.get_mut(&fd).ok_or(Error::new(EBADF))? { + let guard = self.handles.get_mut(&fd).ok_or(Error::new(EBADF))?; + match &mut *guard { &mut Handle::ConfigureEndpoints(port_num) => { self.configure_endpoints(port_num, buf)?; Ok(buf.len()) @@ -1667,7 +1635,7 @@ impl Xhci { _ => return Err(Error::new(EIO)), }) } - pub fn on_req_reset_device( + pub async fn on_req_reset_device( &mut self, port_num: usize, endp_num: u8, @@ -1678,8 +1646,8 @@ impl Xhci { } // Change the endpoint state from anything, but most likely HALTED (otherwise resetting // would be quite meaningless), to stopped. - self.reset_endpoint(port_num, endp_num, false)?; - self.restart_endpoint(port_num, endp_num)?; + self.reset_endpoint(port_num, endp_num, false).await?; + self.restart_endpoint(port_num, endp_num).await?; if clear_feature { self.device_req_no_data( @@ -1695,37 +1663,46 @@ impl Xhci { } Ok(()) } - pub fn restart_endpoint(&mut self, port_num: usize, endp_num: u8) -> Result<()> { + pub async fn restart_endpoint(&self, port_num: usize, endp_num: u8) -> Result<()> { let port_state = self .port_states .get_mut(&port_num) .ok_or(Error::new(EBADFD))?; + + let endp_desc = port_state + .dev_desc + .config_descs + .get(0) + .ok_or(Error::new(EIO))? + .interface_descs + .get(0) + .ok_or(Error::new(EIO))? + .endpoints + .get(endp_num as usize - 1) + .ok_or(Error::new(EBADFD))?; + let direction = if endp_num != 0 { - let endp_desc = port_state - .dev_desc - .config_descs - .get(0) - .ok_or(Error::new(EIO))? - .interface_descs - .get(0) - .ok_or(Error::new(EIO))? - .endpoints - .get(endp_num as usize - 1) - .ok_or(Error::new(EBADFD))?; endp_desc.direction() } else { EndpDirection::Bidirectional }; - let endpoint_state: &mut EndpointState = port_state + + let endpoint_state = port_state .endpoint_states .get_mut(&endp_num) .ok_or(Error::new(EBADFD))?; + let (has_streams, ring) = match &mut endpoint_state.transfer { &mut super::RingOrStreams::Ring(ref mut ring) => (false, ring), &mut super::RingOrStreams::Streams(ref mut arr) => { (true, arr.rings.get_mut(&1).ok_or(Error::new(EBADFD))?) } }; + let doorbell = Self::endp_doorbell( + endp_num, + endp_desc, + if has_streams { stream_id } else { 0 }, + ); let (cmd, cycle) = ring.next(); cmd.transfer_no_op(0, false, false, false, cycle); @@ -1733,14 +1710,12 @@ impl Xhci { let deque_ptr_and_cycle = ring.register(); let slot = port_state.slot; - self.set_tr_deque_ptr(port_num, endp_num, deque_ptr_and_cycle)?; + self.set_tr_deque_ptr(port_num, endp_num, deque_ptr_and_cycle).await?; + + let stream_id = 1u16; - self.dbs[slot as usize].write(Self::endp_doorbell( - endp_num, - self.endp_desc(port_num, endp_num)?, - if has_streams { stream_id } else { 0 }, - )); + self.dbs.lock().unwrap()[slot as usize].write(doorbell); Ok(()) } pub fn endp_direction(&self, port_num: usize, endp_num: u8) -> Result { @@ -1763,16 +1738,24 @@ impl Xhci { pub fn slot(&self, port_num: usize) -> Result { Ok(self.port_states.get(&port_num).ok_or(Error::new(EIO))?.slot) } - pub fn set_tr_deque_ptr( - &mut self, + pub async fn set_tr_deque_ptr( + &self, port_num: usize, endp_num: u8, deque_ptr_and_cycle: u64, ) -> Result<()> { - let slot = self.slot(port_num)?; - let endp_num_xhc = Self::endp_num_to_dci(endp_num, self.endp_desc(port_num, endp_num)?); + let port_state = self.port_states.get(&port_num).ok_or(Error::new(EBADFD))?; + let slot = port_state.slot; - self.execute_command("SET_TR_DEQUEUE_POINTER", |trb, cycle| { + let (cfg_idx, if_idx) = match (port_state.cfg_idx, port_state.if_idx) { + (Some(c), Some(i)) => (c, i), + _ => return Err(Error::new(EIO)), + }; + + let endp_desc = port_state.dev_desc.config_descs[usize::from(cfg_idx)].interface_descs[usize::from(if_idx)].endpoints.get(usize::from(endp_num)).ok_or(Error::new(EBADFD))?; + let endp_num_xhc = Self::endp_num_to_dci(endp_num, endp_desc); + + let (event_trb, command_trb) = self.execute_command(|trb, cycle| { trb.set_tr_deque_ptr( deque_ptr_and_cycle, cycle, @@ -1781,11 +1764,11 @@ impl Xhci { endp_num_xhc, slot, ) - })?; + }).await; - Ok(()) + handle_event_trb("SET_TR_DEQUEUE_PTR", &event_trb, &command_trb) } - pub fn on_write_endp_ctl( + pub async fn on_write_endp_ctl( &mut self, port_num: usize, endp_num: u8, @@ -1808,9 +1791,7 @@ impl Xhci { } }, XhciEndpCtlReq::Reset { no_clear_feature } => match ep_if_state { - EndpIfState::Init => { - self.on_req_reset_device(port_num, endp_num, !no_clear_feature)? - } + EndpIfState::Init => self.on_req_reset_device(port_num, endp_num, !no_clear_feature).await?, other => { return Err(Error::new(EBADF)); } @@ -1874,7 +1855,11 @@ impl Xhci { endp_num: u8, buf: &[u8], ) -> Result { - let ep_if_state = &mut self.endpoint_state_mut(port_num, endp_num)?.driver_if_state; + let port_state = self.port_states.get(&port_num).ok_or(Error::new(EBADFD))?; + let endpoint_state = port_state.endpoint_states.get_mut(&endp_num).ok_or(Error::new(EBADFD))?; + + let ep_if_state = &mut endpoint_state.driver_if_state; + match ep_if_state { &mut EndpIfState::WaitingForDataPipe { direction: XhciEndpCtlDirection::Out, @@ -1892,7 +1877,9 @@ impl Xhci { // invoking further data transfer calls if any single transfer returns fewer bytes // than requested. - let ep_if_state = &mut self.endpoint_state_mut(port_num, endp_num)?.driver_if_state; + let port_state = self.port_states.get(&port_num).ok_or(Error::new(EBADFD))?; + let endpoint_state = port_state.endpoint_states.get_mut(&endp_num).ok_or(Error::new(EBADFD))?; + let ep_if_state = &mut endpoint_state.driver_if_state; if let &mut EndpIfState::WaitingForDataPipe { direction: XhciEndpCtlDirection::Out, @@ -2005,6 +1992,18 @@ impl Xhci { _ => return Err(Error::new(EBADF)), } } + pub fn event_handler_finished(&self) { + // write 1 to EHB to clear it + self.run.lock().unwrap().ints[0].erdp.writef(1 << 3, true); + } +} +fn handle_event_trb(name: &str, event_trb: &Trb, command_trb: &Trb) -> Result<()> { + if event_trb.completion_code() == TrbCompletionCode::Success as u8 { + Ok(()) + } else { + println!("{} command (TRB {:?}) failed with event trb {:?}", name, command_trb, event_trb); + Err(Error::new(EIO)) + } } use std::ops::{Add, Div, Rem}; pub fn div_round_up(a: T, b: T) -> T diff --git a/xhcid/src/xhci/trb.rs b/xhcid/src/xhci/trb.rs index 4ad02bee37..c23f543a38 100644 --- a/xhcid/src/xhci/trb.rs +++ b/xhcid/src/xhci/trb.rs @@ -5,6 +5,7 @@ use syscall::io::{Io, Mmio}; use super::context::StreamContextType; #[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum TrbType { Reserved, /* Transfer */ @@ -33,8 +34,8 @@ pub enum TrbType { ForceHeader, NoOpCmd, /* Reserved */ - Rsv24, - Rsv25, + GetExtendedProperty, + SetExtendedProperty, Rsv26, Rsv27, Rsv28, @@ -54,6 +55,7 @@ pub enum TrbType { } #[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum TrbCompletionCode { Invalid, Success, @@ -157,6 +159,34 @@ impl Trb { pub fn completion_param(&self) -> u32 { self.status.read() & TRB_STATUS_COMPLETION_PARAM_MASK } + fn has_completion_trb_pointer(&self) -> bool { + if self.completion_code() == TrbCompletionCode::RingUnderrun as u8 || self.completion_code() == TrbCompletionCode::RingOverrun as u8 { + false + } else if self.completion_code() == TrbCompletionCode::VfEventRingFull as u8 { + false + } else { + true + } + } + pub fn completion_trb_pointer(&self) -> Option { + debug_assert_eq!(self.trb_type(), TrbType::CommandCompletion as u8); + + if self.has_completion_trb_pointer() { + Some(self.data.read()) + } else { + None + } + } + pub fn transfer_event_trb_pointer(&self) -> Option { + debug_assert_eq!(self.trb_type(), TrbType::Transfer as u8); + + if self.has_completion_trb_pointer() { + Some(self.data.read()) + } else { + None + } + } + pub fn event_slot(&self) -> u8 { (self.control.read() >> 24) as u8 } @@ -383,6 +413,43 @@ impl Trb { | ((TrbType::Normal as u32) << 10), ) } + pub fn is_command_trb(&self) -> bool { + let valid_trb_types = [ + TrbType::NoOpCmd as u8, + TrbType::EnableSlot as u8, + TrbType::DisableSlot as u8, + TrbType::AddressDevice as u8, + TrbType::ConfigureEndpoint as u8, + TrbType::EvaluateContext as u8, + TrbType::ResetEndpoint as u8, + TrbType::StopEndpoint as u8, + TrbType::SetTrDequeuePointer as u8, + TrbType::ResetDevice as u8, + TrbType::ForceEvent as u8, + TrbType::NegotiateBandwidth as u8, + TrbType::SetLatencyToleranceValue as u8, + TrbType::GetPortBandwidth as u8, + TrbType::ForceHeader as u8, + TrbType::GetExtendedProperty as u8, + TrbType::SetExtendedProperty as u8, + ]; + valid_trb_types.contains(&self.trb_type()) + } + pub fn is_transfer_trb(&self) -> bool { + // XXX: Unfortunately, the only way to use match statements with integer constants, is to + // precast them into valid enum values, which either requires a derive macro such as + // num_traits's #[derive(FromPrimitive)], or manually writing the reverse match statement + // first. + let valid_trb_types = [ + TrbType::Normal as u8, + TrbType::SetupStage as u8, + TrbType::DataStage as u8, + TrbType::StatusStage as u8, + TrbType::Isoch as u8, + TrbType::NoOp as u8, + ]; + valid_trb_types.contains(&self.trb_type()) + } } impl fmt::Debug for Trb {