diff --git a/Cargo.lock b/Cargo.lock index 9398d25a36..e68857c1cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -42,13 +42,18 @@ dependencies = [ "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "atty" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -133,7 +138,7 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -144,10 +149,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "chashmap" version = "2.2.2" -source = "git+https://gitlab.redox-os.org/redox-os/chashmap.git#da92c702e052cde00db5e409dfb234af71928152" +source = "git+https://gitlab.redox-os.org/redox-os/chashmap.git#9a36a4df91930628390d70b697800e32b9e7c9bd" dependencies = [ "owning_ref 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -157,7 +162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -355,7 +360,7 @@ dependencies = [ "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -374,10 +379,10 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -395,8 +400,8 @@ dependencies = [ "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -439,7 +444,7 @@ name = "iovec" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -478,7 +483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -497,7 +502,7 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "owning_ref 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -560,7 +565,7 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -579,7 +584,7 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -593,7 +598,7 @@ version = "0.6.7" source = "git+https://gitlab.redox-os.org/redox-os/mio-uds#22580ca398cdb5ed6f50fb61134e5579e2213999" dependencies = [ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)", ] @@ -614,7 +619,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -627,7 +632,7 @@ dependencies = [ "extra 0.1.0 (git+https://gitlab.redox-os.org/redox-os/libextra.git)", "hyper 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)", "ntpclient 0.0.1 (git+https://github.com/willem66745/ntpclient-rust)", "pbr 1.0.2 (git+https://github.com/a8m/pb)", @@ -647,7 +652,7 @@ source = "git+https://gitlab.redox-os.org/redox-os/netutils.git?branch=redox-uni dependencies = [ "arg_parser 0.1.0 (git+https://gitlab.redox-os.org/redox-os/arg-parser.git)", "extra 0.1.0 (git+https://gitlab.redox-os.org/redox-os/libextra.git)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "ntpclient 0.0.1 (git+https://github.com/willem66745/ntpclient-rust)", "pbr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -669,7 +674,7 @@ version = "0.0.1" source = "git+https://github.com/willem66745/ntpclient-rust#7e3bdf60eb940825789a8da5181025320e3050b0" dependencies = [ "byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -711,11 +716,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -727,11 +732,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "nvmed" version = "0.1.0" dependencies = [ + "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "block-io-wrapper 0.1.0", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "partitionlib 0.1.0 (git+https://gitlab.redox-os.org/redox-os/partitionlib.git)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "spin 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", + "pcid 0.1.0", + "redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git?tag=v0.1.0)", + "redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -762,12 +773,11 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -775,7 +785,7 @@ name = "parking_lot_core" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", @@ -784,15 +794,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.6.2" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -812,9 +821,9 @@ version = "1.0.2" source = "git+https://github.com/a8m/pb#87c29c05486afa7335916c870ea3621ff7ef2966" dependencies = [ "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -823,9 +832,9 @@ name = "pbr" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.5 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -836,13 +845,15 @@ dependencies = [ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git?tag=v0.1.0)", "redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -860,7 +871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "pin-utils" -version = "0.1.0-alpha.4" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -909,7 +920,7 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -964,7 +975,7 @@ name = "rand_jitter" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -976,7 +987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1019,10 +1030,11 @@ dependencies = [ [[package]] name = "redox-log" version = "0.1.0" -source = "git+https://gitlab.redox-os.org/redox-os/redox-log.git#08693d48b2d7b56fcb07a1e62e257bacce749cef" +source = "git+https://gitlab.redox-os.org/redox-os/redox-log.git?tag=v0.1.0#30f6bf2464c462c32cd215bc0f1eedafa0707a04" dependencies = [ "chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "termion 1.5.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1060,9 +1072,9 @@ name = "ring" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1161,7 +1173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "sdl2-sys 0.32.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1173,7 +1185,7 @@ version = "0.32.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1232,7 +1244,7 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1284,7 +1296,7 @@ name = "termion" version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1300,15 +1312,15 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror-impl 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "thiserror-impl" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1318,11 +1330,10 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1334,7 +1345,7 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-codec 0.1.1 (git+https://gitlab.redox-os.org/redox-os/tokio)", "tokio-current-thread 0.1.4 (git+https://gitlab.redox-os.org/redox-os/tokio)", "tokio-executor 0.1.5 (git+https://gitlab.redox-os.org/redox-os/tokio)", @@ -1405,7 +1416,7 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.5 (git+https://gitlab.redox-os.org/redox-os/tokio)", @@ -1434,7 +1445,7 @@ dependencies = [ "crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-executor 0.1.5 (git+https://gitlab.redox-os.org/redox-os/tokio)", ] @@ -1472,7 +1483,7 @@ dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.16 (git+https://gitlab.redox-os.org/redox-os/mio)", "mio-uds 0.6.7 (git+https://gitlab.redox-os.org/redox-os/mio-uds)", @@ -1520,7 +1531,7 @@ name = "unicode-normalization" version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1573,7 +1584,7 @@ dependencies = [ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)", - "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "xhcid 0.1.0", ] @@ -1700,13 +1711,13 @@ dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "pcid 0.1.0", "plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git)", + "redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git?tag=v0.1.0)", "redox_event 0.1.0 (git+https://gitlab.redox-os.org/redox-os/event.git)", "redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "thiserror 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1714,6 +1725,7 @@ dependencies = [ "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum arg_parser 0.1.0 (git+https://gitlab.redox-os.org/redox-os/arg-parser.git)" = "" "checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" @@ -1726,7 +1738,7 @@ dependencies = [ "checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855" "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cc 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum chashmap 2.2.2 (git+https://gitlab.redox-os.org/redox-os/chashmap.git)" = "" "checksum chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" @@ -1753,7 +1765,7 @@ dependencies = [ "checksum futures-task 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" "checksum futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22766cf25d64306bedf0384da004d05c9974ab104fcc4528f1236181c18004c5" "checksum gpt 0.6.3 (git+https://gitlab.redox-os.org/redox-os/gpt)" = "" -"checksum hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +"checksum hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" "checksum hyper 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)" = "0a0652d9a2609a968c14be1a9ea00bf4b1d64e2e1f53a1b51b6fff3a6e829273" "checksum hyper-rustls 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "71f7b2e5858ab9e19771dc361159f09ee5031734a6f7471fe0947db0238d92b7" @@ -1763,10 +1775,10 @@ dependencies = [ "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)" = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" +"checksum libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)" = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" -"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +"checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" @@ -1787,19 +1799,19 @@ dependencies = [ "checksum num-integer 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" "checksum num-iter 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "dfb0800a0291891dd9f4fe7bd9c19384f98f7fbe0cd0f39a2c6b88b9868bbc00" "checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" -"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" "checksum orbclient 0.3.27 (registry+https://github.com/rust-lang/crates.io-index)" = "f8b18f57ab94fbd058e30aa57f712ec423c0bb7403f8493a6c58eef0c36d9402" "checksum owning_ref 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" +"checksum parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" "checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" -"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" "checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" -"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +"checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" "checksum partitionlib 0.1.0 (git+https://gitlab.redox-os.org/redox-os/partitionlib.git)" = "" "checksum pbr 1.0.2 (git+https://github.com/a8m/pb)" = "" "checksum pbr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4403eb718d70c03ee279e51737782902c68cca01e870a33b6a2f9dfb50b9cd83" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" -"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" +"checksum pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" "checksum plain 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" "checksum proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)" = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" "checksum proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" @@ -1817,7 +1829,7 @@ dependencies = [ "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" "checksum ransid 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "22b9af151b0590163dfa64e1c92c0831377d61942df4c19820c704390ebc3045" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -"checksum redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git)" = "" +"checksum redox-log 0.1.0 (git+https://gitlab.redox-os.org/redox-os/redox-log.git?tag=v0.1.0)" = "" "checksum redox_event 0.1.0 (git+https://gitlab.redox-os.org/redox-os/event.git)" = "" "checksum redox_syscall 0.1.56 (git+https://gitlab.redox-os.org/redox-os/syscall.git)" = "" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" @@ -1842,7 +1854,7 @@ dependencies = [ "checksum serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)" = "da07b57ee2623368351e9a0488bb0b261322a15a6e0ae53e243cbdc0f4208da9" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" -"checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +"checksum smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" "checksum spin 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ceac490aa12c567115b40b7b7fceca03a6c9d53d5defea066123debc83c5dc1f" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum stb_truetype 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "9d1bec4382294c5a680fcebd29f8451e8d8c04479a026f6909004e2ab1cb425d" @@ -1851,9 +1863,9 @@ dependencies = [ "checksum syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" "checksum termion 1.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c22cec9d8978d906be5ac94bceb5a010d885c626c4c8855721a4dbd20e3ac905" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" -"checksum thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f0570dc61221295909abdb95c739f2e74325e14293b2026b0a7e195091ec54ae" -"checksum thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "227362df41d566be41a28f64401e07a043157c21c14b9785a0d8e256f940a8fd" -"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" +"checksum thiserror 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "54b3d3d2ff68104100ab257bb6bb0cb26c901abe4bd4ba15961f3bf867924012" +"checksum thiserror-impl 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "ca972988113b7715266f91250ddb98070d033c62a011fa0fcc57434a649310dd" +"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" "checksum tokio 0.1.13 (git+https://gitlab.redox-os.org/redox-os/tokio)" = "" "checksum tokio-codec 0.1.1 (git+https://gitlab.redox-os.org/redox-os/tokio)" = "" "checksum tokio-current-thread 0.1.4 (git+https://gitlab.redox-os.org/redox-os/tokio)" = "" diff --git a/initfs.toml b/initfs.toml index 6861491750..9d95b91f9f 100644 --- a/initfs.toml +++ b/initfs.toml @@ -28,6 +28,7 @@ name = "NVME storage" class = 1 subclass = 8 command = ["nvmed", "$NAME", "$BAR0", "$BARSIZE0", "$IRQ"] +use_channel = true # vboxd [[drivers]] @@ -43,5 +44,5 @@ name = "XHCI" class = 12 subclass = 3 interface = 48 -command = ["xhcid", "$NAME", "$BAR0", "$IRQ"] -channel_name = "pcid-xhcid" +command = ["xhcid"] +use_channel = true diff --git a/nvmed/Cargo.toml b/nvmed/Cargo.toml index b0bbfee4b2..dca9a5dee1 100644 --- a/nvmed/Cargo.toml +++ b/nvmed/Cargo.toml @@ -4,8 +4,15 @@ version = "0.1.0" edition = "2018" [dependencies] +arrayvec = "0.5" bitflags = "0.7" -spin = "0.4" -redox_syscall = "0.1" +crossbeam-channel = "0.4" +futures = "0.3" +log = "0.4" +redox-log = { git = "https://gitlab.redox-os.org/redox-os/redox-log.git", tag = "v0.1.0" } +redox_syscall = { git = "https://gitlab.redox-os.org/redox-os/syscall.git" } partitionlib = { git = "https://gitlab.redox-os.org/redox-os/partitionlib.git" } +smallvec = "1" block-io-wrapper = { path = "../block-io-wrapper" } + +pcid = { path = "../pcid" } diff --git a/nvmed/src/main.rs b/nvmed/src/main.rs index fb37c0f6b9..dc5585c37d 100644 --- a/nvmed/src/main.rs +++ b/nvmed/src/main.rs @@ -1,125 +1,382 @@ -#![feature(asm)] - -extern crate bitflags; -extern crate spin; -extern crate syscall; - -use std::{env, usize}; +use std::convert::TryInto; use std::fs::File; use std::io::{ErrorKind, Read, Write}; -use std::os::unix::io::{RawFd, FromRawFd}; +use std::os::unix::io::{FromRawFd, RawFd}; +use std::ptr::NonNull; +use std::sync::{Arc, Mutex}; +use std::{slice, usize}; -use syscall::{EVENT_READ, PHYSMAP_NO_CACHE, PHYSMAP_WRITE, Event, Packet, Result, SchemeBlockMut}; +use pcid_interface::{PciBar, PciFeature, PciFeatureInfo, PciFunction, PcidServerHandle}; +use syscall::{ + CloneFlags, Event, Mmio, Packet, Result, SchemeBlockMut, PHYSMAP_NO_CACHE, + PHYSMAP_WRITE, +}; +use redox_log::{OutputBuilder, RedoxLogger}; -use self::nvme::Nvme; +use self::nvme::{InterruptMethod, InterruptSources, Nvme}; use self::scheme::DiskScheme; mod nvme; mod scheme; -fn main() { - let mut args = env::args().skip(1); - - let mut name = args.next().expect("nvmed: no name provided"); - name.push_str("_nvme"); - - let bar_str = args.next().expect("nvmed: no address provided"); - let bar = usize::from_str_radix(&bar_str, 16).expect("nvmed: failed to parse address"); - - let bar_size_str = args.next().expect("nvmed: no address size provided"); - let bar_size = usize::from_str_radix(&bar_size_str, 16).expect("nvmed: failed to parse address size"); - - let irq_str = args.next().expect("nvmed: no irq provided"); - let irq = irq_str.parse::().expect("nvmed: failed to parse irq"); - - print!("{}", format!(" + NVME {} on: {:X} size: {} IRQ: {}\n", name, bar, bar_size, irq)); - - // Daemonize - if unsafe { syscall::clone(0).unwrap() } == 0 { - let address = unsafe { - syscall::physmap(bar, bar_size, PHYSMAP_WRITE | PHYSMAP_NO_CACHE) - .expect("nvmed: failed to map address") - }; - { - let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC) - .expect("nvmed: failed to open event queue"); - let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) }; - - let irq_fd = syscall::open( - &format!("irq:{}", irq), - syscall::O_RDWR | syscall::O_NONBLOCK | syscall::O_CLOEXEC - ).expect("nvmed: failed to open irq file"); - syscall::write(event_fd, &syscall::Event { - id: irq_fd, - flags: syscall::EVENT_READ, - data: 0, - }).expect("nvmed: failed to watch irq file events"); - let mut irq_file = unsafe { File::from_raw_fd(irq_fd as RawFd) }; - - let scheme_name = format!("disk/{}", name); - let socket_fd = syscall::open( - &format!(":{}", scheme_name), - syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC - ).expect("nvmed: failed to create disk scheme"); - syscall::write(event_fd, &syscall::Event { - id: socket_fd, - flags: syscall::EVENT_READ, - data: 1, - }).expect("nvmed: failed to watch disk scheme events"); - let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) }; - - syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace"); - - let mut nvme = Nvme::new(address).expect("nvmed: failed to allocate driver data"); - let namespaces = unsafe { nvme.init() }; - let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); - let mut todo = Vec::new(); - 'events: loop { - let mut event = Event::default(); - if event_file.read(&mut event).expect("nvmed: failed to read event queue") == 0 { - break; - } - - match event.data { - 0 => { - let mut irq = [0; 8]; - if irq_file.read(&mut irq).expect("nvmed: failed to read irq file") >= irq.len() { - if scheme.irq() { - irq_file.write(&irq).expect("nvmed: failed to write irq file"); - } - } - }, - 1 => loop { - let mut packet = Packet::default(); - match socket_file.read(&mut packet) { - Ok(0) => break 'events, - Ok(_) => (), - Err(err) => match err.kind() { - ErrorKind::WouldBlock => break, - _ => Err(err).expect("nvmed: failed to read disk scheme"), - } - } - todo.push(packet); - }, - unknown => { - panic!("nvmed: unknown event data {}", unknown); - }, - } - - let mut i = 0; - while i < todo.len() { - if let Some(a) = scheme.handle(&todo[i]) { - let mut packet = todo.remove(i); - packet.a = a; - socket_file.write(&packet).expect("nvmed: failed to write disk scheme"); - } else { - i += 1; - } - } - } - - //TODO: destroy NVMe stuff - } - unsafe { let _ = syscall::physunmap(address); } +/// A wrapper for a BAR allocation. +pub struct Bar { + ptr: NonNull, + physical: usize, + bar_size: usize, +} +impl Bar { + pub fn allocate(bar: usize, bar_size: usize) -> Result { + Ok(Self { + ptr: NonNull::new( + unsafe { syscall::physmap(bar, bar_size, PHYSMAP_NO_CACHE | PHYSMAP_WRITE)? as *mut u8 }, + ) + .expect("Mapping a BAR resulted in a nullptr"), + physical: bar, + bar_size, + }) } } + +impl Drop for Bar { + fn drop(&mut self) { + let _ = unsafe { syscall::physunmap(self.physical) }; + } +} + +/// The PCI BARs that may be allocated. +#[derive(Default)] +pub struct AllocatedBars(pub [Mutex>; 6]); + +/// Get the most optimal yet functional interrupt mechanism: either (in the order of preference): +/// MSI-X, MSI, and INTx# pin. Returns both runtime interrupt structures (MSI/MSI-X capability +/// structures), and the handles to the interrupts. +fn get_int_method( + pcid_handle: &mut PcidServerHandle, + function: &PciFunction, + allocated_bars: &AllocatedBars, +) -> Result<(InterruptMethod, InterruptSources)> { + log::trace!("Begin get_int_method"); + use pcid_interface::irq_helpers; + + let features = pcid_handle.fetch_all_features().unwrap(); + + let has_msi = features.iter().any(|(feature, _)| feature.is_msi()); + let has_msix = features.iter().any(|(feature, _)| feature.is_msix()); + + // TODO: Allocate more than one vector when possible and useful. + if has_msix { + // Extended message signaled interrupts. + use self::nvme::MsixCfg; + use pcid_interface::msi::MsixTableEntry; + + let mut capability_struct = match pcid_handle.feature_info(PciFeature::MsiX).unwrap() { + PciFeatureInfo::MsiX(msix) => msix, + _ => unreachable!(), + }; + fn bar_base( + allocated_bars: &AllocatedBars, + function: &PciFunction, + bir: u8, + ) -> Result> { + let bir = usize::from(bir); + let mut bar_guard = allocated_bars.0[bir].lock().unwrap(); + match &mut *bar_guard { + &mut Some(ref bar) => Ok(bar.ptr), + bar_to_set @ &mut None => { + let bar = match function.bars[bir] { + PciBar::Memory(addr) => addr, + other => panic!("Expected memory BAR, found {:?}", other), + }; + let bar_size = function.bar_sizes[bir]; + + let bar = Bar::allocate(bar as usize, bar_size as usize)?; + *bar_to_set = Some(bar); + Ok(bar_to_set.as_ref().unwrap().ptr) + } + } + } + let table_bar_base: *mut u8 = + bar_base(allocated_bars, function, capability_struct.table_bir())?.as_ptr(); + let pba_bar_base: *mut u8 = + bar_base(allocated_bars, function, capability_struct.pba_bir())?.as_ptr(); + let table_base = + unsafe { table_bar_base.offset(capability_struct.table_offset() as isize) }; + let pba_base = unsafe { pba_bar_base.offset(capability_struct.pba_offset() as isize) }; + + let vector_count = capability_struct.table_size(); + let table_entries: &'static mut [MsixTableEntry] = unsafe { + slice::from_raw_parts_mut(table_base as *mut MsixTableEntry, vector_count as usize) + }; + let pba_entries: &'static mut [Mmio] = unsafe { + slice::from_raw_parts_mut( + table_base as *mut Mmio, + (vector_count as usize + 63) / 64, + ) + }; + + // Mask all interrupts in case some earlier driver/os already unmasked them (according to + // the PCI Local Bus spec 3.0, they are masked after system reset). + for table_entry in table_entries.iter_mut() { + table_entry.mask(); + } + + pcid_handle.enable_feature(PciFeature::MsiX).unwrap(); + capability_struct.set_msix_enabled(true); // only affects our local mirror of the cap + + let (msix_vector_number, irq_handle) = { + use msi_x86_64::DeliveryMode; + use pcid_interface::msi::x86_64 as msi_x86_64; + + let entry: &mut MsixTableEntry = &mut table_entries[0]; + + let bsp_cpu_id = + irq_helpers::read_bsp_apic_id().expect("nvmed: failed to read APIC ID"); + let bsp_lapic_id = bsp_cpu_id + .try_into() + .expect("nvmed: BSP local apic ID couldn't fit inside u8"); + let (vector, irq_handle) = irq_helpers::allocate_single_interrupt_vector(bsp_cpu_id) + .expect("nvmed: failed to allocate single MSI-X interrupt vector") + .expect("nvmed: no interrupt vectors left on BSP"); + + let msg_addr = msi_x86_64::message_address(bsp_lapic_id, false, false); + let msg_data = msi_x86_64::message_data_edge_triggered(DeliveryMode::Fixed, vector); + + entry.set_addr_lo(msg_addr); + entry.set_msg_data(msg_data); + + (0, irq_handle) + }; + + let interrupt_method = InterruptMethod::MsiX(MsixCfg { + cap: capability_struct, + table: table_entries, + pba: pba_entries, + }); + let interrupt_sources = + InterruptSources::MsiX(std::iter::once((msix_vector_number, irq_handle)).collect()); + + Ok((interrupt_method, interrupt_sources)) + } else if has_msi { + // Message signaled interrupts. + let capability_struct = match pcid_handle.feature_info(PciFeature::Msi).unwrap() { + PciFeatureInfo::Msi(msi) => msi, + _ => unreachable!(), + }; + + let (msi_vector_number, irq_handle) = { + use msi_x86_64::DeliveryMode; + use pcid_interface::msi::x86_64 as msi_x86_64; + use pcid_interface::{MsiSetFeatureInfo, SetFeatureInfo}; + + let bsp_cpu_id = + irq_helpers::read_bsp_apic_id().expect("nvmed: failed to read BSP APIC ID"); + let bsp_lapic_id = bsp_cpu_id + .try_into() + .expect("nvmed: BSP local apic ID couldn't fit inside u8"); + let (vector, irq_handle) = irq_helpers::allocate_single_interrupt_vector(bsp_cpu_id) + .expect("nvmed: failed to allocate single MSI interrupt vector") + .expect("nvmed: no interrupt vectors left on BSP"); + + let msg_addr = msi_x86_64::message_address(bsp_lapic_id, false, false); + let msg_data = + msi_x86_64::message_data_edge_triggered(DeliveryMode::Fixed, vector) as u16; + + pcid_handle.set_feature_info(SetFeatureInfo::Msi(MsiSetFeatureInfo { + message_address: Some(msg_addr), + message_upper_address: Some(0), + message_data: Some(msg_data), + multi_message_enable: Some(0), // enable 2^0=1 vectors + mask_bits: None, + })).unwrap(); + + (0, irq_handle) + }; + + let interrupt_method = InterruptMethod::Msi(capability_struct); + let interrupt_sources = + InterruptSources::Msi(std::iter::once((msi_vector_number, irq_handle)).collect()); + + pcid_handle.enable_feature(PciFeature::Msi).unwrap(); + + Ok((interrupt_method, interrupt_sources)) + } else if function.legacy_interrupt_pin.is_some() { + // INTx# pin based interrupts. + let irq_handle = File::open(format!("irq:{}", function.legacy_interrupt_line)) + .expect("nvmed: failed to open INTx# interrupt line"); + Ok((InterruptMethod::Intx, InterruptSources::Intx(irq_handle))) + } else { + // No interrupts at all + todo!("handling of no interrupts") + } +} + +fn setup_logging() -> Option<&'static RedoxLogger> { + let mut logger = RedoxLogger::new() + .with_output( + OutputBuilder::stderr() + .with_filter(log::LevelFilter::Info) // limit global output to important info + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ); + + #[cfg(target_os = "redox")] + match OutputBuilder::in_redox_logging_scheme("disk", "pcie", "nvme.log") { + Ok(b) => logger = logger.with_output( + // TODO: Add a configuration file for this + b.with_filter(log::LevelFilter::Info) + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("nvmed: failed to create nvme.log: {}", error), + } + + #[cfg(target_os = "redox")] + match OutputBuilder::in_redox_logging_scheme("disk", "pcie", "nvme.ansi.log") { + Ok(b) => logger = logger.with_output( + b.with_filter(log::LevelFilter::Info) + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("nvmed: failed to create nvme.ansi.log: {}", error), + } + + match logger.enable() { + Ok(logger_ref) => { + eprintln!("nvmed: enabled logger"); + Some(logger_ref) + } + Err(error) => { + eprintln!("nvmed: failed to set default logger: {}", error); + None + } + } +} + +fn main() { + // Daemonize + if unsafe { syscall::clone(CloneFlags::empty()).unwrap() } != 0 { + return; + } + + let _logger_ref = setup_logging(); + + let mut pcid_handle = + PcidServerHandle::connect_default().expect("nvmed: failed to setup channel to pcid"); + let pci_config = pcid_handle + .fetch_config() + .expect("nvmed: failed to fetch config"); + + let bar = match pci_config.func.bars[0] { + PciBar::Memory(mem) => mem, + other => panic!("received a non-memory BAR ({:?})", other), + }; + let bar_size = pci_config.func.bar_sizes[0]; + let irq = pci_config.func.legacy_interrupt_line; + + let mut name = pci_config.func.name(); + name.push_str("_nvme"); + + log::info!("NVME PCI CONFIG: {:?}", pci_config); + + let allocated_bars = AllocatedBars::default(); + + let address = unsafe { + syscall::physmap( + bar as usize, + bar_size as usize, + PHYSMAP_WRITE | PHYSMAP_NO_CACHE, + ) + .expect("nvmed: failed to map address") + }; + *allocated_bars.0[0].lock().unwrap() = Some(Bar { + physical: bar as usize, + bar_size: bar_size as usize, + ptr: NonNull::new(address as *mut u8).expect("Physmapping BAR gave nullptr"), + }); + let event_fd = syscall::open("event:", syscall::O_RDWR | syscall::O_CLOEXEC) + .expect("nvmed: failed to open event queue"); + let mut event_file = unsafe { File::from_raw_fd(event_fd as RawFd) }; + + let scheme_name = format!("disk/{}", name); + let socket_fd = syscall::open( + &format!(":{}", scheme_name), + syscall::O_RDWR | syscall::O_CREAT | syscall::O_NONBLOCK | syscall::O_CLOEXEC, + ) + .expect("nvmed: failed to create disk scheme"); + + syscall::write( + event_fd, + &syscall::Event { + id: socket_fd, + flags: syscall::EVENT_READ, + data: 0, + }, + ) + .expect("nvmed: failed to watch disk scheme events"); + + let mut socket_file = unsafe { File::from_raw_fd(socket_fd as RawFd) }; + + let (reactor_sender, reactor_receiver) = crossbeam_channel::unbounded(); + let (interrupt_method, interrupt_sources) = + get_int_method(&mut pcid_handle, &pci_config.func, &allocated_bars) + .expect("nvmed: failed to find a suitable interrupt method"); + let mut nvme = Nvme::new(address, interrupt_method, pcid_handle, reactor_sender) + .expect("nvmed: failed to allocate driver data"); + unsafe { nvme.init() } + log::debug!("Finished base initialization"); + let nvme = Arc::new(nvme); + let reactor_thread = nvme::cq_reactor::start_cq_reactor_thread(Arc::clone(&nvme), interrupt_sources, reactor_receiver); + let namespaces = futures::executor::block_on(nvme.init_with_queues()); + + syscall::setrens(0, 0).expect("nvmed: failed to enter null namespace"); + + let mut scheme = DiskScheme::new(scheme_name, nvme, namespaces); + let mut todo = Vec::new(); + 'events: loop { + let mut event = Event::default(); + if event_file + .read(&mut event) + .expect("nvmed: failed to read event queue") + == 0 + { + break; + } + + match event.data { + 0 => loop { + let mut packet = Packet::default(); + match socket_file.read(&mut packet) { + Ok(0) => break 'events, + Ok(_) => (), + Err(err) => match err.kind() { + ErrorKind::WouldBlock => break, + _ => Err(err).expect("nvmed: failed to read disk scheme"), + }, + } + todo.push(packet); + }, + unknown => { + panic!("nvmed: unknown event data {}", unknown); + } + } + + let mut i = 0; + while i < todo.len() { + if let Some(a) = scheme.handle(&todo[i]) { + let mut packet = todo.remove(i); + packet.a = a; + socket_file + .write(&packet) + .expect("nvmed: failed to write disk scheme"); + } else { + i += 1; + } + } + } + + //TODO: destroy NVMe stuff + reactor_thread.join().expect("nvmed: failed to join reactor thread"); +} diff --git a/nvmed/src/nvme.rs b/nvmed/src/nvme.rs deleted file mode 100644 index 8754ded99b..0000000000 --- a/nvmed/src/nvme.rs +++ /dev/null @@ -1,655 +0,0 @@ -use std::{ptr, thread}; -use std::collections::BTreeMap; -use syscall::io::{Dma, Io, Mmio}; -use syscall::error::{Error, Result, EINVAL}; - -#[derive(Clone, Copy)] -#[repr(packed)] -pub struct NvmeCmd { - /// Opcode - opcode: u8, - /// Flags - flags: u8, - /// Command ID - cid: u16, - /// Namespace identifier - nsid: u32, - /// Reserved - _rsvd: u64, - /// Metadata pointer - mptr: u64, - /// Data pointer - dptr: [u64; 2], - /// Command dword 10 - cdw10: u32, - /// Command dword 11 - cdw11: u32, - /// Command dword 12 - cdw12: u32, - /// Command dword 13 - cdw13: u32, - /// Command dword 14 - cdw14: u32, - /// Command dword 15 - cdw15: u32, -} - -impl NvmeCmd { - pub fn create_io_completion_queue(cid: u16, qid: u16, ptr: usize, size: u16) -> Self { - Self { - opcode: 5, - flags: 0, - cid: cid, - nsid: 0, - _rsvd: 0, - mptr: 0, - dptr: [ptr as u64, 0], - cdw10: ((size as u32) << 16) | (qid as u32), - cdw11: 1 /* Physically Contiguous */, //TODO: IV, IEN - cdw12: 0, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn create_io_submission_queue(cid: u16, qid: u16, ptr: usize, size: u16, cqid: u16) -> Self { - Self { - opcode: 1, - flags: 0, - cid: cid, - nsid: 0, - _rsvd: 0, - mptr: 0, - dptr: [ptr as u64, 0], - cdw10: ((size as u32) << 16) | (qid as u32), - cdw11: ((cqid as u32) << 16) | 1 /* Physically Contiguous */, //TODO: QPRIO - cdw12: 0, //TODO: NVMSETID - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn identify_namespace(cid: u16, ptr: usize, nsid: u32) -> Self { - Self { - opcode: 6, - flags: 0, - cid: cid, - nsid: nsid, - _rsvd: 0, - mptr: 0, - dptr: [ptr as u64, 0], - cdw10: 0, - cdw11: 0, - cdw12: 0, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn identify_controller(cid: u16, ptr: usize) -> Self { - Self { - opcode: 6, - flags: 0, - cid: cid, - nsid: 0, - _rsvd: 0, - mptr: 0, - dptr: [ptr as u64, 0], - cdw10: 1, - cdw11: 0, - cdw12: 0, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn identify_namespace_list(cid: u16, ptr: usize, base: u32) -> Self { - Self { - opcode: 6, - flags: 0, - cid: cid, - nsid: base, - _rsvd: 0, - mptr: 0, - dptr: [ptr as u64, 0], - cdw10: 2, - cdw11: 0, - cdw12: 0, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn io_read(cid: u16, nsid: u32, lba: u64, blocks_1: u16, ptr0: u64, ptr1: u64) -> Self { - Self { - opcode: 2, - flags: 1 << 6, - cid: cid, - nsid: nsid, - _rsvd: 0, - mptr: 0, - dptr: [ptr0, ptr1], - cdw10: lba as u32, - cdw11: (lba >> 32) as u32, - cdw12: blocks_1 as u32, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } - - pub fn io_write(cid: u16, nsid: u32, lba: u64, blocks_1: u16, ptr0: u64, ptr1: u64) -> Self { - Self { - opcode: 1, - flags: 1 << 6, - cid: cid, - nsid: nsid, - _rsvd: 0, - mptr: 0, - dptr: [ptr0, ptr1], - cdw10: lba as u32, - cdw11: (lba >> 32) as u32, - cdw12: blocks_1 as u32, - cdw13: 0, - cdw14: 0, - cdw15: 0, - } - } -} - -#[derive(Clone, Copy, Debug)] -#[repr(packed)] -pub struct NvmeComp { - command_specific: u32, - _rsvd: u32, - sq_head: u16, - sq_id: u16, - cid: u16, - status: u16, -} - -#[repr(packed)] -pub struct NvmeRegs { - /// Controller Capabilities - cap: Mmio, - /// Version - vs: Mmio, - /// Interrupt mask set - intms: Mmio, - /// Interrupt mask clear - intmc: Mmio, - /// Controller configuration - cc: Mmio, - /// Reserved - _rsvd: Mmio, - /// Controller status - csts: Mmio, - /// NVM subsystem reset - nssr: Mmio, - /// Admin queue attributes - aqa: Mmio, - /// Admin submission queue base address - asq: Mmio, - /// Admin completion queue base address - acq: Mmio, - /// Controller memory buffer location - cmbloc: Mmio, - /// Controller memory buffer size - cmbsz: Mmio, -} - -pub struct NvmeCmdQueue { - data: Dma<[NvmeCmd; 64]>, - i: usize, -} - -impl NvmeCmdQueue { - fn new() -> Result { - Ok(Self { - data: Dma::zeroed()?, - i: 0, - }) - } - - fn submit(&mut self, entry: NvmeCmd) -> usize { - self.data[self.i] = entry; - self.i = (self.i + 1) % self.data.len(); - self.i - } -} - -pub struct NvmeCompQueue { - data: Dma<[NvmeComp; 256]>, - i: usize, - phase: bool, -} - -impl NvmeCompQueue { - fn new() -> Result { - Ok(Self { - data: Dma::zeroed()?, - i: 0, - phase: true, - }) - } - - pub (crate) fn complete(&mut self) -> Option<(usize, NvmeComp)> { - let entry = unsafe { - ptr::read_volatile(self.data.as_ptr().add(self.i)) - }; - // println!("{:?}", entry); - if ((entry.status & 1) == 1) == self.phase { - self.i = (self.i + 1) % self.data.len(); - if self.i == 0 { - self.phase = ! self.phase; - } - Some((self.i, entry)) - } else { - None - } - } - - fn complete_spin(&mut self) -> (usize, NvmeComp) { - loop { - if let Some(some) = self.complete() { - return some; - } else { - unsafe { asm!("pause"); } - } - } - } -} - -pub struct NvmeNamespace { - pub id: u32, - pub blocks: u64, - pub block_size: u64, -} - -pub struct Nvme { - regs: &'static mut NvmeRegs, - submission_queues: [NvmeCmdQueue; 2], - pub (crate) completion_queues: [NvmeCompQueue; 2], - buffer: Dma<[u8; 512 * 4096]>, // 2MB of buffer - buffer_prp: Dma<[u64; 512]>, // 4KB of PRP for the buffer -} - -impl Nvme { - pub fn new(address: usize) -> Result { - Ok(Nvme { - regs: unsafe { &mut *(address as *mut NvmeRegs) }, - submission_queues: [NvmeCmdQueue::new()?, NvmeCmdQueue::new()?], - completion_queues: [NvmeCompQueue::new()?, NvmeCompQueue::new()?], - buffer: Dma::zeroed()?, - buffer_prp: Dma::zeroed()?, - }) - } - - unsafe fn doorbell(&mut self, index: usize) -> &'static mut Mmio { - let dstrd = ((self.regs.cap.read() >> 32) & 0b1111) as usize; - let addr = (self.regs as *mut _ as usize) - + 0x1000 - + index * (4 << dstrd); - &mut *(addr as *mut Mmio) - } - - pub unsafe fn submission_queue_tail(&mut self, qid: u16, tail: u16) { - self.doorbell(2 * (qid as usize)).write(tail as u32); - } - - pub unsafe fn completion_queue_head(&mut self, qid: u16, head: u16) { - self.doorbell(2 * (qid as usize) + 1).write(head as u32) - } - - pub unsafe fn init(&mut self) -> BTreeMap { - for i in 0..self.buffer_prp.len() { - self.buffer_prp[i] = (self.buffer.physical() + i * 4096) as u64; - } - - // println!(" - CAPS: {:X}", self.regs.cap.read()); - // println!(" - VS: {:X}", self.regs.vs.read()); - // println!(" - CC: {:X}", self.regs.cc.read()); - // println!(" - CSTS: {:X}", self.regs.csts.read()); - - // println!(" - Disable"); - self.regs.cc.writef(1, false); - - // println!(" - Waiting for not ready"); - loop { - let csts = self.regs.csts.read(); - // println!("CSTS: {:X}", csts); - if csts & 1 == 1 { - asm!("pause"); - } else { - break; - } - } - - // println!(" - Mask all interrupts"); - self.regs.intms.write(0xFFFFFFFF); - - for (qid, queue) in self.completion_queues.iter().enumerate() { - let data = &queue.data; - // println!(" - completion queue {}: {:X}, {}", qid, data.physical(), data.len()); - } - - for (qid, queue) in self.submission_queues.iter().enumerate() { - let data = &queue.data; - // println!(" - submission queue {}: {:X}, {}", qid, data.physical(), data.len()); - } - - { - let asq = &self.submission_queues[0]; - let acq = &self.completion_queues[0]; - self.regs.aqa.write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); - self.regs.asq.write(asq.data.physical() as u64); - self.regs.acq.write(acq.data.physical() as u64); - - // Set IOCQES, IOSQES, AMS, MPS, and CSS - let mut cc = self.regs.cc.read(); - cc &= 0xFF00000F; - cc |= (4 << 20) | (6 << 16); - self.regs.cc.write(cc); - } - - // println!(" - Enable"); - self.regs.cc.writef(1, true); - - // println!(" - Waiting for ready"); - loop { - let csts = self.regs.csts.read(); - // println!("CSTS: {:X}", csts); - if csts & 1 == 0 { - asm!("pause"); - } else { - break; - } - } - - { - //TODO: Use buffer - let data: Dma<[u8; 4096]> = Dma::zeroed().unwrap(); - - // println!(" - Attempting to identify controller"); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = NvmeCmd::identify_controller( - cid, data.physical() - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to identify controller"); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } - - // println!(" - Dumping identify controller"); - - let mut serial = String::new(); - for &b in &data[4..24] { - if b == 0 { - break; - } - serial.push(b as char); - } - - let mut model = String::new(); - for &b in &data[24..64] { - if b == 0 { - break; - } - model.push(b as char); - } - - let mut firmware = String::new(); - for &b in &data[64..72] { - if b == 0 { - break; - } - firmware.push(b as char); - } - - println!( - " - Model: {} Serial: {} Firmware: {}", - model.trim(), - serial.trim(), - firmware.trim() - ); - } - - let mut nsids = Vec::new(); - { - //TODO: Use buffer - let data: Dma<[u32; 1024]> = Dma::zeroed().unwrap(); - - // println!(" - Attempting to retrieve namespace ID list"); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = NvmeCmd::identify_namespace_list( - cid, data.physical(), 0 - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to retrieve namespace ID list"); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } - - // println!(" - Dumping namespace ID list"); - for &nsid in data.iter() { - if nsid != 0 { - nsids.push(nsid); - } - } - } - - let mut namespaces = BTreeMap::new(); - for &nsid in nsids.iter() { - //TODO: Use buffer - let data: Dma<[u8; 4096]> = Dma::zeroed().unwrap(); - - // println!(" - Attempting to identify namespace {}", nsid); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = NvmeCmd::identify_namespace( - cid, data.physical(), nsid - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to identify namespace {}", nsid); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } - - // println!(" - Dumping identify namespace"); - - - let size = *(data.as_ptr().offset(0) as *const u64); - let capacity = *(data.as_ptr().offset(8) as *const u64); - println!( - " - ID: {} Size: {} Capacity: {}", - nsid, - size, - capacity - ); - - //TODO: Read block size - - namespaces.insert(nsid, NvmeNamespace { - id: nsid, - blocks: size, - block_size: 512, // TODO - }); - } - - for io_qid in 1..self.completion_queues.len() { - let (ptr, len) = { - let queue = &self.completion_queues[io_qid]; - (queue.data.physical(), queue.data.len()) - }; - - // println!(" - Attempting to create I/O completion queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = NvmeCmd::create_io_completion_queue( - cid, io_qid as u16, ptr, (len - 1) as u16 - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to create I/O completion queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } - } - - for io_qid in 1..self.submission_queues.len() { - let (ptr, len) = { - let queue = &self.submission_queues[io_qid]; - (queue.data.physical(), queue.data.len()) - }; - - // println!(" - Attempting to create I/O submission queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - //TODO: Get completion queue ID through smarter mechanism - let entry = NvmeCmd::create_io_submission_queue( - cid, io_qid as u16, ptr, (len - 1) as u16, io_qid as u16 - ); - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - // println!(" - Waiting to create I/O submission queue {}", io_qid); - { - let qid = 0; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - self.completion_queue_head(qid as u16, head as u16); - } - } - - // println!(" - Complete"); - - namespaces - } - - unsafe fn namespace_rw(&mut self, nsid: u32, lba: u64, blocks_1: u16, write: bool) -> Result<()> { - //TODO: Get real block size - let block_size = 512; - - let bytes = ((blocks_1 as u64) + 1) * block_size; - let (ptr0, ptr1) = if bytes <= 4096 { - (self.buffer_prp[0], 0) - } else if bytes <= 8192 { - (self.buffer_prp[0], self.buffer_prp[1]) - } else { - (self.buffer_prp[0], (self.buffer_prp.physical() + 8) as u64) - }; - - { - let qid = 1; - let queue = &mut self.submission_queues[qid]; - let cid = queue.i as u16; - let entry = if write { - NvmeCmd::io_write( - cid, nsid, lba, blocks_1, ptr0, ptr1 - ) - } else { - NvmeCmd::io_read( - cid, nsid, lba, blocks_1, ptr0, ptr1 - ) - }; - let tail = queue.submit(entry); - self.submission_queue_tail(qid as u16, tail as u16); - } - - { - let qid = 1; - let queue = &mut self.completion_queues[qid]; - let (head, entry) = queue.complete_spin(); - //TODO: Handle errors - self.completion_queue_head(qid as u16, head as u16); - } - - Ok(()) - } - - pub unsafe fn namespace_read(&mut self, nsid: u32, mut lba: u64, buf: &mut [u8]) -> Result> { - //TODO: Use interrupts - - //TODO: Get real block size - let block_size = 512; - - for chunk in buf.chunks_mut(self.buffer.len()) { - let blocks = (chunk.len() + block_size - 1) / block_size; - - assert!(blocks > 0); - assert!(blocks <= 0x1_0000); - - self.namespace_rw(nsid, lba, (blocks - 1) as u16, false)?; - - chunk.copy_from_slice(&self.buffer[..chunk.len()]); - - lba += blocks as u64; - } - - Ok(Some(buf.len())) - } - - pub unsafe fn namespace_write(&mut self, nsid: u32, mut lba: u64, buf: &[u8]) -> Result> { - //TODO: Use interrupts - - //TODO: Get real block size - let block_size = 512; - - for chunk in buf.chunks(self.buffer.len()) { - let blocks = (chunk.len() + block_size - 1) / block_size; - - assert!(blocks > 0); - assert!(blocks <= 0x1_0000); - - self.buffer[..chunk.len()].copy_from_slice(chunk); - - self.namespace_rw(nsid, lba, (blocks - 1) as u16, true)?; - - lba += blocks as u64; - } - - Ok(Some(buf.len())) - } -} diff --git a/nvmed/src/nvme/cmd.rs b/nvmed/src/nvme/cmd.rs new file mode 100644 index 0000000000..251fe0da5f --- /dev/null +++ b/nvmed/src/nvme/cmd.rs @@ -0,0 +1,162 @@ +use super::NvmeCmd; + +impl NvmeCmd { + pub fn create_io_completion_queue( + cid: u16, + qid: u16, + ptr: usize, + size: u16, + iv: Option, + ) -> Self { + const DW11_PHYSICALLY_CONTIGUOUS_BIT: u32 = 0x0000_0001; + const DW11_ENABLE_INTERRUPTS_BIT: u32 = 0x0000_0002; + const DW11_INTERRUPT_VECTOR_SHIFT: u8 = 16; + + Self { + opcode: 5, + flags: 0, + cid, + nsid: 0, + _rsvd: 0, + mptr: 0, + dptr: [ptr as u64, 0], + cdw10: ((size as u32) << 16) | (qid as u32), + + cdw11: DW11_PHYSICALLY_CONTIGUOUS_BIT + | if let Some(iv) = iv { + // enable interrupts if a vector is present + DW11_ENABLE_INTERRUPTS_BIT | (u32::from(iv) << DW11_INTERRUPT_VECTOR_SHIFT) + } else { + 0 + }, + + cdw12: 0, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + pub fn create_io_submission_queue( + cid: u16, + qid: u16, + ptr: usize, + size: u16, + cqid: u16, + ) -> Self { + Self { + opcode: 1, + flags: 0, + cid, + nsid: 0, + _rsvd: 0, + mptr: 0, + dptr: [ptr as u64, 0], + cdw10: ((size as u32) << 16) | (qid as u32), + cdw11: ((cqid as u32) << 16) | 1, /* Physically Contiguous */ + //TODO: QPRIO + cdw12: 0, //TODO: NVMSETID + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + pub fn identify_namespace(cid: u16, ptr: usize, nsid: u32) -> Self { + Self { + opcode: 6, + flags: 0, + cid, + nsid, + _rsvd: 0, + mptr: 0, + dptr: [ptr as u64, 0], + cdw10: 0, + cdw11: 0, + cdw12: 0, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + pub fn identify_controller(cid: u16, ptr: usize) -> Self { + Self { + opcode: 6, + flags: 0, + cid, + nsid: 0, + _rsvd: 0, + mptr: 0, + dptr: [ptr as u64, 0], + cdw10: 1, + cdw11: 0, + cdw12: 0, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + pub fn identify_namespace_list(cid: u16, ptr: usize, base: u32) -> Self { + Self { + opcode: 6, + flags: 0, + cid, + nsid: base, + _rsvd: 0, + mptr: 0, + dptr: [ptr as u64, 0], + cdw10: 2, + cdw11: 0, + cdw12: 0, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + pub fn get_features(cid: u16, ptr: usize, fid: u8) -> Self { + Self { + opcode: 0xA, + dptr: [ptr as u64, 0], + cdw10: u32::from(fid), // TODO: SEL + ..Default::default() + } + } + + pub fn io_read(cid: u16, nsid: u32, lba: u64, blocks_1: u16, ptr0: u64, ptr1: u64) -> Self { + Self { + opcode: 2, + flags: 1 << 6, + cid, + nsid, + _rsvd: 0, + mptr: 0, + dptr: [ptr0, ptr1], + cdw10: lba as u32, + cdw11: (lba >> 32) as u32, + cdw12: blocks_1 as u32, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } + + pub fn io_write(cid: u16, nsid: u32, lba: u64, blocks_1: u16, ptr0: u64, ptr1: u64) -> Self { + Self { + opcode: 1, + flags: 1 << 6, + cid, + nsid, + _rsvd: 0, + mptr: 0, + dptr: [ptr0, ptr1], + cdw10: lba as u32, + cdw11: (lba >> 32) as u32, + cdw12: blocks_1 as u32, + cdw13: 0, + cdw14: 0, + cdw15: 0, + } + } +} diff --git a/nvmed/src/nvme/cq_reactor.rs b/nvmed/src/nvme/cq_reactor.rs new file mode 100644 index 0000000000..d327e7056e --- /dev/null +++ b/nvmed/src/nvme/cq_reactor.rs @@ -0,0 +1,386 @@ +//! The Completion Queue Reactor. Functions like any other async/await reactor, but is driven by +//! IRQs triggering wakeups in order to poll NVME completion queues (see `CompletionFuture`). +//! +//! While the reactor is primarily intended to wait for IRQs and then poll completion queues, it +//! can also be used for notifying when a full submission queue can submit a new command (see +//! `AvailableSqEntryFuture`). + +use std::convert::TryFrom; +use std::fs::File; +use std::future::Future; +use std::io::prelude::*; +use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::{mem, task, thread}; + +use syscall::data::Event; +use syscall::Result; + +use crossbeam_channel::Receiver; + +use super::{CmdId, CqId, InterruptSources, Nvme, NvmeComp, NvmeCmd, SqId}; + +/// A notification request, sent by the future in order to tell the completion thread that the +/// current task wants a notification when a matching completion queue entry has been seen. +#[derive(Debug)] +pub enum NotifReq { + RequestCompletion { + cq_id: CqId, + sq_id: SqId, + cmd_id: CmdId, + + waker: task::Waker, + + // TODO: Get rid of this allocation, or maybe a thread-local vec for reusing. + // TODO: Maybe the `remem` crate. + message: Arc>>, + }, + RequestAvailSubmission { + sq_id: SqId, + waker: task::Waker, + } +} + +enum PendingReq { + PendingCompletion { + waker: task::Waker, + message: Arc>>, + cq_id: CqId, + sq_id: SqId, + cmd_id: CmdId, + }, + PendingAvailSubmission { + waker: task::Waker, + sq_id: SqId, + }, +} +struct CqReactor { + int_sources: InterruptSources, + nvme: Arc, + pending_reqs: Vec, + // used to store commands that may be completed before a completion is requested + receiver: Receiver, + event_queue: File, +} +impl CqReactor { + fn create_event_queue(int_sources: &mut InterruptSources) -> Result { + use syscall::flag::*; + let fd = syscall::open("event:", O_CLOEXEC | O_RDWR)?; + let mut file = unsafe { File::from_raw_fd(fd as RawFd) }; + + for (num, irq_handle) in int_sources.iter_mut() { + if file + .write(&Event { + id: irq_handle.as_raw_fd() as usize, + flags: syscall::EVENT_READ, + data: num as usize, + }) + .unwrap() + == 0 + { + panic!("Failed to setup event queue for {} {:?}", num, irq_handle); + } + } + Ok(file) + } + fn new( + nvme: Arc, + mut int_sources: InterruptSources, + receiver: Receiver, + ) -> Result { + Ok(Self { + event_queue: Self::create_event_queue(&mut int_sources)?, + int_sources, + nvme, + pending_reqs: Vec::new(), + receiver, + }) + } + fn handle_notif_reqs_raw(pending_reqs: &mut Vec, receiver: &Receiver, block_until_first: bool) { + let mut blocking_iter; + let mut nonblocking_iter; + + let iter: &mut dyn Iterator = if block_until_first { + blocking_iter = std::iter::once(receiver.recv().unwrap()).chain(receiver.try_iter()); + &mut blocking_iter + } else { + nonblocking_iter = receiver.try_iter(); + &mut nonblocking_iter + }; + + for req in iter { + log::trace!("Got notif req: {:?}", req); + match req { + NotifReq::RequestCompletion { + sq_id, + cq_id, + cmd_id, + waker, + message, + } => pending_reqs.push(PendingReq::PendingCompletion { + sq_id, + cq_id, + cmd_id, + message, + waker, + }), + NotifReq::RequestAvailSubmission { sq_id, waker } => pending_reqs.push(PendingReq::PendingAvailSubmission { sq_id, waker, }), + } + } + } + fn poll_completion_queues(&mut self, iv: u16) -> Option<()> { + let ivs_read_guard = self.nvme.cqs_for_ivs.read().unwrap(); + let cqs_read_guard = self.nvme.completion_queues.read().unwrap(); + + let mut entry_count = 0; + + let cq_ids = ivs_read_guard.get(&iv)?; + + for cq_id in cq_ids.iter().copied() { + let mut completion_queue_guard = cqs_read_guard.get(&cq_id)?.lock().unwrap(); + let &mut (ref mut completion_queue, _) = &mut *completion_queue_guard; + + while let Some((head, entry)) = completion_queue.complete() { + unsafe { self.nvme.completion_queue_head(cq_id, head) }; + + log::trace!("Got completion queue entry (CQID {}): {:?} at {}", cq_id, entry, head); + + { + let submission_queues_read_lock = self.nvme.submission_queues.read().unwrap(); + // this lock is actually important, since it will block during submission from other + // threads. the lock won't be held for long by the submitters, but it still prevents + // the entry being lost before this reactor is actually able to respond: + let &(ref sq_lock, corresponding_cq_id) = submission_queues_read_lock.get(&{entry.sq_id}).expect("nvmed: internal error: queue returned from controller doesn't exist"); + assert_eq!(cq_id, corresponding_cq_id); + let mut sq_guard = sq_lock.lock().unwrap(); + sq_guard.head = entry.sq_head; + // the channel still has to be polled twice though: + Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, false); + } + + + Self::try_notify_futures(&mut self.pending_reqs, cq_id, &entry); + + entry_count += 1; + } + } + if entry_count == 0 {} + + Some(()) + } + fn finish_pending_completion(pending_reqs: &mut Vec, req_cq_id: CqId, cq_id: CqId, sq_id: SqId, cmd_id: CmdId, entry: &NvmeComp, i: usize) -> bool { + if req_cq_id == cq_id + && sq_id == entry.sq_id + && cmd_id == entry.cid + { + let (waker, message) = match pending_reqs.remove(i) { + PendingReq::PendingCompletion { waker, message, .. } => (waker, message), + _ => unreachable!(), + }; + + *message.lock().unwrap() = Some(CompletionMessage { cq_entry: *entry }); + waker.wake(); + + true + } else { + false + } + } + fn finish_pending_avail_submission(pending_reqs: &mut Vec, sq_id: SqId, entry: &NvmeComp, i: usize) -> bool { + if sq_id == entry.sq_id { + let waker = match pending_reqs.remove(i) { + PendingReq::PendingAvailSubmission { waker, .. } => waker, + _ => unreachable!(), + }; + waker.wake(); + + true + } else { + false + } + } + fn try_notify_futures(pending_reqs: &mut Vec, cq_id: CqId, entry: &NvmeComp) -> Option<()> { + let mut i = 0usize; + + let mut futures_notified = 0; + + while i < pending_reqs.len() { + match &pending_reqs[i] { + &PendingReq::PendingCompletion { cq_id: req_cq_id, sq_id, cmd_id, .. } => if Self::finish_pending_completion(pending_reqs, req_cq_id, cq_id, sq_id, cmd_id, entry, i) { + futures_notified += 1; + } else { + i += 1; + } + &PendingReq::PendingAvailSubmission { sq_id, .. } => if Self::finish_pending_avail_submission(pending_reqs, sq_id, entry, i) { + futures_notified += 1; + } else { + i += 1; + } + } + } + if futures_notified == 0 {} + Some(()) + } + + fn run(mut self) { + log::debug!("Running CQ reactor"); + let mut event = Event::default(); + let mut irq_word = [0u8; 8]; // stores the IRQ count + + const WORD_SIZE: usize = mem::size_of::(); + + loop { + let block_until_first = self.pending_reqs.is_empty(); + Self::handle_notif_reqs_raw(&mut self.pending_reqs, &self.receiver, block_until_first); + log::trace!("Handled notif reqs"); + + // block on getting the next event + if self.event_queue.read(&mut event).unwrap() == 0 { + // event queue has been destroyed + break; + } + + let (vector, irq_handle) = match self.int_sources.iter_mut().nth(event.data) { + Some(s) => s, + None => continue, + }; + if irq_handle.read(&mut irq_word[..WORD_SIZE]).unwrap() == 0 { + continue; + } + // acknowledge the interrupt (only necessary for level-triggered INTx# interrups) + if irq_handle.write(&irq_word[..WORD_SIZE]).unwrap() == 0 { + continue; + } + log::trace!("NVME IRQ: vector {}", vector); + self.nvme.set_vector_masked(vector, true); + self.poll_completion_queues(vector); + self.nvme.set_vector_masked(vector, false); + } + } +} + +pub fn start_cq_reactor_thread( + nvme: Arc, + interrupt_sources: InterruptSources, + receiver: Receiver, +) -> thread::JoinHandle<()> { + // Actually, nothing prevents us from spawning additional threads. the channel is MPMC and + // everything is properly synchronized. I'm not saying this is strictly required, but with + // multiple completion queues it might actually be worth considering. The (in-kernel) IRQ + // subsystem can have some room for improvement regarding lowering the latency, but MSI-X allows + // multiple vectors to point to different CPUs, so that the load can be balanced across the + // logical processors. + thread::spawn(move || { + CqReactor::new(nvme, interrupt_sources, receiver) + .expect("nvmed: failed to setup CQ reactor") + .run() + }) +} + +#[derive(Debug)] +pub struct CompletionMessage { + cq_entry: NvmeComp, +} + +enum CompletionFutureState<'a, F> { + // the future is in its initial state: the command has not been submitted yet, and no interest + // has been registered. this state will repeat until a free submission queue entry appears to + // it, which it probably will since queues aren't supposed to be nearly always be full. + PendingSubmission { + cmd_init: F, + nvme: &'a Nvme, + sq_id: SqId, + }, + PendingCompletion { + nvme: &'a Nvme, + cq_id: CqId, + cmd_id: CmdId, + sq_id: SqId, + message: Arc>>, + }, + Finished, + Placeholder, +} +pub struct CompletionFuture<'a, F> { + state: CompletionFutureState<'a, F>, +} + +// enum not self-referential +impl Unpin for CompletionFuture<'_, F> {} + +impl Future for CompletionFuture<'_, F> +where + F: FnOnce(CmdId) -> NvmeCmd, +{ + type Output = NvmeComp; + + fn poll(self: Pin<&mut Self>, context: &mut task::Context) -> task::Poll { + let this = &mut self.get_mut().state; + + match mem::replace(this, CompletionFutureState::Placeholder) { + CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id } => { + let sqs_read_guard = nvme.submission_queues.read().unwrap(); + let &(ref sq_lock, cq_id) = sqs_read_guard + .get(&sq_id) + .expect("nvmed: internal error: given SQ for SQ ID not there"); + let mut sq_guard = sq_lock.lock().unwrap(); + let sq = &mut *sq_guard; + + if sq.is_full() { + // when the CQ reactor gets a new completion queue entry, it'll lock the + // submisson queue it came from. since we're holding the same lock, this + // message will always be sent before the reactor is done with the entry. + nvme.reactor_sender.send(NotifReq::RequestAvailSubmission { sq_id, waker: context.waker().clone() }).unwrap(); + *this = CompletionFutureState::PendingSubmission { cmd_init, nvme, sq_id }; + return task::Poll::Pending; + } + + let cmd_id = + u16::try_from(sq.tail).expect("nvmed: internal error: CQ has more than 2^16 entries"); + let tail = sq.submit_unchecked(cmd_init(cmd_id)); + let tail = u16::try_from(tail).unwrap(); + + // make sure that we register interest before the reactor can get notified + let message = Arc::new(Mutex::new(None)); + *this = CompletionFutureState::PendingCompletion { nvme, cq_id, cmd_id, sq_id, message: Arc::clone(&message), }; + nvme.reactor_sender.send(NotifReq::RequestCompletion { cq_id, sq_id, cmd_id, message, waker: context.waker().clone() }).expect("reactor dead"); + unsafe { nvme.submission_queue_tail(sq_id, tail) }; + task::Poll::Pending + } + CompletionFutureState::PendingCompletion { + message, + cq_id, + cmd_id, + sq_id, + nvme, + } => { + if let Some(value) = message.lock().unwrap().take() { + *this = CompletionFutureState::Finished; + return task::Poll::Ready(value.cq_entry); + } + nvme.reactor_sender.send(NotifReq::RequestCompletion { + cq_id, + sq_id, + cmd_id, + waker: context.waker().clone(), + message: Arc::clone(&message), + }).expect("reactor dead"); + *this = CompletionFutureState::PendingCompletion { message, cq_id, cmd_id, sq_id, nvme }; + task::Poll::Pending + } + CompletionFutureState::Finished => { + panic!("calling poll() on an already finished CompletionFuture") + } + CompletionFutureState::Placeholder => unreachable!(), + } + } +} + +impl Nvme { + pub fn submit_and_complete_command NvmeCmd>(&self, sq_id: SqId, cmd_init: F) -> CompletionFuture { + CompletionFuture { + state: CompletionFutureState::PendingSubmission { cmd_init, nvme: &self, sq_id }, + } + } +} diff --git a/nvmed/src/nvme/identify.rs b/nvmed/src/nvme/identify.rs new file mode 100644 index 0000000000..e5c5db5a53 --- /dev/null +++ b/nvmed/src/nvme/identify.rs @@ -0,0 +1,216 @@ +use syscall::Dma; + +use super::{Nvme, NvmeCmd, NvmeNamespace}; + +/// See NVME spec section 5.15.2.2. +#[derive(Clone, Copy)] +#[repr(packed)] +pub struct IdentifyControllerData { + /// PCI vendor ID, always the same as in the PCI function header. + pub vid: u16, + /// PCI subsystem vendor ID. + pub ssvid: u16, + /// ASCII + pub serial_no: [u8; 20], + /// ASCII + pub model_no: [u8; 48], + /// ASCII + pub firmware_rev: [u8; 8], + // TODO: Lots of fields + pub _4k_pad: [u8; 4096 - 72], +} + +/// See NVME spec section 5.15.2.1. +#[derive(Clone, Copy)] +#[repr(packed)] +pub struct IdentifyNamespaceData { + pub nsze: u64, + pub ncap: u64, + pub nuse: u64, + + pub nsfeat: u8, + pub nlbaf: u8, + pub flbas: u8, + pub mc: u8, + + pub dpc: u8, + pub dps: u8, + pub nmic: u8, + pub rescap: u8, + // 32 + pub fpi: u8, + pub dlfeat: u8, + pub nawun: u16, + + pub nawupf: u16, + pub nacwu: u16, + // 40 + pub nabsn: u16, + pub nabo: u16, + + pub nabspf: u16, + pub noiob: u16, + // 48 + pub nvmcap: u128, + // 64 + pub npwg: u16, + pub npwa: u16, + pub npdg: u16, + pub npda: u16, + // 72 + pub nows: u16, + pub _rsvd1: [u8; 18], + // 92 + + pub anagrpid: u32, + pub _rsvd2: [u8; 3], + pub nsattr: u8, + + // 100 + pub nvmsetid: u16, + pub endgid: u16, + pub nguid: [u8; 16], + pub eui64: u64, + + pub lba_format_support: [LbaFormat; 16], + pub _rsvd3: [u8; 192], + pub vendor_specific: [u8; 3712], +} + +impl IdentifyNamespaceData { + pub fn size_in_blocks(&self) -> u64 { + self.nsze + } + pub fn capacity_in_blocks(&self) -> u64 { + self.ncap + } + /// Guaranteed to be within 0..=15 + pub fn formatted_lba_size_idx(&self) -> usize { + (self.flbas & 0xF) as usize + } + pub fn formatted_lba_size(&self) -> &LbaFormat { + &self.lba_format_support[self.formatted_lba_size_idx()] + } + pub fn has_metadata_after_data(&self) -> bool { + (self.flbas & (1 << 4)) != 0 + } +} + +#[derive(Clone, Copy)] +#[repr(packed)] +pub struct LbaFormat(pub u32); + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RelativePerformance { + Best = 0b00, + Better, + Good, + Degraded, +} +impl Ord for RelativePerformance { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // higher performance is better, hence reversed + Ord::cmp(&(*self as u8), &(*other as u8)).reverse() + } +} +impl PartialOrd for RelativePerformance { + fn partial_cmp(&self, other: &Self) -> Option { + Some(Ord::cmp(self, other)) + } +} + +impl LbaFormat { + pub fn relative_performance(&self) -> RelativePerformance { + match ((self.0 >> 24) & 0b11) { + 0b00 => RelativePerformance::Best, + 0b01 => RelativePerformance::Better, + 0b10 => RelativePerformance::Good, + 0b11 => RelativePerformance::Degraded, + _ => unreachable!(), + } + } + pub fn is_available(&self) -> bool { + self.log_lba_data_size() != 0 + } + pub fn log_lba_data_size(&self) -> u8 { + ((self.0 >> 16) & 0xFF) as u8 + } + pub fn lba_data_size(&self) -> Option { + if self.log_lba_data_size() < 9 { return None } + if self.log_lba_data_size() >= 32 { return None } + Some(1u64 << self.log_lba_data_size()) + } + pub fn metadata_size(&self) -> u16 { + (self.0 & 0xFFFF) as u16 + } +} + +impl Nvme { + /// Returns the serial number, model, and firmware, in that order. + pub async fn identify_controller(&self) { + // TODO: Use same buffer + let data: Dma = Dma::zeroed().unwrap(); + + // println!(" - Attempting to identify controller"); + let comp = self + .submit_and_complete_admin_command(|cid| NvmeCmd::identify_controller(cid, data.physical())) + .await; + log::trace!("Completion: {:?}", comp); + + // println!(" - Dumping identify controller"); + + let model_cow = String::from_utf8_lossy(&data.model_no); + let serial_cow = String::from_utf8_lossy(&data.serial_no); + let fw_cow = String::from_utf8_lossy(&data.firmware_rev); + + let model = model_cow.trim(); + let serial = serial_cow.trim(); + let firmware = fw_cow.trim(); + + println!( + " - Model: {} Serial: {} Firmware: {}", + model, serial, firmware, + ); + } + pub async fn identify_namespace_list(&self, base: u32) -> Vec { + // TODO: Use buffer + let data: Dma<[u32; 1024]> = Dma::zeroed().unwrap(); + + // println!(" - Attempting to retrieve namespace ID list"); + let comp = self + .submit_and_complete_admin_command(|cid| { + NvmeCmd::identify_namespace_list(cid, data.physical(), base) + }) + .await; + + log::trace!("Completion2: {:?}", comp); + + // println!(" - Dumping namespace ID list"); + data.iter().copied().take_while(|&nsid| nsid != 0).collect() + } + pub async fn identify_namespace(&self, nsid: u32) -> NvmeNamespace { + //TODO: Use buffer + let data: Dma = Dma::zeroed().unwrap(); + + // println!(" - Attempting to identify namespace {}", nsid); + let comp = self + .submit_and_complete_admin_command(|cid| NvmeCmd::identify_namespace(cid, data.physical(), nsid)) + .await; + + // println!(" - Dumping identify namespace"); + + let size = data.size_in_blocks(); + let capacity = data.capacity_in_blocks(); + log::info!("NSID: {} Size: {} Capacity: {}", nsid, size, capacity); + + let block_size = data.formatted_lba_size().lba_data_size().expect("nvmed: error: size outside 512-2^64 range"); + log::debug!("NVME block size: {}", block_size); + + NvmeNamespace { + id: nsid, + blocks: size, + block_size, + } + } +} diff --git a/nvmed/src/nvme/mod.rs b/nvmed/src/nvme/mod.rs new file mode 100644 index 0000000000..f5af0f0c37 --- /dev/null +++ b/nvmed/src/nvme/mod.rs @@ -0,0 +1,582 @@ +use std::collections::BTreeMap; +use std::convert::TryFrom; +use std::fs::File; +use std::ptr; +use std::sync::atomic::{AtomicU16, AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Mutex, RwLock}; + +use crossbeam_channel::Sender; +use smallvec::{smallvec, SmallVec}; + +use syscall::error::{Error, Result, EINVAL}; +use syscall::io::{Dma, Io, Mmio}; + +pub mod cmd; +pub mod cq_reactor; +pub mod identify; +pub mod queues; + +use self::cq_reactor::NotifReq; +pub use self::queues::{NvmeCmd, NvmeCmdQueue, NvmeComp, NvmeCompQueue}; + +use pcid_interface::msi::{MsiCapability, MsixCapability, MsixTableEntry}; +use pcid_interface::PcidServerHandle; + +/// Used in conjunction with `InterruptMethod`, primarily by the CQ reactor. +#[derive(Debug)] +pub enum InterruptSources { + MsiX(BTreeMap), + Msi(BTreeMap), + Intx(File), +} +impl InterruptSources { + pub fn iter_mut(&mut self) -> impl Iterator { + use std::collections::btree_map::IterMut as BTreeIterMut; + use std::iter::Once; + + enum IterMut<'a> { + Msi(BTreeIterMut<'a, u8, File>), + MsiX(BTreeIterMut<'a, u16, File>), + Intx(Once<&'a mut File>), + } + impl<'a> Iterator for IterMut<'a> { + type Item = (u16, &'a mut File); + + fn next(&mut self) -> Option { + match self { + &mut Self::Msi(ref mut iter) => iter + .next() + .map(|(&vector, handle)| (u16::from(vector), handle)), + &mut Self::MsiX(ref mut iter) => { + iter.next().map(|(&vector, handle)| (vector, handle)) + } + &mut Self::Intx(ref mut iter) => iter.next().map(|handle| (0u16, handle)), + } + } + fn size_hint(&self) -> (usize, Option) { + match self { + &Self::Msi(ref iter) => iter.size_hint(), + &Self::MsiX(ref iter) => iter.size_hint(), + &Self::Intx(ref iter) => iter.size_hint(), + } + } + } + + match self { + &mut Self::MsiX(ref mut map) => IterMut::MsiX(map.iter_mut()), + &mut Self::Msi(ref mut map) => IterMut::Msi(map.iter_mut()), + &mut Self::Intx(ref mut single) => IterMut::Intx(std::iter::once(single)), + } + } +} + +/// The way interrupts are sent. Unlike other PCI-based interfaces, like XHCI, it doesn't seem like +/// NVME supports operating with interrupts completely disabled. +pub enum InterruptMethod { + /// Traditional level-triggered, INTx# interrupt pins. + Intx, + /// Message signaled interrupts + Msi(MsiCapability), + /// Extended message signaled interrupts + MsiX(MsixCfg), +} +impl InterruptMethod { + fn is_intx(&self) -> bool { + if let Self::Intx = self { + true + } else { + false + } + } + fn is_msi(&self) -> bool { + if let Self::Msi(_) = self { + true + } else { + false + } + } + fn is_msix(&self) -> bool { + if let Self::MsiX(_) = self { + true + } else { + false + } + } +} + +pub struct MsixCfg { + pub cap: MsixCapability, + pub table: &'static mut [MsixTableEntry], + pub pba: &'static mut [Mmio], +} + +#[repr(packed)] +pub struct NvmeRegs { + /// Controller Capabilities + cap: Mmio, + /// Version + vs: Mmio, + /// Interrupt mask set + intms: Mmio, + /// Interrupt mask clear + intmc: Mmio, + /// Controller configuration + cc: Mmio, + /// Reserved + _rsvd: Mmio, + /// Controller status + csts: Mmio, + /// NVM subsystem reset + nssr: Mmio, + /// Admin queue attributes + aqa: Mmio, + /// Admin submission queue base address + asq: Mmio, + /// Admin completion queue base address + acq: Mmio, + /// Controller memory buffer location + cmbloc: Mmio, + /// Controller memory buffer size + cmbsz: Mmio, +} + +#[derive(Debug)] +pub struct NvmeNamespace { + pub id: u32, + pub blocks: u64, + pub block_size: u64, +} + +pub type CqId = u16; +pub type SqId = u16; +pub type CmdId = u16; +pub type AtomicCqId = AtomicU16; +pub type AtomicSqId = AtomicU16; +pub type AtomicCmdId = AtomicU16; + +pub struct Nvme { + interrupt_method: Mutex, + pcid_interface: Mutex, + regs: RwLock<&'static mut NvmeRegs>, + + pub(crate) submission_queues: RwLock, CqId)>>, + pub(crate) completion_queues: + RwLock)>>>, + + // maps interrupt vectors with the completion queues they have + cqs_for_ivs: RwLock>>, + + buffer: Mutex>, // 2MB of buffer + buffer_prp: Mutex>, // 4KB of PRP for the buffer + reactor_sender: Sender, + + next_sqid: AtomicSqId, + next_cqid: AtomicCqId, + + next_avail_submission_epoch: AtomicU64, +} +unsafe impl Send for Nvme {} +unsafe impl Sync for Nvme {} + +/// How to handle full submission queues. +pub enum FullSqHandling { + /// Return an error immediately prior to posting the command. + ErrorDirectly, + + /// Tell the IRQ reactor that we want to be notified when a command on the same submission + /// queue has been completed. + Wait, +} + +impl Nvme { + pub fn new( + address: usize, + interrupt_method: InterruptMethod, + pcid_interface: PcidServerHandle, + reactor_sender: Sender, + ) -> Result { + Ok(Nvme { + regs: RwLock::new(unsafe { &mut *(address as *mut NvmeRegs) }), + submission_queues: RwLock::new( + std::iter::once((0u16, (Mutex::new(NvmeCmdQueue::new()?), 0u16))).collect(), + ), + completion_queues: RwLock::new( + std::iter::once((0u16, Mutex::new((NvmeCompQueue::new()?, smallvec!(0))))).collect(), + ), + // map the zero interrupt vector (which according to the spec shall always point to the + // admin completion queue) to CQID 0 (admin completion queue) + cqs_for_ivs: RwLock::new(std::iter::once((0, smallvec!(0))).collect()), + buffer: Mutex::new(Dma::zeroed()?), + buffer_prp: Mutex::new(Dma::zeroed()?), + interrupt_method: Mutex::new(interrupt_method), + pcid_interface: Mutex::new(pcid_interface), + reactor_sender, + + next_sqid: AtomicSqId::new(0), + next_cqid: AtomicCqId::new(0), + next_avail_submission_epoch: AtomicU64::new(0), + }) + } + /// Write to a doorbell register. + /// + /// # Locking + /// Locks `regs`. + unsafe fn doorbell_write(&self, index: usize, value: u32) { + use std::ops::DerefMut; + + let mut regs_guard = self.regs.write().unwrap(); + let mut regs: &mut NvmeRegs = regs_guard.deref_mut(); + + let dstrd = ((regs.cap.read() >> 32) & 0b1111) as usize; + let addr = (regs as *mut NvmeRegs as usize) + 0x1000 + index * (4 << dstrd); + (&mut *(addr as *mut Mmio)).write(value); + } + + pub unsafe fn submission_queue_tail(&self, qid: u16, tail: u16) { + self.doorbell_write(2 * (qid as usize), u32::from(tail)); + } + + pub unsafe fn completion_queue_head(&self, qid: u16, head: u16) { + self.doorbell_write(2 * (qid as usize) + 1, u32::from(head)); + } + + pub unsafe fn init(&mut self) { + let mut buffer = self.buffer.get_mut().unwrap(); + let mut buffer_prp = self.buffer_prp.get_mut().unwrap(); + + for i in 0..buffer_prp.len() { + buffer_prp[i] = (buffer.physical() + i * 4096) as u64; + } + + { + let regs = self.regs.read().unwrap(); + log::debug!("CAPS: {:X}", regs.cap.read()); + log::debug!("VS: {:X}", regs.vs.read()); + log::debug!("CC: {:X}", regs.cc.read()); + log::debug!("CSTS: {:X}", regs.csts.read()); + } + + log::debug!("Disabling controller."); + self.regs.get_mut().unwrap().cc.writef(1, false); + + log::trace!("Waiting for not ready."); + loop { + let csts = self.regs.get_mut().unwrap().csts.read(); + log::trace!("CSTS: {:X}", csts); + if csts & 1 == 1 { + std::arch::x86_64::_mm_pause(); + } else { + break; + } + } + + match self.interrupt_method.get_mut().unwrap() { + &mut InterruptMethod::Intx | InterruptMethod::Msi(_) => { + self.regs.get_mut().unwrap().intms.write(0xFFFF_FFFF); + self.regs.get_mut().unwrap().intmc.write(0x0000_0001); + } + &mut InterruptMethod::MsiX(ref mut cfg) => { + cfg.table[0].unmask(); + } + } + + for (qid, queue) in self.completion_queues.get_mut().unwrap().iter_mut() { + let &(ref cq, ref sq_ids) = &*queue.get_mut().unwrap(); + let data = &cq.data; + log::debug!("completion queue {}: {:X}, {}, (submission queue ids: {:?}", qid, data.physical(), data.len(), sq_ids); + } + + for (qid, (queue, cq_id)) in self.submission_queues.get_mut().unwrap().iter_mut() { + let data = &queue.get_mut().unwrap().data; + log::debug!("submission queue {}: {:X}, {}, attached to CQID: {}", qid, data.physical(), data.len(), cq_id); + } + + { + let regs = self.regs.get_mut().unwrap(); + let submission_queues = self.submission_queues.get_mut().unwrap(); + let completion_queues = self.completion_queues.get_mut().unwrap(); + + let asq = submission_queues.get_mut(&0).unwrap().0.get_mut().unwrap(); + let (acq, _) = completion_queues.get_mut(&0).unwrap().get_mut().unwrap(); + regs.aqa + .write(((acq.data.len() as u32 - 1) << 16) | (asq.data.len() as u32 - 1)); + regs.asq.write(asq.data.physical() as u64); + regs.acq.write(acq.data.physical() as u64); + + // Set IOCQES, IOSQES, AMS, MPS, and CSS + let mut cc = regs.cc.read(); + cc &= 0xFF00000F; + cc |= (4 << 20) | (6 << 16); + regs.cc.write(cc); + } + + log::debug!("Enabling controller."); + self.regs.get_mut().unwrap().cc.writef(1, true); + + log::debug!("Waiting for ready"); + loop { + let csts = self.regs.get_mut().unwrap().csts.read(); + log::debug!("CSTS: {:X}", csts); + if csts & 1 == 0 { + std::arch::x86_64::_mm_pause(); + } else { + break; + } + } + } + + /// Masks or unmasks multiple vectors. + /// + /// # Panics + /// Will panic if the same vector is called twice with different mask flags. + pub fn set_vectors_masked(&self, vectors: impl IntoIterator) { + let mut interrupt_method_guard = self.interrupt_method.lock().unwrap(); + + match &mut *interrupt_method_guard { + &mut InterruptMethod::Intx => { + let mut iter = vectors.into_iter(); + let (vector, mask) = match iter.next() { + Some(f) => f, + None => return, + }; + assert_eq!( + iter.next(), + None, + "nvmed: internal error: multiple vectors on INTx#" + ); + assert_eq!(vector, 0, "nvmed: internal error: nonzero vector on INTx#"); + if mask { + self.regs.write().unwrap().intms.write(0x0000_0001); + } else { + self.regs.write().unwrap().intmc.write(0x0000_0001); + } + } + &mut InterruptMethod::Msi(ref mut cap) => { + let mut to_mask = 0x0000_0000; + let mut to_clear = 0x0000_0000; + + for (vector, mask) in vectors { + assert!( + vector < (1 << cap.multi_message_enable()), + "nvmed: internal error: MSI vector out of range" + ); + let vector = vector as u8; + + if mask { + assert_ne!( + to_clear & (1 << vector), + (1 << vector), + "nvmed: internal error: cannot both mask and set" + ); + to_mask |= 1 << vector; + } else { + assert_ne!( + to_mask & (1 << vector), + (1 << vector), + "nvmed: internal error: cannot both mask and set" + ); + to_clear |= 1 << vector; + } + } + + if to_mask != 0 { + self.regs.write().unwrap().intms.write(to_mask); + } + if to_clear != 0 { + self.regs.write().unwrap().intmc.write(to_clear); + } + } + &mut InterruptMethod::MsiX(ref mut cfg) => { + for (vector, mask) in vectors { + cfg.table + .get_mut(vector as usize) + .expect("nvmed: internal error: MSI-X vector out of range") + .set_masked(mask); + } + } + } + } + pub fn set_vector_masked(&self, vector: u16, masked: bool) { + self.set_vectors_masked(std::iter::once((vector, masked))) + } + + pub async fn submit_and_complete_admin_command NvmeCmd>(&self, cmd_init: F) -> NvmeComp { + self.submit_and_complete_command(0, cmd_init).await + } + + pub async fn create_io_completion_queue(&self, io_cq_id: CqId, vector: Option) { + let (ptr, len) = { + let mut completion_queues_guard = self.completion_queues.write().unwrap(); + + let queue_guard = completion_queues_guard + .entry(io_cq_id) + .or_insert_with(|| { + let queue = NvmeCompQueue::new() + .expect("nvmed: failed to allocate I/O completion queue"); + let sqs = SmallVec::new(); + Mutex::new((queue, sqs)) + }) + .get_mut() + .unwrap(); + + let &(ref queue, _) = &*queue_guard; + (queue.data.physical(), queue.data.len()) + }; + + let len = + u16::try_from(len).expect("nvmed: internal error: I/O CQ longer than 2^16 entries"); + let raw_len = len + .checked_sub(1) + .expect("nvmed: internal error: CQID 0 for I/O CQ"); + + let comp = self + .submit_and_complete_admin_command(|cid| { + NvmeCmd::create_io_completion_queue(cid, io_cq_id, ptr, raw_len, vector) + }) + .await; + + if let Some(vector) = vector { + self.cqs_for_ivs + .write() + .unwrap() + .entry(vector) + .or_insert_with(SmallVec::new) + .push(io_cq_id); + } + } + pub async fn create_io_submission_queue(&self, io_sq_id: SqId, io_cq_id: CqId) { + let (ptr, len) = { + let mut submission_queues_guard = self.submission_queues.write().unwrap(); + + let (queue_lock, _) = submission_queues_guard + .entry(io_sq_id) + .or_insert_with(|| { + (Mutex::new( + NvmeCmdQueue::new() + .expect("nvmed: failed to allocate I/O completion queue"), + ), io_cq_id) + }); + let queue = queue_lock.get_mut().unwrap(); + + (queue.data.physical(), queue.data.len()) + }; + + let len = + u16::try_from(len).expect("nvmed: internal error: I/O SQ longer than 2^16 entries"); + let raw_len = len + .checked_sub(1) + .expect("nvmed: internal error: SQID 0 for I/O SQ"); + + let comp = self + .submit_and_complete_admin_command(|cid| { + NvmeCmd::create_io_submission_queue(cid, io_sq_id, ptr, raw_len, io_cq_id) + }) + .await; + } + + pub async fn init_with_queues(&self) -> BTreeMap { + log::trace!("preinit"); + let ((), nsids) = + futures::join!(self.identify_controller(), self.identify_namespace_list(0)); + log::debug!("first commands"); + + let mut namespaces = BTreeMap::new(); + + for nsid in nsids.iter().copied() { + namespaces.insert(nsid, self.identify_namespace(nsid).await); + } + + // TODO: Multiple queues + self.create_io_completion_queue(1, Some(0)).await; + self.create_io_submission_queue(1, 1).await; + + namespaces + } + + async fn namespace_rw( + &self, + namespace: &NvmeNamespace, + nsid: u32, + lba: u64, + blocks_1: u16, + write: bool, + ) -> Result<()> { + let block_size = namespace.block_size; + + let buffer_prp_guard = self.buffer_prp.lock().unwrap(); + + let bytes = ((blocks_1 as u64) + 1) * block_size; + let (ptr0, ptr1) = if bytes <= 4096 { + (buffer_prp_guard[0], 0) + } else if bytes <= 8192 { + (buffer_prp_guard[0], buffer_prp_guard[1]) + } else { + (buffer_prp_guard[0], (buffer_prp_guard.physical() + 8) as u64) + }; + + let comp = self.submit_and_complete_command(1, |cid| { + if write { + NvmeCmd::io_write(cid, nsid, lba, blocks_1, ptr0, ptr1) + } else { + NvmeCmd::io_read(cid, nsid, lba, blocks_1, ptr0, ptr1) + } + }).await; + // TODO: Handle errors + + Ok(()) + } + + pub async fn namespace_read( + &self, + namespace: &NvmeNamespace, + nsid: u32, + mut lba: u64, + buf: &mut [u8], + ) -> Result> { + let block_size = namespace.block_size as usize; + + let buffer_guard = self.buffer.lock().unwrap(); + + for chunk in buf.chunks_mut(buffer_guard.len()) { + let blocks = (chunk.len() + block_size - 1) / block_size; + + assert!(blocks > 0); + assert!(blocks <= 0x1_0000); + + self.namespace_rw(namespace, nsid, lba, (blocks - 1) as u16, false).await?; + + chunk.copy_from_slice(&buffer_guard[..chunk.len()]); + + lba += blocks as u64; + } + + Ok(Some(buf.len())) + } + + pub async fn namespace_write( + &self, + namespace: &NvmeNamespace, + nsid: u32, + mut lba: u64, + buf: &[u8], + ) -> Result> { + let block_size = namespace.block_size as usize; + + let mut buffer_guard = self.buffer.lock().unwrap(); + + for chunk in buf.chunks(buffer_guard.len()) { + let blocks = (chunk.len() + block_size - 1) / block_size; + + assert!(blocks > 0); + assert!(blocks <= 0x1_0000); + + buffer_guard[..chunk.len()].copy_from_slice(chunk); + + self.namespace_rw(namespace, nsid, lba, (blocks - 1) as u16, true).await?; + + lba += blocks as u64; + } + + Ok(Some(buf.len())) + } +} diff --git a/nvmed/src/nvme/queues.rs b/nvmed/src/nvme/queues.rs new file mode 100644 index 0000000000..bfc68b1d9f --- /dev/null +++ b/nvmed/src/nvme/queues.rs @@ -0,0 +1,120 @@ +use std::ptr; +use syscall::{Dma, Result}; + +/// A submission queue entry. +#[derive(Clone, Copy, Debug, Default)] +#[repr(packed)] +pub struct NvmeCmd { + /// Opcode + pub opcode: u8, + /// Flags + pub flags: u8, + /// Command ID + pub cid: u16, + /// Namespace identifier + pub nsid: u32, + /// Reserved + pub _rsvd: u64, + /// Metadata pointer + pub mptr: u64, + /// Data pointer + pub dptr: [u64; 2], + /// Command dword 10 + pub cdw10: u32, + /// Command dword 11 + pub cdw11: u32, + /// Command dword 12 + pub cdw12: u32, + /// Command dword 13 + pub cdw13: u32, + /// Command dword 14 + pub cdw14: u32, + /// Command dword 15 + pub cdw15: u32, +} + +/// A completion queue entry. +#[derive(Clone, Copy, Debug)] +#[repr(packed)] +pub struct NvmeComp { + pub command_specific: u32, + pub _rsvd: u32, + pub sq_head: u16, + pub sq_id: u16, + pub cid: u16, + pub status: u16, +} + +/// Completion queue +pub struct NvmeCompQueue { + pub data: Dma<[NvmeComp]>, + pub head: u16, + pub phase: bool, +} + +impl NvmeCompQueue { + pub fn new() -> Result { + Ok(Self { + data: unsafe { Dma::zeroed_unsized(256)? }, + head: 0, + phase: true, + }) + } + + /// Get a new completion queue entry, or return None if no entry is available yet. + pub(crate) fn complete(&mut self) -> Option<(u16, NvmeComp)> { + let entry = unsafe { ptr::read_volatile(self.data.as_ptr().add(self.head as usize)) }; + if ((entry.status & 1) == 1) == self.phase { + self.head = (self.head + 1) % (self.data.len() as u16); + if self.head == 0 { + self.phase = !self.phase; + } + Some((self.head, entry)) + } else { + None + } + } + + /// Get a new CQ entry, busy waiting until an entry appears. + fn complete_spin(&mut self) -> (u16, NvmeComp) { + loop { + if let Some(some) = self.complete() { + return some; + } else { + unsafe { std::arch::x86_64::_mm_pause() } + } + } + } +} + +/// Submission queue +pub struct NvmeCmdQueue { + pub data: Dma<[NvmeCmd]>, + pub tail: u16, + pub head: u16, +} + +impl NvmeCmdQueue { + pub fn new() -> Result { + Ok(Self { + data: unsafe { Dma::zeroed_unsized(64)? }, + tail: 0, + head: 0, + }) + } + + pub fn is_empty(&self) -> bool { + self.head == self.tail + } + pub fn is_full(&self) -> bool { + self.head == self.tail + 1 + } + + /// Add a new submission command entry to the queue. The caller must ensure that the queue have free + /// entries; this can be checked using `is_full`. + pub fn submit_unchecked(&mut self, entry: NvmeCmd) -> u16 { + unsafe { ptr::write_volatile(&mut self.data[self.tail as usize] as *mut _, entry) } + self.tail = (self.tail + 1) % (self.data.len() as u16); + self.tail + } +} diff --git a/nvmed/src/scheme.rs b/nvmed/src/scheme.rs index 108518a3e6..aabec0389e 100644 --- a/nvmed/src/scheme.rs +++ b/nvmed/src/scheme.rs @@ -1,13 +1,15 @@ use std::collections::BTreeMap; -use std::{cmp, str}; use std::convert::{TryFrom, TryInto}; use std::fmt::Write; -use std::io::prelude::*; use std::io; +use std::io::prelude::*; +use std::sync::Arc; +use std::{cmp, str}; + use syscall::{ - Error, EACCES, EBADF, EINVAL, EISDIR, ENOENT, EOVERFLOW, Result, - Io, SchemeBlockMut, Stat, MODE_DIR, MODE_FILE, O_DIRECTORY, - O_STAT, SEEK_CUR, SEEK_END, SEEK_SET}; + Error, Io, Result, SchemeBlockMut, Stat, EACCES, EBADF, EINVAL, EISDIR, ENOENT, EOVERFLOW, + MODE_DIR, MODE_FILE, O_DIRECTORY, O_STAT, SEEK_CUR, SEEK_END, SEEK_SET, +}; use crate::nvme::{Nvme, NvmeNamespace}; @@ -15,8 +17,8 @@ use partitionlib::{LogicalBlockSize, PartitionTable}; #[derive(Clone)] enum Handle { - List(Vec, usize), // entries, offset - Disk(u32, usize), // disk num, offset + List(Vec, usize), // entries, offset + Disk(u32, usize), // disk num, offset Partition(u32, u32, usize), // disk num, part num, offset } @@ -32,23 +34,35 @@ impl AsRef for DiskWrapper { } impl DiskWrapper { - fn pt(disk: &mut NvmeNamespace, nvme: &mut Nvme) -> Option { + fn pt(disk: &mut NvmeNamespace, nvme: &Nvme) -> Option { let bs = match disk.block_size { 512 => LogicalBlockSize::Lb512, 4096 => LogicalBlockSize::Lb4096, _ => return None, }; - struct Device<'a, 'b> { disk: &'a mut NvmeNamespace, nvme: &'a mut Nvme, offset: u64, block_bytes: &'b mut [u8] } + struct Device<'a, 'b> { + disk: &'a mut NvmeNamespace, + nvme: &'a Nvme, + offset: u64, + block_bytes: &'b mut [u8], + } impl<'a, 'b> Seek for Device<'a, 'b> { fn seek(&mut self, from: io::SeekFrom) -> io::Result { let size_u = self.disk.blocks * self.disk.block_size; - let size = i64::try_from(size_u).or(Err(io::Error::new(io::ErrorKind::Other, "Disk larger than 2^63 - 1 bytes")))?; + let size = i64::try_from(size_u).or(Err(io::Error::new( + io::ErrorKind::Other, + "Disk larger than 2^63 - 1 bytes", + )))?; self.offset = match from { io::SeekFrom::Start(new_pos) => cmp::min(size_u, new_pos), - io::SeekFrom::Current(new_pos) => cmp::max(0, cmp::min(size, self.offset as i64 + new_pos)) as u64, - io::SeekFrom::End(new_pos) => cmp::max(0, cmp::min(size + new_pos, size)) as u64, + io::SeekFrom::Current(new_pos) => { + cmp::max(0, cmp::min(size, self.offset as i64 + new_pos)) as u64 + } + io::SeekFrom::End(new_pos) => { + cmp::max(0, cmp::min(size + new_pos, size)) as u64 + } }; Ok(self.offset) @@ -68,20 +82,29 @@ impl DiskWrapper { return Err(io::Error::from_raw_os_error(syscall::EOVERFLOW)); } loop { - match unsafe { - nvme.namespace_read(disk.id, block, block_bytes).map_err(|err| io::Error::from_raw_os_error(err.errno))? - } { + match futures::executor::block_on(nvme.namespace_read(disk, disk.id, block, block_bytes)) + .map_err(|err| io::Error::from_raw_os_error(err.errno))? { Some(bytes) => { assert_eq!(bytes, block_bytes.len()); assert_eq!(bytes, blksize as usize); return Ok(()); } - None => { std::thread::yield_now(); continue } - // TODO: Does this driver have (internal) error handling at all? + None => { + std::thread::yield_now(); + continue; + } // TODO: Does this driver have (internal) error handling at all? } } }; - let bytes_read = block_io_wrapper::read(self.offset, blksize.try_into().expect("Unreasonable block size above 2^32 bytes"), buf, self.block_bytes, read_block)?; + let bytes_read = block_io_wrapper::read( + self.offset, + blksize + .try_into() + .expect("Unreasonable block size above 2^32 bytes"), + buf, + self.block_bytes, + read_block, + )?; self.offset += bytes_read as u64; Ok(bytes_read) } @@ -89,9 +112,19 @@ impl DiskWrapper { let mut block_bytes = [0u8; 4096]; - partitionlib::get_partitions(&mut Device { disk, nvme, offset: 0, block_bytes: &mut block_bytes[..bs.into()] }, bs).ok().flatten() + partitionlib::get_partitions( + &mut Device { + disk, + nvme, + offset: 0, + block_bytes: &mut block_bytes[..bs.into()], + }, + bs, + ) + .ok() + .flatten() } - fn new(mut inner: NvmeNamespace, nvme: &mut Nvme) -> Self { + fn new(mut inner: NvmeNamespace, nvme: &Nvme) -> Self { Self { pt: Self::pt(&mut inner, nvme), inner, @@ -101,46 +134,37 @@ impl DiskWrapper { pub struct DiskScheme { scheme_name: String, - nvme: Nvme, + nvme: Arc, disks: BTreeMap, handles: BTreeMap, - next_id: usize + next_id: usize, } impl DiskScheme { - pub fn new(scheme_name: String, mut nvme: Nvme, disks: BTreeMap) -> DiskScheme { + pub fn new( + scheme_name: String, + nvme: Arc, + disks: BTreeMap, + ) -> DiskScheme { DiskScheme { scheme_name, - disks: disks.into_iter().map(|(k, v)| (k, DiskWrapper::new(v, &mut nvme))).collect(), + disks: disks + .into_iter() + .map(|(k, v)| (k, DiskWrapper::new(v, &nvme))) + .collect(), nvme, handles: BTreeMap::new(), - next_id: 0 + next_id: 0, } } } -impl DiskScheme { - pub fn irq(&mut self) -> bool { - let mut found_completion = false; - - let nvme = &mut self.nvme; - for qid in 0..nvme.completion_queues.len() { - while let Some((head, entry)) = nvme.completion_queues[qid].complete() { - found_completion = true; - println!("nvmed: Unhandled completion {:?}", entry); - //TODO: Handle errors - unsafe { nvme.completion_queue_head(qid as u16, head as u16); } - } - } - - found_completion - } -} - impl SchemeBlockMut for DiskScheme { fn open(&mut self, path: &[u8], flags: usize, uid: u32, _gid: u32) -> Result> { if uid == 0 { - let path_str = str::from_utf8(path).or(Err(Error::new(ENOENT)))?.trim_matches('/'); + let path_str = str::from_utf8(path) + .or(Err(Error::new(ENOENT)))? + .trim_matches('/'); if path_str.is_empty() { if flags & O_DIRECTORY == O_DIRECTORY || flags & O_STAT == O_STAT { let mut list = String::new(); @@ -175,10 +199,18 @@ impl SchemeBlockMut for DiskScheme { let part_num = part_num_str.parse::().or(Err(Error::new(ENOENT)))?; if let Some(disk) = self.disks.get(&nsid) { - if disk.pt.as_ref().ok_or(Error::new(ENOENT))?.partitions.get(part_num as usize).is_some() { + if disk + .pt + .as_ref() + .ok_or(Error::new(ENOENT))? + .partitions + .get(part_num as usize) + .is_some() + { let id = self.next_id; self.next_id += 1; - self.handles.insert(id, Handle::Partition(nsid, part_num, 0)); + self.handles + .insert(id, Handle::Partition(nsid, part_num, 0)); Ok(Some(id)) } else { Err(Error::new(ENOENT)) @@ -204,7 +236,7 @@ impl SchemeBlockMut for DiskScheme { } fn dup(&mut self, id: usize, buf: &[u8]) -> Result> { - if ! buf.is_empty() { + if !buf.is_empty() { return Err(Error::new(EINVAL)); } @@ -225,22 +257,36 @@ impl SchemeBlockMut for DiskScheme { stat.st_mode = MODE_DIR; stat.st_size = data.len() as u64; Ok(Some(0)) - }, + } Handle::Disk(number, _) => { let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?; stat.st_mode = MODE_FILE; stat.st_blocks = disk.as_ref().blocks; - stat.st_blksize = disk.as_ref().block_size.try_into().expect("Unreasonable block size of over 2^32 bytes"); + stat.st_blksize = disk + .as_ref() + .block_size + .try_into() + .expect("Unreasonable block size of over 2^32 bytes"); stat.st_size = disk.as_ref().blocks * disk.as_ref().block_size; Ok(Some(0)) } Handle::Partition(disk_num, part_num, _) => { let disk = self.disks.get_mut(&disk_num).ok_or(Error::new(EBADF))?; - let part = disk.pt.as_ref().ok_or(Error::new(EBADF))?.partitions.get(part_num as usize).ok_or(Error::new(EBADF))?; + let part = disk + .pt + .as_ref() + .ok_or(Error::new(EBADF))? + .partitions + .get(part_num as usize) + .ok_or(Error::new(EBADF))?; stat.st_mode = MODE_FILE; stat.st_size = part.size * disk.as_ref().block_size; stat.st_blocks = part.size; - stat.st_blksize = disk.as_ref().block_size.try_into().expect("Unreasonable block size of over 2^32 bytes"); + stat.st_blksize = disk + .as_ref() + .block_size + .try_into() + .expect("Unreasonable block size of over 2^32 bytes"); Ok(Some(0)) } } @@ -301,9 +347,7 @@ impl SchemeBlockMut for DiskScheme { Handle::Disk(number, ref mut size) => { let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; - if let Some(count) = unsafe { - self.nvme.namespace_read(disk.as_ref().id, (*size as u64)/block_size, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref(), disk.as_ref().id, (*size as u64) / block_size, buf))? { *size += count; Ok(Some(count)) } else { @@ -312,7 +356,13 @@ impl SchemeBlockMut for DiskScheme { } Handle::Partition(disk_num, part_num, ref mut offset) => { let disk = self.disks.get_mut(&disk_num).ok_or(Error::new(EBADF))?; - let part = disk.pt.as_ref().ok_or(Error::new(EBADF))?.partitions.get(part_num as usize).ok_or(Error::new(EBADF))?; + let part = disk + .pt + .as_ref() + .ok_or(Error::new(EBADF))? + .partitions + .get(part_num as usize) + .ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; let rel_block = (*offset as u64) / block_size; @@ -322,9 +372,7 @@ impl SchemeBlockMut for DiskScheme { let abs_block = part.start_lba + rel_block; - if let Some(count) = unsafe { - self.nvme.namespace_read(disk.as_ref().id, abs_block, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_read(disk.as_ref(), disk.as_ref().id, abs_block, buf))? { *offset += count; Ok(Some(count)) } else { @@ -336,15 +384,12 @@ impl SchemeBlockMut for DiskScheme { fn write(&mut self, id: usize, buf: &[u8]) -> Result> { match *self.handles.get_mut(&id).ok_or(Error::new(EBADF))? { - Handle::List(_, _) => { - Err(Error::new(EBADF)) - }, + Handle::List(_, _) => Err(Error::new(EBADF)), Handle::Disk(number, ref mut size) => { let disk = self.disks.get_mut(&number).ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; - if let Some(count) = unsafe { - self.nvme.namespace_write(disk.as_ref().id, (*size as u64)/block_size, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme + .namespace_write(disk.as_ref(), disk.as_ref().id, (*size as u64) / block_size, buf))? { *size += count; Ok(Some(count)) } else { @@ -353,7 +398,13 @@ impl SchemeBlockMut for DiskScheme { } Handle::Partition(disk_num, part_num, ref mut offset) => { let disk = self.disks.get_mut(&disk_num).ok_or(Error::new(EBADF))?; - let part = disk.pt.as_ref().ok_or(Error::new(EBADF))?.partitions.get(part_num as usize).ok_or(Error::new(EBADF))?; + let part = disk + .pt + .as_ref() + .ok_or(Error::new(EBADF))? + .partitions + .get(part_num as usize) + .ok_or(Error::new(EBADF))?; let block_size = disk.as_ref().block_size; let rel_block = (*offset as u64) / block_size; @@ -363,9 +414,7 @@ impl SchemeBlockMut for DiskScheme { let abs_block = part.start_lba + rel_block; - if let Some(count) = unsafe { - self.nvme.namespace_write(disk.as_ref().id, abs_block, buf)? - } { + if let Some(count) = futures::executor::block_on(self.nvme.namespace_write(disk.as_ref(), disk.as_ref().id, abs_block, buf))? { *offset += count; Ok(Some(count)) } else { @@ -381,9 +430,13 @@ impl SchemeBlockMut for DiskScheme { let len = handle.len() as usize; *size = match whence { SEEK_SET => cmp::min(len, pos), - SEEK_CUR => cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize, - SEEK_END => cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize, - _ => return Err(Error::new(EINVAL)) + SEEK_CUR => { + cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize + } + SEEK_END => { + cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize + } + _ => return Err(Error::new(EINVAL)), }; Ok(Some(*size)) @@ -393,24 +446,38 @@ impl SchemeBlockMut for DiskScheme { let len = (disk.as_ref().blocks * disk.as_ref().block_size) as usize; *size = match whence { SEEK_SET => cmp::min(len, pos), - SEEK_CUR => cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize, - SEEK_END => cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize, - _ => return Err(Error::new(EINVAL)) + SEEK_CUR => { + cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize + } + SEEK_END => { + cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize + } + _ => return Err(Error::new(EINVAL)), }; Ok(Some(*size)) } Handle::Partition(disk_num, part_num, ref mut size) => { let disk = self.disks.get_mut(&disk_num).ok_or(Error::new(EBADF))?; - let part = disk.pt.as_ref().ok_or(Error::new(EBADF))?.partitions.get(part_num as usize).ok_or(Error::new(EBADF))?; + let part = disk + .pt + .as_ref() + .ok_or(Error::new(EBADF))? + .partitions + .get(part_num as usize) + .ok_or(Error::new(EBADF))?; let len = (part.size * disk.as_ref().block_size) as usize; *size = match whence { SEEK_SET => cmp::min(len, pos), - SEEK_CUR => cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize, - SEEK_END => cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize, - _ => return Err(Error::new(EINVAL)) + SEEK_CUR => { + cmp::max(0, cmp::min(len as isize, *size as isize + pos as isize)) as usize + } + SEEK_END => { + cmp::max(0, cmp::min(len as isize, len as isize + pos as isize)) as usize + } + _ => return Err(Error::new(EINVAL)), }; Ok(Some(*size)) @@ -419,6 +486,9 @@ impl SchemeBlockMut for DiskScheme { } fn close(&mut self, id: usize) -> Result> { - self.handles.remove(&id).ok_or(Error::new(EBADF)).and(Ok(Some(0))) + self.handles + .remove(&id) + .ok_or(Error::new(EBADF)) + .and(Ok(Some(0))) } } diff --git a/pcid/Cargo.toml b/pcid/Cargo.toml index dae6e66cce..71c5682ed4 100644 --- a/pcid/Cargo.toml +++ b/pcid/Cargo.toml @@ -16,7 +16,9 @@ bincode = "1.2" bitflags = "1" byteorder = "1.2" libc = "0.2" +log = "0.4" plain = "0.2" +redox-log = { git = "https://gitlab.redox-os.org/redox-os/redox-log.git", tag = "v0.1.0" } redox_syscall = { git = "https://gitlab.redox-os.org/redox-os/syscall.git" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/pcid/src/config.rs b/pcid/src/config.rs index 3c79746816..c74f05842a 100644 --- a/pcid/src/config.rs +++ b/pcid/src/config.rs @@ -19,5 +19,5 @@ pub struct DriverConfig { pub device: Option, pub device_id_range: Option>, pub command: Option>, - pub channel_name: Option, + pub use_channel: Option, } diff --git a/pcid/src/driver_interface/irq_helpers.rs b/pcid/src/driver_interface/irq_helpers.rs new file mode 100644 index 0000000000..e0d63164e5 --- /dev/null +++ b/pcid/src/driver_interface/irq_helpers.rs @@ -0,0 +1,155 @@ +//! IRQ helpers. +//! +//! This module allows easy handling of the `irq:` scheme, and allocating interrupt vectors for use +//! by INTx#, MSI, or MSI-X. + +use std::convert::TryFrom; +use std::fs::{self, File}; +use std::io::{self, prelude::*}; +use std::num::NonZeroU8; + +/// Read the local APIC ID of the bootstrap processor. +pub fn read_bsp_apic_id() -> io::Result { + let mut buffer = [0u8; 8]; + + let mut file = File::open("irq:bsp")?; + let bytes_read = file.read(&mut buffer)?; + + (if bytes_read == 8 { + usize::try_from(u64::from_le_bytes(buffer)) + } else if bytes_read == 4 { + usize::try_from(u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]])) + } else { + panic!("`irq:` scheme responded with {} bytes, expected {}", bytes_read, std::mem::size_of::()); + }).or(Err(io::Error::new(io::ErrorKind::InvalidData, "bad BSP int size"))) +} + +// TODO: Perhaps read the MADT instead? +/// Obtains an interator over all of the visible CPU ids, for use in IRQ allocation and MSI +/// capability structs or MSI-X tables. +pub fn cpu_ids() -> io::Result> + 'static> { + Ok(fs::read_dir("irq:")? + .filter_map(|entry| -> Option> { match entry { + Ok(e) => { + let path = e.path(); + let file_name = path.file_name()?.to_str()?; + // the file name should be in the format `cpu-` + if ! file_name.starts_with("cpu-") { + return None; + } + u8::from_str_radix(&file_name[4..], 16).map(usize::from).map(Ok).ok() + } + Err(e) => Some(Err(e)), + } })) +} + +/// Allocate multiple interrupt vectors, from the IDT of the specified processor, returning the +/// start vector and the IRQ handles. +/// +/// The alignment is a requirement for the allocation range. For example, with an alignment of 8, +/// only ranges that begin with a multiple of eight are accepted. The IRQ handles returned will +/// always correspond to the subsequent IRQ numbers beginning the first value in the return tuple. +/// +/// This function is not actually guaranteed to allocate all of the IRQs specified in `count`, +/// since another process might already have requested one vector in the range. The caller must +/// check that the returned vector have the same length as `count`. In the future this function may +/// perhaps lock the entire directory to prevent this from happening, or maybe find the smallest free +/// range with the minimum alignment, to allow other drivers to obtain their necessary IRQs. +/// +/// Note that this count/alignment restriction is only mandatory for MSI; MSI-X allows for +/// individually allocated vectors that might be spread out, even on multiple CPUs. Thus, multiple +/// invocations with alignment 1 and count 1 are totally acceptable, although allocating in bulk +/// minimizes the initialization overhead. +pub fn allocate_aligned_interrupt_vectors(cpu_id: usize, alignment: NonZeroU8, count: u8) -> io::Result)>> { + let cpu_id = u8::try_from(cpu_id).expect("usize cpu ids not implemented yet"); + if count == 0 { return Ok(None) } + + let available_irqs = fs::read_dir(format!("irq:cpu-{:02x}", cpu_id))?; + let mut available_irq_numbers = available_irqs.filter_map(|entry| -> Option> { + let entry = match entry { + Ok(e) => e, + Err(err) => return Some(Err(err)), + }; + + let path = entry.path(); + + let file_name = match path.file_name() { + Some(f) => f, + None => return None, + }; + + let path_str = match file_name.to_str() { + Some(s) => s, + None => return None, + }; + + match path_str.parse::() { + Ok(p) => Some(Ok(p)), + Err(_) => None, + } + }); + + // TODO: fcntl F_SETLK on `irq:/`? + + let mut handles = Vec::with_capacity(usize::from(count)); + + let mut index = 0; + let mut first = None; + + while let Some(number) = available_irq_numbers.next() { + let number = number?; + + // Skip until a suitable alignment is found. + if number % u8::from(alignment) != 0 { + continue; + } + let first = *first.get_or_insert(number); + let irq_number = first + index; + + // From the point where the range is aligned, we can start to advance until `count` IRQs + // have been allocated. + if index >= count { + break; + } + + // if found, reserve the irq + let irq_handle = match File::create(format!("irq:cpu-{:02x}/{}", cpu_id, irq_number)) { + Ok(handle) => handle, + + // return early if the entire range couldn't be allocated + Err(err) if err.kind() == io::ErrorKind::NotFound => break, + + Err(err) => return Err(err), + }; + handles.push(irq_handle); + index += 1; + } + if handles.is_empty() { + return Ok(None); + } + let first = match first { + Some(f) => f, + None => return Ok(None), + }; + + Ok(Some((first + 32, handles))) +} + +/// Allocate at most `count` interrupt vectors, which can start at any offset. Unless MSI is used +/// and an entire aligned range of vectors is needed, this function should be used. +pub fn allocate_interrupt_vectors(cpu_id: usize, count: u8) -> io::Result)>> { + allocate_aligned_interrupt_vectors(cpu_id, NonZeroU8::new(1).unwrap(), count) +} + +/// Allocate a single interrupt vector, returning both the vector number (starting from 32 up to +/// 254), and its IRQ handle which is then reserved. Returns Ok(None) if allocation fails due to +/// no available IRQs. +pub fn allocate_single_interrupt_vector(cpu_id: usize) -> io::Result> { + let (base, mut files) = match allocate_interrupt_vectors(cpu_id, 1) { + Ok(Some((base, files))) => (base, files), + Ok(None) => return Ok(None), + Err(err) => return Err(err), + }; + assert_eq!(files.len(), 1); + Ok(Some((base, files.pop().unwrap()))) +} diff --git a/pcid/src/driver_interface.rs b/pcid/src/driver_interface/mod.rs similarity index 74% rename from pcid/src/driver_interface.rs rename to pcid/src/driver_interface/mod.rs index 28c06ef02a..3716574c10 100644 --- a/pcid/src/driver_interface.rs +++ b/pcid/src/driver_interface/mod.rs @@ -10,6 +10,8 @@ use thiserror::Error; pub use crate::pci::PciBar; pub use crate::pci::msi; +pub mod irq_helpers; + #[derive(Clone, Copy, Debug, Serialize, Deserialize)] #[repr(u8)] pub enum LegacyInterruptPin { @@ -40,7 +42,9 @@ pub struct PciFunction { /// BAR sizes pub bar_sizes: [u32; 6], - /// Legacy IRQ line + /// Legacy IRQ line: It's the responsibility of pcid to make sure that it be mapped in either + /// the I/O APIC or the 8259 PIC, so that the subdriver can map the interrupt vector directly. + /// The vector to map is always this field, plus 32. pub legacy_interrupt_line: u8, /// Legacy interrupt pin (INTx#), none if INTx# interrupts aren't supported at all. @@ -51,6 +55,11 @@ pub struct PciFunction { /// Device ID pub devid: u16, } +impl PciFunction { + pub fn name(&self) -> String { + format!("pci-{:>02X}.{:>02X}.{:>02X}", self.bus_num, self.dev_num, self.func_num) + } +} #[derive(Clone, Debug, Serialize, Deserialize)] pub struct SubdriverArguments { @@ -114,6 +123,48 @@ pub enum PcidClientHandleError { } pub type Result = std::result::Result; +// TODO: Remove these "features" and just go strait to the actual thing. + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct MsiSetFeatureInfo { + /// The Multi Message Enable field of the Message Control in the MSI Capability Structure, + /// is the log2 of the interrupt vectors, minus one. Can only be 0b000..=0b101. + pub multi_message_enable: Option, + + /// The system-specific message address, must be DWORD aligned. + /// + /// The message address contains things like the CPU that will be targeted, at least on + /// x86_64. + pub message_address: Option, + + /// The upper 32 bits of the 64-bit message address. Not guaranteed to exist, and is + /// reserved on x86_64 (currently). + pub message_upper_address: Option, + + /// The message data, containing the actual interrupt vector (lower 8 bits), etc. + /// + /// The spec mentions that the lower N bits can be modified, where N is the multi message + /// enable, which means that the vector set here has to be aligned to that number, and that + /// all vectors in that range have to be allocated. + pub message_data: Option, + + /// A bitmap of the vectors that are masked. This field is not guaranteed (and not likely, + /// at least according to the feature flags I got from QEMU), to exist. + pub mask_bits: Option, +} + +/// Some flags that might be set simultaneously, but separately. +#[derive(Debug, Serialize, Deserialize)] +#[non_exhaustive] +pub enum SetFeatureInfo { + Msi(MsiSetFeatureInfo), + + MsiX { + /// Masks the entire function, and all of its vectors. + function_mask: Option, + }, +} + #[derive(Debug, Serialize, Deserialize)] #[non_exhaustive] pub enum PcidClientRequest { @@ -122,12 +173,14 @@ pub enum PcidClientRequest { EnableFeature(PciFeature), FeatureStatus(PciFeature), FeatureInfo(PciFeature), + SetFeatureInfo(SetFeatureInfo), } #[derive(Debug, Serialize, Deserialize)] #[non_exhaustive] pub enum PcidServerResponseError { NonexistentFeature(PciFeature), + InvalidBitPattern, } #[derive(Debug, Serialize, Deserialize)] @@ -139,6 +192,7 @@ pub enum PcidClientResponse { FeatureStatus(PciFeature, FeatureStatus), Error(PcidServerResponseError), FeatureInfo(PciFeature, PciFeatureInfo), + SetFeatureInfo(PciFeature), } // TODO: Ideally, pcid might have its own scheme, like lots of other Redox drivers, where this kind of IPC is done. Otherwise, instead of writing serde messages over @@ -226,4 +280,11 @@ impl PcidServerHandle { other => Err(PcidClientHandleError::InvalidResponse(other)), } } + pub fn set_feature_info(&mut self, info: SetFeatureInfo) -> Result<()> { + self.send(&PcidClientRequest::SetFeatureInfo(info))?; + match self.recv()? { + PcidClientResponse::SetFeatureInfo(_) => Ok(()), + other => Err(PcidClientHandleError::InvalidResponse(other)), + } + } } diff --git a/pcid/src/main.rs b/pcid/src/main.rs index d2369d1707..fa34f78314 100644 --- a/pcid/src/main.rs +++ b/pcid/src/main.rs @@ -9,6 +9,9 @@ use std::{env, io, i64, thread}; use syscall::iopl; +use log::{error, info, warn, trace}; +use redox_log::{OutputBuilder, RedoxLogger}; + use crate::config::Config; use crate::pci::{CfgAccess, Pci, PciIter, PciBar, PciBus, PciClass, PciDev, PciFunc, PciHeader, PciHeaderError, PciHeaderType}; use crate::pci::cap::Capability as PciCapability; @@ -29,7 +32,7 @@ pub struct DriverHandler { state: Arc, } -fn with_pci_func_raw T>(pci: &Pci, bus_num: u8, dev_num: u8, func_num: u8, function: F) -> T { +fn with_pci_func_raw T>(pci: &dyn CfgAccess, bus_num: u8, dev_num: u8, func_num: u8, function: F) -> T { let bus = PciBus { pci, num: bus_num, @@ -46,7 +49,7 @@ fn with_pci_func_raw T>(pci: &Pci, bus_num: u8, dev_nu } impl DriverHandler { fn with_pci_func_raw T>(&self, function: F) -> T { - with_pci_func_raw(&self.state.pci, self.bus_num, self.dev_num, self.func_num, function) + with_pci_func_raw(self.state.preferred_cfg_access(), self.bus_num, self.dev_num, self.func_num, function) } fn respond(&mut self, request: driver_interface::PcidClientRequest, args: &driver_interface::SubdriverArguments) -> driver_interface::PcidClientResponse { use driver_interface::*; @@ -70,7 +73,7 @@ impl DriverHandler { None => return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(feature)), }; unsafe { - with_pci_func_raw(&self.state.pci, self.bus_num, self.dev_num, self.func_num, |func| { + with_pci_func_raw(self.state.preferred_cfg_access(), self.bus_num, self.dev_num, self.func_num, |func| { capability.set_enabled(true); capability.write_message_control(func, offset); }); @@ -83,7 +86,7 @@ impl DriverHandler { None => return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(feature)), }; unsafe { - with_pci_func_raw(&self.state.pci, self.bus_num, self.dev_num, self.func_num, |func| { + with_pci_func_raw(self.state.preferred_cfg_access(), self.bus_num, self.dev_num, self.func_num, |func| { capability.set_msix_enabled(true); capability.write_a(func, offset); }); @@ -115,6 +118,56 @@ impl DriverHandler { return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(feature)); } }), + PcidClientRequest::SetFeatureInfo(info_to_set) => match info_to_set { + SetFeatureInfo::Msi(info_to_set) => if let Some((offset, info)) = self.capabilities.iter_mut().find_map(|(offset, capability)| Some((*offset, capability.as_msi_mut()?))) { + if let Some(mme) = info_to_set.multi_message_enable { + if info.multi_message_capable() < mme || mme > 0b101 { + return PcidClientResponse::Error(PcidServerResponseError::InvalidBitPattern); + } + info.set_multi_message_enable(mme); + + } + if let Some(message_addr) = info_to_set.message_address { + if message_addr & 0b11 != 0 { + return PcidClientResponse::Error(PcidServerResponseError::InvalidBitPattern); + } + info.set_message_address(message_addr); + } + if let Some(message_addr_upper) = info_to_set.message_upper_address { + info.set_message_upper_address(message_addr_upper); + } + if let Some(message_data) = info_to_set.message_data { + if message_data & ((1 << info.multi_message_enable()) - 1) != 0 { + return PcidClientResponse::Error(PcidServerResponseError::InvalidBitPattern); + } + info.set_message_data(message_data); + } + if let Some(mask_bits) = info_to_set.mask_bits { + info.set_mask_bits(mask_bits); + } + unsafe { + with_pci_func_raw(self.state.preferred_cfg_access(), self.bus_num, self.dev_num, self.func_num, |func| { + info.write_all(func, offset); + }); + } + PcidClientResponse::SetFeatureInfo(PciFeature::Msi) + } else { + return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(PciFeature::Msi)); + } + SetFeatureInfo::MsiX { function_mask } => if let Some((offset, info)) = self.capabilities.iter_mut().find_map(|(offset, capability)| Some((*offset, capability.as_msix_mut()?))) { + if let Some(mask) = function_mask { + info.set_function_mask(mask); + unsafe { + with_pci_func_raw(self.state.preferred_cfg_access(), self.bus_num, self.dev_num, self.func_num, |func| { + info.write_a(func, offset); + }); + } + } + PcidClientResponse::SetFeatureInfo(PciFeature::MsiX) + } else { + return PcidClientResponse::Error(PcidServerResponseError::NonexistentFeature(PciFeature::MsiX)); + } + } } } fn handle_spawn(mut self, pcid_to_client_write: Option, pcid_from_client_read: Option, args: driver_interface::SubdriverArguments) { @@ -139,13 +192,15 @@ pub struct State { } impl State { fn preferred_cfg_access(&self) -> &dyn CfgAccess { - self.pcie.as_ref().map(|pcie| pcie as &dyn CfgAccess).unwrap_or(&*self.pci as &dyn CfgAccess) + // TODO + //self.pcie.as_ref().map(|pcie| pcie as &dyn CfgAccess).unwrap_or(&*self.pci as &dyn CfgAccess) + &*self.pci as &dyn CfgAccess } } fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, dev_num: u8, func_num: u8, header: PciHeader) { - let pci = &state.pci; + let pci = state.preferred_cfg_access(); let raw_class: u8 = header.class().into(); let mut string = format!("PCI {:>02X}/{:>02X}/{:>02X} {:>04X}:{:>04X} {:>02X}.{:>02X}.{:>02X}.{:>02X} {:?}", @@ -193,7 +248,7 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, string.push('\n'); - print!("{}", string); + info!("{}", string); for driver in config.drivers.iter() { if let Some(class) = driver.class { @@ -312,7 +367,7 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, }; crate::pci::cap::CapabilitiesIter { inner: crate::pci::cap::CapabilityOffsetsIter::new(header.cap_pointer(), &func) }.collect::>() }; - println!("PCI DEVICE CAPABILITIES for {}: {:?}", args.iter().map(|string| string.as_ref()).nth(0).unwrap_or("[unknown]"), capabilities); + info!("PCI DEVICE CAPABILITIES for {}: {:?}", args.iter().map(|string| string.as_ref()).nth(0).unwrap_or("[unknown]"), capabilities); use driver_interface::LegacyInterruptPin; @@ -324,7 +379,7 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, 4 => Some(LegacyInterruptPin::IntD), other => { - println!("pcid: invalid interrupt pin: {}", other); + warn!("pcid: invalid interrupt pin: {}", other); None } }; @@ -353,7 +408,7 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, "$BUS" => format!("{:>02X}", bus_num), "$DEV" => format!("{:>02X}", dev_num), "$FUNC" => format!("{:>02X}", func_num), - "$NAME" => format!("pci-{:>02X}.{:>02X}.{:>02X}", bus_num, dev_num, func_num), + "$NAME" => func.name(), "$BAR0" => format!("{}", bars[0]), "$BAR1" => format!("{}", bars[1]), "$BAR2" => format!("{}", bars[2]), @@ -374,9 +429,9 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, command.arg(&arg); } - println!("PCID SPAWN {:?}", command); + info!("PCID SPAWN {:?}", command); - let (pcid_to_client_write, pcid_from_client_read, envs) = if driver.channel_name.is_some() { + let (pcid_to_client_write, pcid_from_client_read, envs) = if driver.use_channel.unwrap_or(false) { let mut fds1 = [0usize; 2]; let mut fds2 = [0usize; 2]; @@ -407,16 +462,56 @@ fn handle_parsed_header(state: Arc, config: &Config, bus_num: u8, }); match child.wait() { Ok(_status) => (), - Err(err) => println!("pcid: failed to wait for {:?}: {}", command, err), + Err(err) => error!("pcid: failed to wait for {:?}: {}", command, err), } } - Err(err) => println!("pcid: failed to execute {:?}: {}", command, err) + Err(err) => error!("pcid: failed to execute {:?}: {}", command, err) } } } } } +fn setup_logging() -> Option<&'static RedoxLogger> { + let mut logger = RedoxLogger::new() + .with_output( + OutputBuilder::stderr() + .with_ansi_escape_codes() + .with_filter(log::LevelFilter::Info) + .flush_on_newline(true) + .build() + ); + + match OutputBuilder::in_redox_logging_scheme("bus", "pci", "pcid.log") { + Ok(b) => logger = logger.with_output( + b.with_filter(log::LevelFilter::Trace) + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("pcid: failed to open pcid.log"), + } + match OutputBuilder::in_redox_logging_scheme("bus", "pci", "pcid.ansi.log") { + Ok(b) => logger = logger.with_output( + b.with_filter(log::LevelFilter::Trace) + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("pcid: failed to open pcid.ansi.log"), + } + + match logger.enable() { + Ok(logger_ref) => { + eprintln!("pcid: enabled logger"); + Some(logger_ref) + } + Err(error) => { + eprintln!("pcid: failed to set default logger: {}", error); + None + } + } +} + fn main() { let mut config = Config::default(); @@ -447,6 +542,8 @@ fn main() { } } + let _logger_ref = setup_logging(); + let pci = Arc::new(Pci::new()); let state = Arc::new(State { @@ -454,7 +551,7 @@ fn main() { pcie: match Pcie::new(Arc::clone(&pci)) { Ok(pcie) => Some(pcie), Err(error) => { - println!("Couldn't retrieve PCIe info, perhaps the kernel is not compiled with acpi? Using the PCI 3.0 configuration space instead. Error: {:?}", error); + info!("Couldn't retrieve PCIe info, perhaps the kernel is not compiled with acpi? Using the PCI 3.0 configuration space instead. Error: {:?}", error); None } }, @@ -463,7 +560,7 @@ fn main() { let pci = state.preferred_cfg_access(); - print!("PCI BS/DV/FN VEND:DEVI CL.SC.IN.RV\n"); + info!("PCI BS/DV/FN VEND:DEVI CL.SC.IN.RV"); 'bus: for bus in PciIter::new(pci) { 'dev: for dev in bus.devs() { @@ -476,16 +573,16 @@ fn main() { Err(PciHeaderError::NoDevice) => { if func_num == 0 { if dev.num == 0 { - // println!("PCI {:>02X}: no bus", bus.num); + trace!("PCI {:>02X}: no bus", bus.num); continue 'bus; } else { - // println!("PCI {:>02X}/{:>02X}: no dev", bus.num, dev.num); + trace!("PCI {:>02X}/{:>02X}: no dev", bus.num, dev.num); continue 'dev; } } }, Err(PciHeaderError::UnknownHeaderType(id)) => { - println!("pcid: unknown header type: {}", id); + warn!("pcid: unknown header type: {}", id); } } } diff --git a/pcid/src/pci/cap.rs b/pcid/src/pci/cap.rs index d808776cec..15e2077305 100644 --- a/pcid/src/pci/cap.rs +++ b/pcid/src/pci/cap.rs @@ -25,7 +25,7 @@ where if self.offset == 0 { return None }; - let first_dword = dbg!(self.reader.read_u32(dbg!(u16::from(self.offset)))); + let first_dword = self.reader.read_u32(u16::from(self.offset)); let next = ((first_dword >> 8) & 0xFF) as u8; let offset = self.offset; @@ -41,6 +41,7 @@ pub enum CapabilityId { Msi = 0x05, MsiX = 0x11, Pcie = 0x10, + Sata = 0x12, // only on AHCI functions } #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)] @@ -48,13 +49,13 @@ pub enum MsiCapability { _32BitAddress { message_control: u32, message_address: u32, - message_data: u32, + message_data: u16, }, _64BitAddress { message_control: u32, message_address_lo: u32, message_address_hi: u32, - message_data: u32, + message_data: u16, }, _32BitAddressWithPvm { message_control: u32, @@ -85,11 +86,12 @@ pub struct MsixCapability { pub c: u32, } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub enum Capability { Msi(MsiCapability), MsiX(MsixCapability), Pcie(PcieCapability), + FunctionSpecific(u8, Vec), // TODO: Arrayvec Other(u8), } @@ -156,6 +158,8 @@ impl Capability { Self::parse_msix(reader, offset) } else if capability_id == CapabilityId::Pcie as u8 { Self::parse_pcie(reader, offset) + } else if capability_id == CapabilityId::Sata as u8 { + Self::FunctionSpecific(capability_id, reader.read_range(offset.into(), 8)) } else { Self::Other(capability_id) //panic!("unimplemented or malformed capability id: {}", capability_id) diff --git a/pcid/src/pci/mod.rs b/pcid/src/pci/mod.rs index bea70cd448..3307c857c2 100644 --- a/pcid/src/pci/mod.rs +++ b/pcid/src/pci/mod.rs @@ -8,6 +8,8 @@ pub use self::dev::{PciDev, PciDevIter}; pub use self::func::PciFunc; pub use self::header::{PciHeader, PciHeaderError, PciHeaderType}; +use log::info; + mod bar; mod bus; pub mod cap; @@ -45,7 +47,7 @@ impl Pci { fn set_iopl() { // make sure that pcid is not granted io port permission unless pcie memory-mapped // configuration space is not available. - println!("PCI: couldn't find or access PCIe extended configuration, and thus falling back to PCI 3.0 io ports"); + info!("PCI: couldn't find or access PCIe extended configuration, and thus falling back to PCI 3.0 io ports"); unsafe { syscall::iopl(3).expect("pcid: failed to set iopl to 3"); } } fn address(bus: u8, dev: u8, func: u8, offset: u8) -> u32 { diff --git a/pcid/src/pci/msi.rs b/pcid/src/pci/msi.rs index cd1561b755..145f611533 100644 --- a/pcid/src/pci/msi.rs +++ b/pcid/src/pci/msi.rs @@ -48,13 +48,13 @@ impl MsiCapability { message_control: dword, message_address_lo: reader.read_u32(u16::from(offset + 4)), message_address_hi: reader.read_u32(u16::from(offset + 8)), - message_data: reader.read_u32(u16::from(offset + 12)), + message_data: reader.read_u32(u16::from(offset + 12)) as u16, } } else { Self::_32BitAddress { message_control: dword, message_address: reader.read_u32(u16::from(offset + 4)), - message_data: reader.read_u32(u16::from(offset + 8)), + message_data: reader.read_u32(u16::from(offset + 8)) as u16, } } } @@ -80,7 +80,7 @@ impl MsiCapability { | Self::_64BitAddressWithPvm { ref mut message_control, .. } => *message_control = new_message_control, } } - pub unsafe fn write_message_control(&mut self, writer: &W, offset: u8) { + pub unsafe fn write_message_control(&self, writer: &W, offset: u8) { writer.write_u32(u16::from(offset), self.message_control_raw()); } pub fn is_pvt_capable(&self) -> bool { @@ -100,14 +100,106 @@ impl MsiCapability { pub fn multi_message_capable(&self) -> u8 { ((self.message_control() & Self::MC_MULTI_MESSAGE_MASK) >> Self::MC_MULTI_MESSAGE_SHIFT) as u8 } - pub fn multi_message_enabled(&self) -> u8 { + pub fn multi_message_enable(&self) -> u8 { ((self.message_control() & Self::MC_MULTI_MESSAGE_ENABLE_MASK) >> Self::MC_MULTI_MESSAGE_ENABLE_SHIFT) as u8 } - pub fn set_multi_message_enabled(&mut self, log_mme: u8) { + pub fn set_multi_message_enable(&mut self, log_mme: u8) { let mut new_message_control = self.message_control() & (!Self::MC_MULTI_MESSAGE_ENABLE_MASK); new_message_control |= (u16::from(log_mme) << Self::MC_MULTI_MESSAGE_ENABLE_SHIFT); self.set_message_control(new_message_control); } + + pub fn message_address(&self) -> u32 { + match self { + &Self::_32BitAddress { message_address, .. } | &Self::_32BitAddressWithPvm { message_address, .. } => message_address, + &Self::_64BitAddress { message_address_lo, .. } | &Self::_64BitAddressWithPvm { message_address_lo, .. } => message_address_lo, + } + } + pub fn message_upper_address(&self) -> Option { + match self { + &Self::_64BitAddress { message_address_hi, .. } | &Self::_64BitAddressWithPvm { message_address_hi, .. } => Some(message_address_hi), + &Self::_32BitAddress { .. } | &Self::_32BitAddressWithPvm { .. } => None, + } + } + pub fn message_data(&self) -> u16 { + match self { + &Self::_32BitAddress { message_data, .. } | &Self::_64BitAddress { message_data, .. } => message_data, + &Self::_32BitAddressWithPvm { message_data, .. } | &Self::_64BitAddressWithPvm { message_data, .. } => message_data as u16, + } + } + pub fn mask_bits(&self) -> Option { + match self { + &Self::_32BitAddressWithPvm { mask_bits, .. } | &Self::_64BitAddressWithPvm { mask_bits, .. } => Some(mask_bits), + &Self::_32BitAddress { .. } | &Self::_64BitAddress { .. } => None, + } + } + pub fn pending_bits(&self) -> Option { + match self { + &Self::_32BitAddressWithPvm { pending_bits, .. } | &Self::_64BitAddressWithPvm { pending_bits, .. } => Some(pending_bits), + &Self::_32BitAddress { .. } | &Self::_64BitAddress { .. } => None, + } + } + pub fn set_message_address(&mut self, message_address: u32) { + assert_eq!(message_address & 0xFFFF_FFFC, message_address, "unaligned message address (this should already be validated)"); + match self { + &mut Self::_32BitAddress { message_address: ref mut addr, .. } | &mut Self::_32BitAddressWithPvm { message_address: ref mut addr, .. } => *addr = message_address, + &mut Self::_64BitAddress { message_address_lo: ref mut addr, .. } | &mut Self::_64BitAddressWithPvm { message_address_lo: ref mut addr, .. } => *addr = message_address, + } + } + pub fn set_message_upper_address(&mut self, message_upper_address: u32) -> Option<()> { + match self { + &mut Self::_64BitAddress { ref mut message_address_hi, .. } | &mut Self::_64BitAddressWithPvm { ref mut message_address_hi, .. } => *message_address_hi = message_upper_address, + &mut Self::_32BitAddress { .. } | &mut Self::_32BitAddressWithPvm { .. } => return None, + } + Some(()) + } + pub fn set_message_data(&mut self, value: u16) { + match self { + &mut Self::_32BitAddress { ref mut message_data, .. } | &mut Self::_64BitAddress { ref mut message_data, .. } => *message_data = value, + &mut Self::_32BitAddressWithPvm { ref mut message_data, .. } | &mut Self::_64BitAddressWithPvm { ref mut message_data, .. } => { + *message_data &= 0xFFFF_0000; + *message_data |= u32::from(value); + } + } + } + pub fn set_mask_bits(&mut self, mask_bits: u32) -> Option<()> { + match self { + &mut Self::_32BitAddressWithPvm { mask_bits: ref mut bits, .. } | &mut Self::_64BitAddressWithPvm { mask_bits: ref mut bits, .. } => *bits = mask_bits, + &mut Self::_32BitAddress { .. } | &mut Self::_64BitAddress { .. } => return None, + } + Some(()) + } + pub unsafe fn write_message_address(&self, writer: &W, offset: u8) { + writer.write_u32(u16::from(offset) + 4, self.message_address()) + } + pub unsafe fn write_message_upper_address(&self, writer: &W, offset: u8) -> Option<()> { + let value = self.message_upper_address()?; + writer.write_u32(u16::from(offset + 8), value); + Some(()) + } + pub unsafe fn write_message_data(&self, writer: &W, offset: u8) { + match self { + &Self::_32BitAddress { message_data, .. } => writer.write_u32(u16::from(offset + 8), message_data.into()), + &Self::_32BitAddressWithPvm { message_data, .. } => writer.write_u32(u16::from(offset + 8), message_data), + &Self::_64BitAddress { message_data, .. } => writer.write_u32(u16::from(offset + 12), message_data.into()), + &Self::_64BitAddressWithPvm { message_data, .. } => writer.write_u32(u16::from(offset + 12), message_data), + } + } + pub unsafe fn write_mask_bits(&self, writer: &W, offset: u8) -> Option<()> { + match self { + &Self::_32BitAddressWithPvm { mask_bits, .. } => writer.write_u32(u16::from(offset + 12), mask_bits), + &Self::_64BitAddressWithPvm { mask_bits, .. } => writer.write_u32(u16::from(offset + 16), mask_bits), + &Self::_32BitAddress { .. } | &Self::_64BitAddress { .. } => return None, + } + Some(()) + } + pub unsafe fn write_all(&self, writer: &W, offset: u8) { + self.write_message_control(writer, offset); + self.write_message_address(writer, offset); + self.write_message_upper_address(writer, offset); + self.write_message_data(writer, offset); + self.write_mask_bits(writer, offset); + } } impl MsixCapability { @@ -293,12 +385,11 @@ pub mod x86_64 { } // TODO: should the reserved field be preserved? - pub const fn message_address(destination_id: u8, rh: bool, dm: bool, xx: u8) -> u32 { + pub const fn message_address(destination_id: u8, rh: bool, dm: bool) -> u32 { 0xFEE0_0000u32 | ((destination_id as u32) << 12) | ((rh as u32) << 3) | ((dm as u32) << 2) - | xx as u32 } pub const fn message_data(trigger_mode: TriggerMode, level_trigger_mode: LevelTriggerMode, delivery_mode: DeliveryMode, vector: u8) -> u32 { ((trigger_mode as u32) << 15) @@ -321,22 +412,34 @@ impl MsixTableEntry { pub fn addr_hi(&self) -> u32 { self.addr_hi.read() } + pub fn set_addr_lo(&mut self, value: u32) { + self.addr_lo.write(value); + } + pub fn set_addr_hi(&mut self, value: u32) { + self.addr_hi.write(value); + } pub fn msg_data(&self) -> u32 { self.msg_data.read() } pub fn vec_ctl(&self) -> u32 { self.vec_ctl.read() } + pub fn set_msg_data(&mut self, value: u32) { + self.msg_data.write(value); + } pub fn addr(&self) -> u64 { u64::from(self.addr_lo()) | (u64::from(self.addr_hi()) << 32) } pub const VEC_CTL_MASK_BIT: u32 = 1; + pub fn set_masked(&mut self, masked: bool) { + self.vec_ctl.writef(Self::VEC_CTL_MASK_BIT, masked) + } pub fn mask(&mut self) { - self.vec_ctl.writef(Self::VEC_CTL_MASK_BIT, true) + self.set_masked(true); } pub fn unmask(&mut self) { - self.vec_ctl.writef(Self::VEC_CTL_MASK_BIT, false) + self.set_masked(false); } } diff --git a/xhcid/Cargo.toml b/xhcid/Cargo.toml index c1ef18ce37..3817f5f8af 100644 --- a/xhcid/Cargo.toml +++ b/xhcid/Cargo.toml @@ -20,11 +20,12 @@ plain = "0.2" lazy_static = "1.4" log = "0.4" redox_event = { git = "https://gitlab.redox-os.org/redox-os/event.git" } -redox-log = { git = "https://gitlab.redox-os.org/redox-os/redox-log.git" } +redox-log = { git = "https://gitlab.redox-os.org/redox-os/redox-log.git", tag = "v0.1.0" } redox_syscall = { git = "https://gitlab.redox-os.org/redox-os/syscall.git" } serde = { version = "1", features = ["derive"] } serde_json = "1" smallvec = { version = "1", features = ["serde"] } thiserror = "1" toml = "0.5" + pcid = { path = "../pcid" } diff --git a/xhcid/src/main.rs b/xhcid/src/main.rs index 66a84b2262..db0df15acb 100644 --- a/xhcid/src/main.rs +++ b/xhcid/src/main.rs @@ -1,7 +1,7 @@ #[macro_use] extern crate bitflags; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::fs::{self, File}; use std::future::Future; use std::io::{self, Read, Write}; @@ -11,11 +11,13 @@ use std::ptr::NonNull; use std::sync::{Arc, Mutex}; use std::env; -use pcid_interface::{PcidServerHandle, PciFeature, PciFeatureInfo}; +use pcid_interface::{MsiSetFeatureInfo, PcidServerHandle, PciFeature, PciFeatureInfo, SetFeatureInfo}; +use pcid_interface::irq_helpers::{read_bsp_apic_id, allocate_single_interrupt_vector}; use pcid_interface::msi::{MsiCapability, MsixCapability, MsixTableEntry}; use event::{Event, EventQueue}; use log::info; +use redox_log::{RedoxLogger, OutputBuilder}; use syscall::data::Packet; use syscall::error::EWOULDBLOCK; use syscall::flag::{CloneFlags, PHYSMAP_NO_CACHE, PHYSMAP_WRITE}; @@ -32,74 +34,61 @@ pub mod driver_interface; mod usb; mod xhci; -/// Read the local APIC id of the bootstrap processor. -fn read_bsp_apic_id() -> io::Result { - let mut buffer = [0u8; 8]; - - let mut file = File::open("irq:bsp")?; - let bytes_read = file.read(&mut buffer)?; - - Ok(if bytes_read == 8 { - u64::from_le_bytes(buffer) as u32 - } else if bytes_read == 4 { - u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]) - } else { - panic!("`irq:` scheme responded with {} bytes, expected {}", bytes_read, std::mem::size_of::()); - }) -} -/// Allocate an interrupt vector, located at the BSP's IDT. -fn allocate_interrupt_vector() -> io::Result> { - let available_irqs = fs::read_dir("irq:")?; - - for entry in available_irqs { - let entry = entry?; - let path = entry.path(); - - let file_name = match path.file_name() { - Some(f) => f, - None => continue, - }; - - let path_str = match file_name.to_str() { - Some(s) => s, - None => continue, - }; - - if let Ok(irq_number) = path_str.parse::() { - // if found, reserve the irq - let irq_handle = File::create(format!("irq:{}", irq_number))?; - let interrupt_vector = irq_number + 32; - return Ok(Some((interrupt_vector, irq_handle))); - } - } - Ok(None) -} - async fn handle_packet(hci: Arc, packet: Packet) -> Packet { todo!() } +fn setup_logging() -> Option<&'static RedoxLogger> { + let mut logger = RedoxLogger::new() + .with_output( + OutputBuilder::stderr() + .with_filter(log::LevelFilter::Info) // limit global output to important info + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ); + + #[cfg(target_os = "redox")] + match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.log") { + Ok(b) => logger = logger.with_output( + // TODO: Add a configuration file for this + b.with_filter(log::LevelFilter::Trace) + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("Failed to create xhci.log: {}", error), + } + + #[cfg(target_os = "redox")] + match OutputBuilder::in_redox_logging_scheme("usb", "host", "xhci.ansi.log") { + Ok(b) => logger = logger.with_output( + b.with_filter(log::LevelFilter::Trace) + .with_ansi_escape_codes() + .flush_on_newline(true) + .build() + ), + Err(error) => eprintln!("Failed to create xhci.ansi.log: {}", error), + } + + match logger.enable() { + Ok(logger_ref) => { + eprintln!("xhcid: enabled logger"); + Some(logger_ref) + } + Err(error) => { + eprintln!("xhcid: failed to set default logger: {}", error); + None + } + } +} + fn main() { - let mut args = env::args().skip(1); - - let mut name = args.next().expect("xhcid: no name provided"); - name.push_str("_xhci"); - // Daemonize if unsafe { syscall::clone(CloneFlags::empty()).unwrap() } != 0 { return; } - match redox_log::RedoxLogger::new("usb", "host", "xhci.log") { - Ok(logger) => match logger.with_stdout_mirror().enable() { - Ok(_) => { - println!("xhcid: enabled logger"); - log::set_max_level(log::LevelFilter::Debug); - } - Err(error) => eprintln!("xhcid: failed to set default logger: {}", error), - } - Err(error) => eprintln!("xhcid: failed to initialize logger: {}", error), - } + let _logger_ref = setup_logging(); let mut pcid_handle = PcidServerHandle::connect_default().expect("xhcid: failed to setup channel to pcid"); let pci_config = pcid_handle.fetch_config().expect("xhcid: failed to fetch config"); @@ -108,6 +97,9 @@ fn main() { let bar = pci_config.func.bars[0]; let irq = pci_config.func.legacy_interrupt_line; + let mut name = pci_config.func.name(); + name.push_str("_xhci"); + let bar_ptr = match bar { pcid_interface::PciBar::Memory(ptr) => ptr, other => panic!("Expected memory bar, found {}", other), @@ -124,29 +116,45 @@ fn main() { let (has_msi, mut msi_enabled) = all_pci_features.iter().map(|(feature, status)| (feature.is_msi(), status.is_enabled())).find(|&(f, _)| f).unwrap_or((false, false)); let (has_msix, mut msix_enabled) = all_pci_features.iter().map(|(feature, status)| (feature.is_msix(), status.is_enabled())).find(|&(f, _)| f).unwrap_or((false, false)); - dbg!(has_msi, msi_enabled); - dbg!(has_msix, msix_enabled); - if has_msi && !msi_enabled && !has_msix { - pcid_handle.enable_feature(PciFeature::Msi).expect("xhcid: failed to enable MSI"); - info!("Enabled MSI"); msi_enabled = true; } if has_msix && !msix_enabled { - pcid_handle.enable_feature(PciFeature::MsiX).expect("xhcid: failed to enable MSI-X"); - info!("Enabled MSI-X"); msix_enabled = true; } let (mut irq_file, interrupt_method) = if msi_enabled && !msix_enabled { + use pcid_interface::msi::x86_64::{DeliveryMode, self as x86_64_msix}; + let mut capability = match pcid_handle.feature_info(PciFeature::MsiX).expect("xhcid: failed to retrieve the MSI capability structure from pcid") { PciFeatureInfo::Msi(s) => s, PciFeatureInfo::MsiX(_) => panic!(), }; - // use one vector - capability.set_multi_message_enabled(0); + // TODO: Allow allocation of up to 32 vectors. - todo!("msi (msix is implemented though)") + // TODO: Find a way to abstract this away, potantially as a helper module for + // pcid_interface, so that this can be shared between nvmed, xhcid, ixgebd, etc.. + + let destination_id = read_bsp_apic_id().expect("xhcid: failed to read BSP apic id"); + let lapic_id = u8::try_from(destination_id).expect("CPU id didn't fit inside u8"); + let msg_addr = x86_64_msix::message_address(lapic_id, false, false); + + let (vector, interrupt_handle) = allocate_single_interrupt_vector(destination_id).expect("xhcid: failed to allocate interrupt vector").expect("xhcid: no interrupt vectors left"); + let msg_data = x86_64_msix::message_data_edge_triggered(DeliveryMode::Fixed, vector); + + let set_feature_info = MsiSetFeatureInfo { + multi_message_enable: Some(0), + message_address: Some(msg_addr), + message_upper_address: Some(0), + message_data: Some(msg_data as u16), + mask_bits: None, + }; + pcid_handle.set_feature_info(SetFeatureInfo::Msi(set_feature_info)).expect("xhcid: failed to set feature info"); + + pcid_handle.enable_feature(PciFeature::Msi).expect("xhcid: failed to enable MSI"); + info!("Enabled MSI"); + + (Some(interrupt_handle), InterruptMethod::Msi) } else if msix_enabled { let capability = match pcid_handle.feature_info(PciFeature::MsiX).expect("xhcid: failed to retrieve the MSI-X capability structure from pcid") { PciFeatureInfo::Msi(_) => panic!(), @@ -158,7 +166,6 @@ fn main() { let pba_min_length = crate::xhci::scheme::div_round_up(table_size, 8); let pba_base = capability.pba_base_pointer(pci_config.func.bars); - dbg!(table_size, table_base, table_min_length, pba_base); if !(bar_ptr..bar_ptr + 65536).contains(&(table_base as u32 + table_min_length as u32)) { todo!() @@ -178,7 +185,7 @@ fn main() { // Allocate one msi vector. - { + let method = { use pcid_interface::msi::x86_64::{DeliveryMode, self as x86_64_msix}; // primary interrupter @@ -188,11 +195,12 @@ fn main() { let table_entry_pointer = info.table_entry_pointer(k); let destination_id = read_bsp_apic_id().expect("xhcid: failed to read BSP apic id"); + let lapic_id = u8::try_from(destination_id).expect("xhcid: CPU id couldn't fit inside u8"); let rh = false; let dm = false; - let addr = x86_64_msix::message_address(destination_id.try_into().expect("xhcid: BSP apic id couldn't fit u8"), rh, dm, 0b00); + let addr = x86_64_msix::message_address(lapic_id, rh, dm); - let (vector, interrupt_handle) = allocate_interrupt_vector().expect("xhcid: failed to allocate interrupt vector").expect("xhcid: no interrupt vectors left"); + let (vector, interrupt_handle) = allocate_single_interrupt_vector(destination_id).expect("xhcid: failed to allocate interrupt vector").expect("xhcid: no interrupt vectors left"); let msg_data = x86_64_msix::message_data_edge_triggered(DeliveryMode::Fixed, vector); table_entry_pointer.addr_lo.write(addr); @@ -201,7 +209,12 @@ fn main() { table_entry_pointer.vec_ctl.writef(MsixTableEntry::VEC_CTL_MASK_BIT, false); (Some(interrupt_handle), InterruptMethod::MsiX(Mutex::new(info))) - } + }; + + pcid_handle.enable_feature(PciFeature::MsiX).expect("xhcid: failed to enable MSI-X"); + info!("Enabled MSI-X"); + + method } else if pci_config.func.legacy_interrupt_pin.is_some() { // legacy INTx# interrupt pins. (Some(File::open(format!("irq:{}", irq)).expect("xhcid: failed to open legacy IRQ file")), InterruptMethod::Intx) @@ -210,8 +223,6 @@ fn main() { (None, InterruptMethod::Polling) }; - std::thread::sleep(std::time::Duration::from_millis(300)); - print!( "{}", format!(" + XHCI {} on: {} IRQ: {}\n", name, bar, irq) diff --git a/xhcid/src/xhci/irq_reactor.rs b/xhcid/src/xhci/irq_reactor.rs index 581e2214d2..2b80883c29 100644 --- a/xhcid/src/xhci/irq_reactor.rs +++ b/xhcid/src/xhci/irq_reactor.rs @@ -212,7 +212,7 @@ impl IrqReactor { if index >= self.states.len() { break } match self.states[index].kind { - StateKind::CommandCompletion { phys_ptr } if dbg!(trb.trb_type()) == TrbType::CommandCompletion as u8 => if dbg!(trb.completion_trb_pointer()) == Some(phys_ptr) { + StateKind::CommandCompletion { phys_ptr } if trb.trb_type() == TrbType::CommandCompletion as u8 => if trb.completion_trb_pointer() == Some(phys_ptr) { trace!("Found matching command completion future"); let state = self.states.remove(index); @@ -430,7 +430,6 @@ impl Xhci { if ! trb.is_command_trb() { panic!("Invalid TRB type given to next_command_completion_event_trb(): {} (TRB {:?}. Expected command TRB.", trb.trb_type(), trb) } - dbg!(command_ring.trbs.physical()); EventTrbFuture::Pending { state: FutureState { // This is only possible for transfers if they are isochronous, or for Force Event TRBs (virtualization).