diff --git a/.gitignore b/.gitignore index 10954d0e98..8322b69057 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ target image.bin image +image* diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 295e5e0c80..5e74fc8228 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "aes" @@ -23,6 +23,56 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.86" @@ -40,37 +90,20 @@ dependencies = [ [[package]] name = "argon2" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25df3c03f1040d0069fcd3907e24e36d59f9b6fa07ba49be0eb25a794f036ba7" +checksum = "db4ce4441f99dbd377ca8a8f57b698c44d0d6e712d8329b5040da5a64aa1ce73" dependencies = [ "base64ct", "blake2", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - [[package]] name = "base64ct" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.5.0" @@ -133,6 +166,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "cpufeatures" version = "0.2.12" @@ -152,6 +191,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", +] + [[package]] name = "derive_arbitrary" version = "1.3.2" @@ -181,16 +229,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ad847bb2094f110bbdd6fa564894ca4556fd978958e93985420d680d3cb6d14" [[package]] -name = "env_logger" -version = "0.9.3" +name = "env_filter" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ - "atty", - "humantime", "log", "regex", - "termcolor", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", ] [[package]] @@ -200,7 +258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -211,9 +269,9 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fuser" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5910691a0ececcc6eba8bb14029025c2d123e96a53db1533f6a4602861a5aaf7" +checksum = "2e697f6f62c20b6fad1ba0f84ae909f25971cf16e735273524e3977c94604cf8" dependencies = [ "libc", "log", @@ -221,7 +279,6 @@ dependencies = [ "page_size", "pkg-config", "smallvec", - "users", "zerocopy", ] @@ -243,23 +300,38 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "is_terminal_polyfill" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "jiff" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ad87c89110f55e4cd4dc2893a9790820206729eaf221555f742d540b0724a0" dependencies = [ - "libc", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", ] [[package]] -name = "humantime" -version = "2.1.0" +name = "jiff-static" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "d076d5b64a7e2fe6f0743f02c43ca4a6725c0f904203bfe276a5b3e793103605" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "jobserver" @@ -287,26 +359,15 @@ dependencies = [ "once_cell", ] -[[package]] -name = "libredox" -version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3af92c55d7d839293953fcd0fda5ecfe93297cfde6ffbdec13b41d99c0ba6607" -dependencies = [ - "bitflags 2.5.0", - "libc", - "redox_syscall 0.4.1", -] - [[package]] name = "libredox" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.5.0", + "bitflags", "libc", - "redox_syscall 0.5.2", + "redox_syscall", ] [[package]] @@ -333,23 +394,29 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.5.0", + "bitflags", "cfg-if", "cfg_aliases", "libc", ] [[package]] -name = "numtoa" +name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "numtoa" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa2c4e539b869820a2b82e1aef6ff40aa85e65decdd5185e83fb4b1249cd00f" [[package]] name = "once_cell" -version = "1.19.0" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "opaque-debug" @@ -359,9 +426,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "page_size" -version = "0.4.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eebde548fbbf1ea81a99b128872779c437752fb99f217c45245e1a61dcd9edcd" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" dependencies = [ "libc", "winapi", @@ -374,19 +441,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] -name = "proc-macro2" -version = "1.0.85" +name = "portable-atomic" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -404,21 +492,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436d45c2b6a5b159d43da708e62b25be3a4a3d5550d654b72216ade4c4bfd717" [[package]] -name = "redox_syscall" -version = "0.4.1" +name = "redox-scheme" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "6143c4d307e1c99ac14f60b5b07b2dccaf9d17137f7cee4e4e29977dd8014a1b" dependencies = [ - "bitflags 1.3.2", + "libredox", + "redox_syscall", ] [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ - "bitflags 2.5.0", + "bitflags", ] [[package]] @@ -429,7 +518,7 @@ checksum = "20145670ba436b55d91fc92d25e71160fbfbdd57831631c8d7d36377a476f1cb" [[package]] name = "redoxfs" -version = "0.6.4" +version = "0.6.10" dependencies = [ "aes", "argon2", @@ -439,11 +528,12 @@ dependencies = [ "fuser", "getrandom", "libc", - "libredox 0.1.3", + "libredox", "log", "range-tree", "redox-path", - "redox_syscall 0.5.2", + "redox-scheme", + "redox_syscall", "seahash", "termion", "time", @@ -498,11 +588,11 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -511,6 +601,26 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -525,9 +635,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.66" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -543,41 +653,40 @@ dependencies = [ "cfg-if", "fastrand", "rustix", - "windows-sys", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", + "windows-sys 0.52.0", ] [[package]] name = "termion" -version = "2.0.3" +version = "4.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4648c7def6f2043b2568617b9f9b75eae88ca185dbc1f1fda30e95a85d49d7d" +checksum = "3669a69de26799d6321a5aa713f55f7e2cd37bd47be044b50f2acafc42c122bb" dependencies = [ "libc", - "libredox 0.0.2", + "libredox", "numtoa", "redox_termios", ] [[package]] name = "time" -version = "0.1.45" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", ] +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + [[package]] name = "typenum" version = "1.17.0" @@ -591,14 +700,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] -name = "users" -version = "0.11.0" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24cc0f6d6f267b73e5a2cadf007ba8f9bc39c6a6f9666f8cf25ea809a153b032" -dependencies = [ - "libc", - "log", -] +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" @@ -615,12 +720,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -643,15 +742,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" -dependencies = [ - "windows-sys", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -668,10 +758,19 @@ dependencies = [ ] [[package]] -name = "windows-targets" -version = "0.52.5" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -685,57 +784,57 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "zerocopy" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", @@ -743,9 +842,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index e8e2c9bca4..7bb6c9ec30 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -14,7 +14,7 @@ cargo-fuzz = true [dependencies] anyhow = "1.0.86" arbitrary = { version = "1.3.2", features = ["derive"] } -fuser = { version = "0.12.0" } +fuser = { version = "0.14" } libfuzzer-sys = "0.4" nix = { version = "0.29.0", features = ["fs"] } tempfile = "3.10.1" diff --git a/fuzz/fuzz_targets/fuse_fuzz_target.rs b/fuzz/fuzz_targets/fuse_fuzz_target.rs index cef49d038c..d983a6754a 100644 --- a/fuzz/fuzz_targets/fuse_fuzz_target.rs +++ b/fuzz/fuzz_targets/fuse_fuzz_target.rs @@ -315,7 +315,7 @@ fuzz_target!(|test_case: TestCase| -> Corpus { for mount_seq in test_case.mount_sequences.iter() { #[cfg(feature = "log")] - eprintln!("mount fs"); + eprintln!("mount fs: path {:?}, size{}", temp_dir.path(), test_case.disk_size); let disk = create_disk(temp_dir.path(), test_case.disk_size); let operations = mount_seq.operations.clone(); diff --git a/src/block.rs b/src/block.rs index a429ce089f..2df0e7773b 100644 --- a/src/block.rs +++ b/src/block.rs @@ -231,6 +231,19 @@ impl BlockPtr { self.addr().is_null() } + pub fn marker(level: u8) -> Self { + assert!(level <= 0xF); + Self { + addr: (0xFFFF_FFFF_FFFF_FFF0 | (level as u64)).into(), + hash: u64::MAX.into(), + phantom: PhantomData, + } + } + + pub fn is_marker(&self) -> bool { + (self.addr.to_ne() | 0xF) == u64::MAX && self.hash.to_ne() == u64::MAX + } + /// Cast BlockPtr to another type /// /// # Safety @@ -316,3 +329,14 @@ fn block_list_size_test() { fn block_raw_size_test() { assert_eq!(mem::size_of::(), BLOCK_SIZE as usize); } + +#[test] +fn block_ptr_marker_test() { + let ptr = BlockPtr::::marker(0); + assert_eq!(ptr.addr().level().0, 0); + assert!(ptr.is_marker()); + + let ptr = BlockPtr::::marker(2); + assert_eq!(ptr.addr().level().0, 2); + assert!(ptr.is_marker()); +} diff --git a/src/dir.rs b/src/dir.rs index d65a5d9f4a..b5be384226 100644 --- a/src/dir.rs +++ b/src/dir.rs @@ -1,7 +1,6 @@ -use alloc::{boxed::Box, vec}; use core::{mem, ops, slice, str}; -use crate::{BlockLevel, BlockTrait, Node, TreePtr, DIR_ENTRY_MAX_LENGTH, RECORD_LEVEL}; +use crate::{BlockLevel, BlockTrait, Node, TreePtr, BLOCK_SIZE, DIR_ENTRY_MAX_LENGTH}; #[repr(C, packed)] #[derive(Clone, Copy)] @@ -26,7 +25,7 @@ impl DirEntry { self.node_ptr } - pub fn name(&self) -> Option<&str> { + fn name_len(&self) -> usize { let mut len = 0; while len < self.name.len() { if self.name[len] == 0 { @@ -34,9 +33,63 @@ impl DirEntry { } len += 1; } + len + } + + pub fn name(&self) -> Option<&str> { + let len = self.name_len(); //TODO: report utf8 error? str::from_utf8(&self.name[..len]).ok() } + + // 4 bytes TreePtr + // 1 byte name_len + const SERIALIZED_PREFIX_SIZE: usize = mem::size_of::>() + 1; + + pub fn serialized_size(&self) -> usize { + DirEntry::SERIALIZED_PREFIX_SIZE + self.name_len() + } + + fn serialize_into(&self, buf: &mut [u8]) -> Option { + let required = self.serialized_size(); + if buf.len() < required { + return None; + } + + buf[0..4].copy_from_slice(&self.node_ptr().to_bytes()); + buf[4] = self.name_len() as u8; + buf[5..5 + self.name_len()].copy_from_slice(&self.name[..self.name_len()]); + + Some(required) + } + + fn deserialize_from(buf: &[u8]) -> Result<(Self, usize), &'static str> { + if buf.len() <= DirEntry::SERIALIZED_PREFIX_SIZE { + return Err("Buffer too small"); + } + + let node_ptr: TreePtr = + TreePtr::from_bytes(buf[0..4].try_into().expect("Slice must be 4 bytes long")); + let name_len = buf[4] as usize; + + if name_len < 1 || name_len > DIR_ENTRY_MAX_LENGTH { + return Err("Invalid name length"); + } + + if buf.len() < DirEntry::SERIALIZED_PREFIX_SIZE + name_len { + return Err("Buffer too small"); + } + + let mut name = [0u8; DIR_ENTRY_MAX_LENGTH]; + name[..name_len].copy_from_slice( + &buf[DirEntry::SERIALIZED_PREFIX_SIZE..DirEntry::SERIALIZED_PREFIX_SIZE + name_len], + ); + + Ok(( + DirEntry { node_ptr, name }, + DirEntry::SERIALIZED_PREFIX_SIZE + name_len, + )) + } } impl Default for DirEntry { @@ -48,17 +101,19 @@ impl Default for DirEntry { } } -//TODO: this is a box to prevent stack overflows pub struct DirList { - pub entries: Box<[DirEntry]>, + count: u16, + entry_bytes_len: u16, + entry_bytes: [u8; BLOCK_SIZE as usize - 4], } unsafe impl BlockTrait for DirList { fn empty(level: BlockLevel) -> Option { - if level.0 <= RECORD_LEVEL { - let entries = level.bytes() as usize / mem::size_of::(); + if level.0 == 0 { Some(Self { - entries: vec![DirEntry::default(); entries].into_boxed_slice(), + count: 0, + entry_bytes_len: 0, + entry_bytes: [0; BLOCK_SIZE as usize - 4], }) } else { None @@ -68,7 +123,96 @@ unsafe impl BlockTrait for DirList { impl DirList { pub fn is_empty(&self) -> bool { - self.entries.iter().all(|entry| entry.node_ptr().is_null()) + self.count == 0 + } + + pub fn entries(&self) -> DirEntryIterator { + DirEntryIterator { + dir_list: self, + emit_count: 0, + position: 0, + } + } + + fn entry_position_for_name(&self, name: &str) -> Option { + let name_len = name.len(); + let mut position = 0; + let mut entry_id = 0; + + while entry_id < self.count { + let entry_name_len = self.entry_bytes[position + 4] as usize; + if entry_name_len == name_len { + let start = DirEntry::SERIALIZED_PREFIX_SIZE + position; + let entry_name = &self.entry_bytes[start..start + entry_name_len]; + if entry_name == name.as_bytes() { + return Some(position); + } + } + position += DirEntry::SERIALIZED_PREFIX_SIZE + entry_name_len; + entry_id += 1; + } + None + } + + pub fn find_entry(&self, name: &str) -> Option { + if let Some(position) = self.entry_position_for_name(name) { + let (entry, _) = DirEntry::deserialize_from(&self.entry_bytes[position..]).unwrap(); + return Some(entry); + } + None + } + + pub fn remove_entry(&mut self, name: &str) -> bool { + if let Some(position) = self.entry_position_for_name(name) { + let entry_size = + DirEntry::SERIALIZED_PREFIX_SIZE + self.entry_bytes[position + 4] as usize; + let remaining_size = self.entry_bytes_len as usize - position - entry_size; + if remaining_size > 0 { + self.entry_bytes.copy_within( + position + entry_size..self.entry_bytes_len as usize, + position, + ); + } + self.entry_bytes_len -= entry_size as u16; + self.count -= 1; + return true; + } + false + } + + pub fn for_each_entry(&self, mut f: F) + where + F: FnMut(&[u8; 4], &[u8]), + { + let mut position = 0; + let mut entry_id = 0; + + while entry_id < self.count { + let node_ptr_bytes = &self.entry_bytes[position..position + 4]; + //let node_ptr = TreePtr::::from_bytes(node_ptr_bytes.try_into().unwrap()); + let entry_name_len = self.entry_bytes[position + 4] as usize; + let start = DirEntry::SERIALIZED_PREFIX_SIZE + position; + let entry_name = &self.entry_bytes[start..start + entry_name_len]; + + f(node_ptr_bytes.try_into().unwrap(), entry_name); + + position += DirEntry::SERIALIZED_PREFIX_SIZE + entry_name_len; + entry_id += 1; + } + } + + pub fn append(&mut self, entry: &DirEntry) -> bool { + let entry_bytes_len = self.entry_bytes_len as usize; + if let Some(size) = entry.serialize_into(&mut self.entry_bytes[entry_bytes_len..]) { + self.count += 1; + self.entry_bytes_len += size as u16; + return true; + } + false + } + + pub fn entry_count(&self) -> usize { + self.count as usize } } @@ -77,8 +221,8 @@ impl ops::Deref for DirList { fn deref(&self) -> &[u8] { unsafe { slice::from_raw_parts( - self.entries.as_ptr() as *const u8, - self.entries.len() * mem::size_of::(), + self as *const DirList as *const u8, + mem::size_of::(), ) as &[u8] } } @@ -87,22 +231,70 @@ impl ops::Deref for DirList { impl ops::DerefMut for DirList { fn deref_mut(&mut self) -> &mut [u8] { unsafe { - slice::from_raw_parts_mut( - self.entries.as_mut_ptr() as *mut u8, - self.entries.len() * mem::size_of::(), - ) as &mut [u8] + slice::from_raw_parts_mut(self as *mut DirList as *mut u8, mem::size_of::()) + as &mut [u8] } } } -#[test] -fn dir_list_size_test() { - use core::ops::Deref; - for level_i in 0..RECORD_LEVEL { - let level = BlockLevel(level_i); - assert_eq!( - DirList::empty(level).unwrap().deref().len(), - level.bytes() as usize - ); +pub struct DirEntryIterator<'a> { + dir_list: &'a DirList, + emit_count: usize, + position: usize, +} + +impl Iterator for DirEntryIterator<'_> { + type Item = DirEntry; + + fn next(&mut self) -> Option { + if self.emit_count < self.dir_list.entry_count() { + let position = self.position; + let (entry, bytes_read) = + DirEntry::deserialize_from(&self.dir_list.entry_bytes[position..]).unwrap(); + + self.emit_count += 1; + self.position += bytes_read; + + Some(entry) + } else { + None + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use alloc::format; + + #[test] + fn dir_list_size_test() { + use core::ops::Deref; + assert_eq!( + DirList::empty(BlockLevel(0)).unwrap().deref().len(), + BLOCK_SIZE as usize + ); + } + + #[test] + fn test_append() { + let mut dir_list = DirList::empty(BlockLevel(0)).unwrap(); + let dirent = DirEntry::new(TreePtr::new(123), "test000"); + + assert!(dir_list.append(&dirent)); + assert_eq!(dir_list.entry_count(), 1); + assert_eq!(dir_list.entry_bytes_len as usize, dirent.serialized_size()); + + let max_entries = dir_list.entry_bytes.len() / dirent.serialized_size(); + for i in 1..max_entries { + let dirent = DirEntry::new(TreePtr::new(123), format!("test{i:03}").as_str()); + assert!(dir_list.append(&dirent), "Failed on iteration {i}"); + } + let dirent = DirEntry::new(TreePtr::new(123), format!("test{max_entries}").as_str()); + assert!(!dir_list.append(&dirent)); + + for (i, entry) in dir_list.entries().enumerate() { + assert_eq!(entry.name().unwrap(), format!("test{i:03}")); + } } } diff --git a/src/disk/cache.rs b/src/disk/cache.rs index a35b8480c1..ddeb336322 100644 --- a/src/disk/cache.rs +++ b/src/disk/cache.rs @@ -47,7 +47,7 @@ impl Disk for DiskCache { let mut read = 0; let mut failed = false; - for i in 0..(buffer.len() + BLOCK_SIZE as usize - 1) / (BLOCK_SIZE as usize) { + for i in 0..buffer.len().div_ceil(BLOCK_SIZE as usize) { let block_i = block + i as u64; let buffer_i = i * BLOCK_SIZE as usize; @@ -66,7 +66,7 @@ impl Disk for DiskCache { self.inner.read_at(block, buffer)?; read = 0; - for i in 0..(buffer.len() + BLOCK_SIZE as usize - 1) / (BLOCK_SIZE as usize) { + for i in 0..buffer.len().div_ceil(BLOCK_SIZE as usize) { let block_i = block + i as u64; let buffer_i = i * BLOCK_SIZE as usize; @@ -89,7 +89,7 @@ impl Disk for DiskCache { self.inner.write_at(block, buffer)?; let mut written = 0; - for i in 0..(buffer.len() + BLOCK_SIZE as usize - 1) / (BLOCK_SIZE as usize) { + for i in 0..buffer.len().div_ceil(BLOCK_SIZE as usize) { let block_i = block + i as u64; let buffer_i = i * BLOCK_SIZE as usize; diff --git a/src/disk/memory.rs b/src/disk/memory.rs new file mode 100644 index 0000000000..37eec6c865 --- /dev/null +++ b/src/disk/memory.rs @@ -0,0 +1,42 @@ +use syscall::error::{Error, Result, EIO}; + +use crate::disk::Disk; +use crate::BLOCK_SIZE; + +pub struct DiskMemory { + data: Vec, +} + +impl DiskMemory { + pub fn new(size: u64) -> DiskMemory { + DiskMemory { + data: vec![0; size as usize], + } + } +} + +impl Disk for DiskMemory { + unsafe fn read_at(&mut self, block: u64, buffer: &mut [u8]) -> Result { + let offset = (block * BLOCK_SIZE) as usize; + let end = offset + buffer.len(); + if end > self.data.len() { + return Err(Error::new(EIO)); + } + buffer.copy_from_slice(&self.data[offset..end]); + Ok(buffer.len()) + } + + unsafe fn write_at(&mut self, block: u64, buffer: &[u8]) -> Result { + let offset = (block * BLOCK_SIZE) as usize; + let end = offset + buffer.len(); + if end > self.data.len() { + return Err(Error::new(EIO)); + } + self.data[offset..end].copy_from_slice(buffer); + Ok(buffer.len()) + } + + fn size(&mut self) -> Result { + Ok(self.data.len() as u64) + } +} diff --git a/src/disk/mod.rs b/src/disk/mod.rs index 547355d205..9c51cbac26 100644 --- a/src/disk/mod.rs +++ b/src/disk/mod.rs @@ -7,6 +7,8 @@ pub use self::file::DiskFile; #[cfg(feature = "std")] pub use self::io::DiskIo; #[cfg(feature = "std")] +pub use self::memory::DiskMemory; +#[cfg(feature = "std")] pub use self::sparse::DiskSparse; #[cfg(feature = "std")] @@ -16,6 +18,8 @@ mod file; #[cfg(feature = "std")] mod io; #[cfg(feature = "std")] +mod memory; +#[cfg(feature = "std")] mod sparse; /// A disk diff --git a/src/disk/sparse.rs b/src/disk/sparse.rs index ae12192868..11997af717 100644 --- a/src/disk/sparse.rs +++ b/src/disk/sparse.rs @@ -1,7 +1,6 @@ use std::fs::{File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::Path; -use std::u64; use syscall::error::{Error, Result, EIO}; use crate::disk::Disk; diff --git a/src/filesystem.rs b/src/filesystem.rs index 8df906eba7..87f023c7dc 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -118,7 +118,7 @@ impl FileSystem { ctime_nsec: u32, ) -> Result { let size = disk.size()?; - let block_offset = (reserved.len() as u64 + BLOCK_SIZE - 1) / BLOCK_SIZE; + let block_offset = (reserved.len() as u64).div_ceil(BLOCK_SIZE); if size < (block_offset + HEADER_RING + 4) * BLOCK_SIZE { return Err(Error::new(syscall::error::ENOSPC)); diff --git a/src/htree.rs b/src/htree.rs new file mode 100644 index 0000000000..b41620f6c6 --- /dev/null +++ b/src/htree.rs @@ -0,0 +1,691 @@ +use alloc::string::String; +use alloc::vec::Vec; +use core::{fmt, mem, ops, slice}; +use endian_num::Le; +use syscall::error::{Error, Result, EEXIST, EIO}; + +use crate::{ + BlockLevel, BlockPtr, BlockRaw, BlockTrait, DirEntry, DirList, BLOCK_SIZE, RECORD_LEVEL, +}; + +pub const HTREE_IDX_ENTRIES: usize = BLOCK_SIZE as usize / mem::size_of::>(); +const HTREE_IDX_PADDING: usize = + BLOCK_SIZE as usize - mem::size_of::<[HTreePtr; HTREE_IDX_ENTRIES]>(); + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[repr(C, packed)] +pub struct HTreeHash(Le); + +impl HTreeHash { + // Create a MAX constant populated iwth the maximum value of Le minus 1 + pub const MAX: HTreeHash = HTreeHash(Le(u32::MAX - 1)); + + #[cfg(not(test))] + pub fn from_name(name: &str) -> Self { + let hash = seahash::hash(name.as_bytes()) as u32; + // Don't allow the default hash value to be calculated for a real hash + if hash == u32::MAX { + return Self::MAX; + } + Self(hash.into()) + } + + #[cfg(test)] + pub fn from_name(name: &str) -> Self { + // Allow overriding the hashing function to something easily controled for testing. + let hash = if let Some(pos) = name.rfind("__") { + let number_str = &name[pos + 2..]; + number_str.parse::().unwrap() + } else { + seahash::hash(name.as_bytes()) as u32 + }; + + // Don't allow the default hash value to be calculated for a real hash + if hash == u32::MAX { + return Self::MAX; + } + Self(hash.into()) + } + + /// Returns the maximum of two `HTreeHash` values, ignoring the default hash value. + pub fn max_ignoring_default(&self, other: Self) -> Self { + let default = HTreeHash::default(); + if *self == default { + return other; + } + if other == default { + return *self; + } + if *self > other { + *self + } else { + other + } + } + + pub fn find_max(dir_list: &DirList) -> Option { + let mut max_hash = HTreeHash::default(); + dir_list.for_each_entry(|_ptr_bytes, name_bytes| { + let name = String::from_utf8_lossy(name_bytes); + let hash = HTreeHash::from_name(name.as_ref()); + max_hash = max_hash.max_ignoring_default(hash); + }); + + if max_hash == HTreeHash::default() { + None + } else { + Some(max_hash) + } + } +} + +impl Default for HTreeHash { + /// The default hash value is the maximum possible value to push it to the end of the list when sorting. + fn default() -> Self { + Self(u32::MAX.into()) + } +} + +#[repr(C, packed)] +pub struct HTreePtr { + pub htree_hash: HTreeHash, + pub ptr: BlockPtr, +} + +impl HTreePtr { + pub fn new(htree_hash: HTreeHash, ptr: BlockPtr) -> Self { + Self { htree_hash, ptr } + } + + /// Cast HTreePtr to another type + /// + /// # Safety + /// Unsafe because it can be used to transmute types + pub unsafe fn cast(self) -> HTreePtr { + HTreePtr { + htree_hash: self.htree_hash, + ptr: self.ptr.cast(), + } + } +} + +impl HTreePtr { + pub fn is_null(&self) -> bool { + self.ptr.is_null() + } +} + +impl Clone for HTreePtr { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for HTreePtr {} + +impl Default for HTreePtr { + fn default() -> Self { + Self { + htree_hash: HTreeHash::default(), + ptr: BlockPtr::default(), + } + } +} + +impl fmt::Debug for HTreePtr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let htree_hash = self.htree_hash; + let ptr = self.ptr; + f.debug_struct("BlockPtr") + .field("htree_hash", &htree_hash) + .field("ptr", &ptr) + .finish() + } +} + +#[repr(C, packed)] +pub struct HTreeNode { + pub ptrs: [HTreePtr; HTREE_IDX_ENTRIES], + padding: [u8; HTREE_IDX_PADDING], +} + +impl HTreeNode { + pub fn find_max_htree_hash(&self) -> Option { + let mut hash = HTreeHash::default(); + for entry in self.ptrs.iter() { + hash = hash.max_ignoring_default(entry.htree_hash); + } + + if hash != HTreeHash::default() { + Some(hash) + } else { + None + } + } + + pub fn find_ptrs_for_read( + &self, + htree_hash: HTreeHash, + ) -> impl Iterator)> { + let mut last_hash = HTreeHash(0.into()); + self.ptrs + .iter() + .enumerate() + .filter(move |(_idx, entry)| entry.htree_hash >= htree_hash) + .take_while(move |(_idx, entry)| { + let should_take = !entry.is_null() && last_hash <= htree_hash; + last_hash = entry.htree_hash; + should_take + }) + } +} + +unsafe impl BlockTrait for HTreeNode { + fn empty(level: BlockLevel) -> Option { + if level.0 <= RECORD_LEVEL { + Some(Self { + ptrs: [HTreePtr::default(); HTREE_IDX_ENTRIES], + padding: [0; HTREE_IDX_PADDING], + }) + } else { + None + } + } +} + +impl ops::Deref for HTreeNode { + type Target = [u8]; + fn deref(&self) -> &[u8] { + unsafe { + slice::from_raw_parts( + self as *const HTreeNode as *const u8, + mem::size_of::>(), + ) as &[u8] + } + } +} + +impl ops::DerefMut for HTreeNode { + fn deref_mut(&mut self) -> &mut [u8] { + unsafe { + slice::from_raw_parts_mut( + self as *mut HTreeNode as *mut u8, + mem::size_of::>(), + ) as &mut [u8] + } + } +} + +pub fn add_inner_node( + parent: &mut HTreeNode, + new_ptr: HTreePtr, +) -> Result)>> { + // Update the input htree parameters in place + for ptr in parent.ptrs.iter_mut() { + if ptr.is_null() { + *ptr = new_ptr; + parent.ptrs.sort_by(|a, b| a.htree_hash.cmp(&b.htree_hash)); + return Ok(None); + } + } + + // The parent is full. We need to split it into two by half, ordered by the htree hash. + let mut all_ptrs = Vec::with_capacity(parent.ptrs.len() + 1); + for ptr in parent.ptrs.iter() { + all_ptrs.push(*ptr); + } + all_ptrs.push(new_ptr); + all_ptrs.sort_by(|a, b| a.htree_hash.cmp(&b.htree_hash)); + let half_idx = all_ptrs.len() / 2; + + // Find if there are duplicate name hashes on the boundary of where we want to split + let half_name_hash = all_ptrs[half_idx].htree_hash; + let mut first_idx = half_idx; + let mut last_idx = half_idx; + for (i, ptr) in all_ptrs.iter().enumerate() { + if ptr.htree_hash == half_name_hash { + if i < first_idx { + first_idx = i; + } + if i > last_idx { + last_idx = i; + } + } + } + + // Split the entries_with_name_hash list at the index that minimizes the number of entries in each list while keeping the duplicate name hashes together + let split = if (half_idx - first_idx) < (last_idx - half_idx) { + first_idx + } else { + last_idx + }; + + let (ptrs1, ptrs2) = all_ptrs.split_at(split); + + // Update the existing parent with the first half of the entries + let mut htree_idx1 = HTreeNode::empty(BlockLevel::default()).ok_or(Error::new(EIO))?; + htree_idx1.ptrs[..ptrs1.len()].copy_from_slice(ptrs1); + let _ = mem::replace(parent, htree_idx1); + + // Return the second half as a new sibling parent + let mut htree_idx2 = HTreeNode::empty(BlockLevel::default()).ok_or(Error::new(EIO))?; + htree_idx2.ptrs[..ptrs2.len()].copy_from_slice(ptrs2); + + let htree_hash2 = ptrs2[ptrs2.len() - 1].htree_hash; + Ok(Some((htree_hash2, htree_idx2))) +} + +pub fn add_dir_entry( + dir_list: &mut DirList, + htree_hash: &mut HTreeHash, + dirent: DirEntry, +) -> Result> { + if let Some(name) = dirent.name() { + if dir_list.find_entry(name).is_some() { + return Err(Error::new(EEXIST)); + } + } + + // Update the input htree parameters in place + let name = dirent.name().ok_or(Error::new(EIO))?; + if dir_list.append(&dirent) { + *htree_hash = HTreeHash::from_name(name).max_ignoring_default(*htree_hash); + return Ok(None); + } + + // The dir_list is full. We need to split it into two dir_lists by half, ordered by the name hash. + let mut entries_with_name_hash = Vec::with_capacity(dir_list.entry_count() + 1); + for entry in dir_list.entries() { + entries_with_name_hash.push(( + HTreeHash::from_name(entry.name().ok_or(Error::new(EIO))?), + entry, + )); + } + entries_with_name_hash.push((HTreeHash::from_name(dirent.name().unwrap()), dirent)); + entries_with_name_hash.sort_by(|a, b| a.0.cmp(&b.0)); + let half = entries_with_name_hash.len() / 2; + let half_name_hash = entries_with_name_hash[half].0; + + // Find if there are duplicate name hashes on the boundary of where we want to split + let mut first_idx = half; + let mut last_idx = half; + for (i, (name_hash, _)) in entries_with_name_hash.iter().enumerate() { + if *name_hash == half_name_hash { + if i < first_idx { + first_idx = i; + } + if i > last_idx { + last_idx = i; + } + } + } + last_idx += 1; + + // Split the entries_with_name_hash list at the index that minimizes the number of entries in each list while keeping the duplicate name hashes together + let split = if (half - first_idx) < (last_idx - half) { + first_idx + } else { + last_idx + }; + let split = split.max(1); + + let sorted_entries = entries_with_name_hash + .iter() + .map(|(_, entry)| *entry) + .collect::>(); + + let (entries1, entries2) = sorted_entries.split_at(split); + + // Update the existing dir_list with the first half of the entries + let mut new_dir_list = DirList::empty(BlockLevel::default()).ok_or(Error::new(EIO))?; + for entry in entries1.iter() { + new_dir_list.append(entry); + } + let _ = mem::replace(dir_list, new_dir_list); + *htree_hash = entries_with_name_hash[entries1.len() - 1].0; + + // Return the second half of the entries as a new dir_list + let mut new_dir_list = DirList::empty(BlockLevel::default()).ok_or(Error::new(EIO))?; + for entry in entries2.iter() { + new_dir_list.append(entry); + } + let new_name_hash = entries_with_name_hash[entries_with_name_hash.len() - 1].0; + Ok(Some((new_name_hash, new_dir_list))) +} + +// +// MARK: Unit Tests +// + +#[cfg(test)] +mod tests { + use super::*; + use crate::alloc::string::ToString; + use crate::TreePtr; + use alloc::format; + use alloc::string::String; + + #[test] + fn htree_ptr_size_test() { + assert_eq!(mem::size_of::>(), 20); + } + + #[test] + fn htree_node_size_test() { + assert_eq!(mem::size_of::>(), BLOCK_SIZE as usize); + } + + #[test] + fn htree_hash_max_test() { + assert_eq!(HTreeHash::MAX, HTreeHash((u32::MAX - 1).into())); + } + + #[test] + fn htree_hash_max_ignoring_default_test() { + let default = HTreeHash::default(); + let hash1 = HTreeHash(0.into()); + let hash2 = HTreeHash(1.into()); + + assert_eq!(hash1.max_ignoring_default(default), hash1); + assert_eq!(default.max_ignoring_default(hash1), hash1); + assert_eq!(hash1.max_ignoring_default(hash2), hash2); + } + + #[test] + fn htree_node_find_max_htree_hash() { + // In practice, the HTreeHash values should always be in sorted order + let mut htree_node: HTreeNode = HTreeNode::empty(BlockLevel::default()).unwrap(); + htree_node.ptrs[0] = HTreePtr::new(HTreeHash(0.into()), BlockPtr::marker(0)); + htree_node.ptrs[1] = HTreePtr::new(HTreeHash(1.into()), BlockPtr::marker(0)); + htree_node.ptrs[2] = HTreePtr::new(HTreeHash(2.into()), BlockPtr::marker(0)); + + assert_eq!( + htree_node.find_max_htree_hash().unwrap(), + HTreeHash(2.into()) + ); + + htree_node.ptrs[2] = HTreePtr::default(); + assert_eq!( + htree_node.find_max_htree_hash().unwrap(), + HTreeHash(1.into()) + ); + + htree_node.ptrs[1] = HTreePtr::default(); + assert_eq!( + htree_node.find_max_htree_hash().unwrap(), + HTreeHash(0.into()) + ); + + htree_node.ptrs[0] = HTreePtr::default(); + assert!(htree_node.find_max_htree_hash().is_none()); + + // For thoroughness, test with HTreeHash out of order + htree_node.ptrs[2] = HTreePtr::new(HTreeHash(4.into()), BlockPtr::marker(0)); + htree_node.ptrs[4] = HTreePtr::new(HTreeHash(6.into()), BlockPtr::marker(0)); + htree_node.ptrs[6] = HTreePtr::new(HTreeHash(2.into()), BlockPtr::marker(0)); + assert_eq!( + htree_node.find_max_htree_hash().unwrap(), + HTreeHash(6.into()) + ); + } + + #[test] + fn htree_node_find_for_read() { + let mut htree_node: HTreeNode = HTreeNode::empty(BlockLevel::default()).unwrap(); + htree_node.ptrs[0] = HTreePtr::new(HTreeHash(0.into()), BlockPtr::marker(0)); + htree_node.ptrs[1] = HTreePtr::new(HTreeHash(1.into()), BlockPtr::marker(0)); + htree_node.ptrs[2] = HTreePtr::new(HTreeHash(2.into()), BlockPtr::marker(0)); + htree_node.ptrs[3] = HTreePtr::new(HTreeHash(2.into()), BlockPtr::marker(0)); + htree_node.ptrs[4] = HTreePtr::new(HTreeHash(3.into()), BlockPtr::marker(0)); + htree_node.ptrs[5] = HTreePtr::new(HTreeHash(3.into()), BlockPtr::marker(0)); + htree_node.ptrs[6] = HTreePtr::new(HTreeHash(5.into()), BlockPtr::marker(0)); + htree_node.ptrs[7] = HTreePtr::new(HTreeHash(6.into()), BlockPtr::marker(0)); + + // Confirm that a hash that does not exist, but is less than an existing hash results in a single entry + let mut iter = htree_node.find_ptrs_for_read(HTreeHash(4.into())); + let mut val = iter.next().unwrap(); + assert_eq!(val.0, 6); + assert_eq!(val.1.htree_hash, HTreeHash(5.into())); + assert!(iter.next().is_none()); + + // Confirm that a hash that equals an existing hash results in the match and one following entry + let mut iter = htree_node.find_ptrs_for_read(HTreeHash(1.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 1); + assert_eq!(val.1.htree_hash, HTreeHash(1.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 2); + assert_eq!(val.1.htree_hash, HTreeHash(2.into())); + assert!(iter.next().is_none()); + + // Confirm that multiple exact hash matches are all returned plus the next entry + let mut iter = htree_node.find_ptrs_for_read(HTreeHash(2.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 2); + assert_eq!(val.1.htree_hash, HTreeHash(2.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 3); + assert_eq!(val.1.htree_hash, HTreeHash(2.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 4); + assert_eq!(val.1.htree_hash, HTreeHash(3.into())); + assert!(iter.next().is_none()); + + // Confirm that if the last entry matches and the next entry is null, only the match is returned + let mut iter = htree_node.find_ptrs_for_read(HTreeHash(6.into())); + val = iter.next().unwrap(); + assert_eq!(val.0, 7); + assert_eq!(val.1.htree_hash, HTreeHash(6.into())); + assert!(iter.next().is_none()); + + // Confirm that if a hash that is larger than any existing entries, then no entries are returned + let mut iter = htree_node.find_ptrs_for_read(HTreeHash(7.into())); + assert!(iter.next().is_none()); + } + + #[test] + fn add_dir_entry_exists_test() { + let mut dir_list = DirList::empty(BlockLevel::default()).unwrap(); + let mut htree_hash = HTreeHash::default(); + let dirent = DirEntry::new(TreePtr::new(123), "test"); + let new_sibling = add_dir_entry(&mut dir_list, &mut htree_hash, dirent).unwrap(); + assert!(new_sibling.is_none()); + assert_eq!(htree_hash, HTreeHash::from_name("test")); + assert_eq!(dir_list.entries().next().unwrap().name(), Some("test")); + + // Add the same entry again, and it should fail with an appropriate IO error + let dirent = DirEntry::new(TreePtr::new(123), "test"); + let error_expected = add_dir_entry(&mut dir_list, &mut htree_hash, dirent); + assert!(error_expected.is_err()); + assert_eq!(error_expected.err().unwrap().errno, EEXIST); + } + + #[test] + fn add_dir_entry_many_test() { + let mut dir_list = DirList::empty(BlockLevel::default()).unwrap(); + let mut htree_hash = HTreeHash::default(); + let total_count = 16; + + // Fill up the dir_list + for i in 0..total_count { + let v: usize = i % 10; + let dirent = DirEntry::new(TreePtr::new(123), format!("test{v}_{i:0244}").as_str()); + let new_sibling = add_dir_entry(&mut dir_list, &mut htree_hash, dirent).unwrap(); + assert!(new_sibling.is_none()); + } + + // The maximum htree_hash should be retained + let max_tree_hash = + dir_list + .entries() + .enumerate() + .fold(HTreeHash::default(), |max, (i, _)| { + let v = i % 10; + let hash = HTreeHash::from_name(format!("test{v}_{i:0244}").as_str()); + max.max_ignoring_default(hash) + }); + assert_eq!(htree_hash, max_tree_hash); + + // Confirm all the entries exist. Note they happen to be in insert order + for (i, entry) in dir_list.entries().enumerate() { + let v = i % 10; + assert_eq!(entry.name(), Some(format!("test{v}_{i:0244}").as_str())); + } + + // Test a split by adding one more entry + let dirent = DirEntry::new(TreePtr::new(123), "test_split"); + let new_sibling = add_dir_entry(&mut dir_list, &mut htree_hash, dirent).unwrap(); + let (new_sibling_htree_hash, new_sibling_dir_list) = + new_sibling.expect("new_sibling should be created"); + // assert!(new_sibling_dir_list.entries.len() ); + assert!(new_sibling_htree_hash > htree_hash); + + // The htree_hash should be less than the minimum htree_hash in new_sibling_dir_list + let new_sibling_min_htree_hash = new_sibling_dir_list + .entries() + .filter(|entry| !entry.node_ptr().is_null()) + .fold(HTreeHash::default(), |min, entry| { + let hash = HTreeHash::from_name(entry.name().unwrap()); + min.min(hash) + }); + assert!(htree_hash < new_sibling_min_htree_hash); + + // Confirm all the entries exist across both dir_lists + let mut expected_names: Vec = (0..total_count) + .map(|i| { + let v = i % 10; + format!("test{v}_{i:0244}") + }) + .collect(); + expected_names.push("test_split".to_string()); + expected_names.sort(); + + let mut dir_list_entry_count = 0; + for entry in dir_list.entries() { + dir_list_entry_count += 1; + let name = entry.name().unwrap().to_string(); + let _ = expected_names.remove(expected_names.binary_search(&name).unwrap()); + } + + let mut new_sibling_entry_count = 0; + for entry in new_sibling_dir_list.entries() { + new_sibling_entry_count += 1; + let name = entry.name().unwrap().to_string(); + let _ = expected_names.remove(expected_names.binary_search(&name).unwrap()); + } + assert!(expected_names.is_empty()); + + // Confirm that the split is in half + assert!((dir_list_entry_count as i32 - new_sibling_entry_count).abs() <= 1); + } + + #[test] + fn add_inner_node_simple_test() { + let mut htree_node: HTreeNode<_> = HTreeNode::empty(BlockLevel::default()).unwrap(); + let htree_ptr: HTreePtr<_> = HTreePtr:: { + htree_hash: HTreeHash::from_name("test"), + ptr: BlockPtr::marker(0), + }; + let new_sibling = add_inner_node(&mut htree_node, htree_ptr).unwrap(); + assert!(new_sibling.is_none()); + assert_eq!(htree_node.ptrs[0].htree_hash, HTreeHash::from_name("test")); + } + + #[test] + fn add_inner_node_multiple_test() { + let mut htree_node: HTreeNode<_> = HTreeNode::empty(BlockLevel::default()).unwrap(); + + for i in 0..HTREE_IDX_ENTRIES { + let htree_ptr: HTreePtr<_> = HTreePtr:: { + htree_hash: HTreeHash(((100_000 + (i % 10) * 1000 + i) as u32).into()), + ptr: BlockPtr::marker(0), + }; + let new_sibling = add_inner_node(&mut htree_node, htree_ptr).unwrap(); + assert!(new_sibling.is_none()); + + // Confirm that the htree_ptrs are in sorted order at the start of the ptrs list + let mut prev_hash = HTreeHash::default(); + let mut count = 0; + for ptr in htree_node.ptrs.iter() { + if ptr.is_null() { + continue; + } + assert!( + ptr.htree_hash.max_ignoring_default(prev_hash) == ptr.htree_hash, + "index {i}: {:?} > {:?}", + ptr.htree_hash, + prev_hash + ); + prev_hash = ptr.htree_hash; + count += 1; + } + assert_eq!(count, i + 1); + } + + // Confirm all expected hashes are present + let mut expected_hashes: Vec = (0..HTREE_IDX_ENTRIES) + .map(|i| (100_000 + (i % 10) * 1000 + i) as u32) + .collect(); + expected_hashes.sort(); + + for ptr in htree_node.ptrs.iter() { + if ptr.is_null() { + break; + } + let idx = expected_hashes + .binary_search(&ptr.htree_hash.0.into()) + .unwrap(); + expected_hashes.remove(idx); + } + assert!(expected_hashes.is_empty()); + + // Force a split by adding one more entry + let htree_ptr: HTreePtr<_> = HTreePtr:: { + htree_hash: HTreeHash(130_000.into()), + ptr: BlockPtr::marker(0), + }; + + let mut expected_hashes: Vec = (0..HTREE_IDX_ENTRIES) + .map(|i| (100_000 + (i % 10) * 1000 + i) as u32) + .collect(); + expected_hashes.push(130_000); + expected_hashes.sort(); + + let new_sibling = add_inner_node(&mut htree_node, htree_ptr).unwrap(); + let new_sibling = new_sibling.expect("new_sibling should be created"); + + // Confirm all the entries exist across both htree_nodes + let mut htree_node_entry_count = 0; + for ptr in htree_node.ptrs.iter() { + if ptr.ptr.is_null() { + break; + } + htree_node_entry_count += 1; + let idx = expected_hashes + .binary_search(&ptr.htree_hash.0.into()) + .unwrap(); + expected_hashes.remove(idx); + } + + let mut new_sibling_entry_count = 0; + for ptr in new_sibling.1.ptrs.iter() { + if ptr.ptr.is_null() { + break; + } + new_sibling_entry_count += 1; + let idx = expected_hashes + .binary_search(&ptr.htree_hash.0.into()) + .unwrap(); + expected_hashes.remove(idx); + } + assert!( + expected_hashes.is_empty(), + "expected_hashes should be empty, but had length {}: {:?}", + expected_hashes.len(), + expected_hashes + ); + + // Confirm that the split is in half + assert!((htree_node_entry_count as i32 - new_sibling_entry_count).abs() <= 1); + } +} diff --git a/src/lib.rs b/src/lib.rs index 088d47bfb5..25d056a069 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,7 @@ pub const BLOCK_SIZE: u64 = 4096; pub const RECORD_LEVEL: usize = 5; pub const RECORD_SIZE: u64 = BLOCK_SIZE << RECORD_LEVEL; pub const SIGNATURE: &[u8; 8] = b"RedoxFS\0"; -pub const VERSION: u64 = 6; +pub const VERSION: u64 = 7; pub const DIR_ENTRY_MAX_LENGTH: usize = 252; pub static IS_UMT: AtomicUsize = AtomicUsize::new(0); @@ -49,6 +49,7 @@ mod dir; mod disk; mod filesystem; mod header; +mod htree; mod key; #[cfg(all(feature = "std", not(fuzzing)))] mod mount; diff --git a/src/mount/fuse.rs b/src/mount/fuse.rs index 9c1bdec964..186939e319 100644 --- a/src/mount/fuse.rs +++ b/src/mount/fuse.rs @@ -77,7 +77,7 @@ fn node_attr(node: &TreeData) -> FileAttr { ino: node.id() as u64, size: node.data().size(), // Blocks is in 512 byte blocks, not in our block size - blocks: (node.data().size() + BLOCK_SIZE - 1) / BLOCK_SIZE * (BLOCK_SIZE / 512), + blocks: node.data().size().div_ceil(BLOCK_SIZE) * (BLOCK_SIZE / 512), blksize: 512, atime: SystemTime::UNIX_EPOCH + Duration::new(node.data().atime().0, node.data().atime().1), mtime: SystemTime::UNIX_EPOCH + Duration::new(node.data().mtime().0, node.data().mtime().1), @@ -99,7 +99,7 @@ fn node_attr(node: &TreeData) -> FileAttr { } } -impl<'f, D: Disk> Filesystem for Fuse<'f, D> { +impl Filesystem for Fuse<'_, D> { fn lookup(&mut self, _req: &Request, parent_id: u64, name: &OsStr, reply: ReplyEntry) { let parent_ptr = TreePtr::new(parent_id as u32); match self diff --git a/src/tests.rs b/src/tests.rs index c911ab54e9..2000d70ae6 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,8 +1,16 @@ -use crate::{unmount_path, DiskSparse, FileSystem, Node, TreePtr, ALLOC_GC_THRESHOLD}; +use crate::htree::{HTreeHash, HTreeNode, HTreePtr, HTREE_IDX_ENTRIES}; +use crate::{ + unmount_path, BlockAddr, BlockData, BlockLevel, BlockPtr, DirEntry, DirList, DiskMemory, + DiskSparse, FileSystem, Node, TreePtr, ALLOC_GC_THRESHOLD, BLOCK_SIZE, +}; +use core::panic::AssertUnwindSafe; +use std::panic::catch_unwind; use std::path::Path; use std::process::Command; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; +use std::thread::sleep; +use std::time::Duration; use std::{fs, thread, time}; static IMAGE_SEQ: AtomicUsize = AtomicUsize::new(0); @@ -45,7 +53,8 @@ where let join_handle = crate::mount(fs, dbg!(mount_path), move |real_path| { let real_path = real_path.to_owned(); thread::spawn(move || { - let res = callback(&real_path); + let res = catch_unwind(AssertUnwindSafe(|| callback(&real_path))); + let real_path = real_path.to_str().unwrap(); if cfg!(target_os = "redox") { @@ -56,11 +65,17 @@ where } if !unmount_path(real_path).is_ok() { - panic!("umount failed"); + // There seems to be a race condition where the device can be busy when trying to unmount. + // So, we pause for a moment and retry. There will still be an error output to the logs + // for the first failed attempt. + sleep(Duration::from_millis(200)); + if !unmount_path(real_path).is_ok() { + panic!("umount failed"); + } } } - res + res.unwrap() }) }) .unwrap(); @@ -82,6 +97,107 @@ fn simple() { }) } +#[test] +fn create_and_remove_file() { + with_mounted(|path| { + let file_name = "test_file.txt"; + let file_path = path.join(file_name); + + // Create the file + fs::write(&file_path, "Hello, world!").unwrap(); + assert!(fs::exists(&file_path).unwrap()); + + // Read the file + let contents = fs::read_to_string(&file_path).unwrap(); + assert_eq!(contents, "Hello, world!"); + + // Remove the file + fs::remove_file(&file_path).unwrap(); + assert!(!fs::exists(&file_path).unwrap()); + }); +} + +#[test] +fn create_and_remove_directory() { + with_mounted(|path| { + let dir_name = "test_dir"; + let dir_path = path.join(dir_name); + + // Create the directory + fs::create_dir(&dir_path).unwrap(); + assert!(fs::exists(&dir_path).unwrap()); + + // Check that the directory is empty + let entries: Vec<_> = fs::read_dir(&dir_path) + .unwrap() + .map(|e| e.unwrap().file_name()) + .collect(); + assert!(entries.is_empty()); + + // Add a file to the directory + let file_name = "test_file.txt"; + let file_path = dir_path.join(file_name); + fs::write(&file_path, "Hello, world!").unwrap(); + + // Check that the dir cannot be removed when not empty + let error = fs::remove_dir(&dir_path); + assert!(error.is_err()); + assert_eq!( + error.unwrap_err().kind(), + std::io::ErrorKind::DirectoryNotEmpty + ); + + // Remove the file + fs::remove_file(&file_path).unwrap(); + + // Remove the directory + fs::remove_dir(&dir_path).unwrap(); + assert!(!fs::exists(&dir_path).unwrap()); + }); +} + +#[test] +fn create_and_remove_symlink() { + with_mounted(|path| { + let real_file = "real_file.txt"; + let real_path = path.join(real_file); + let symlink_file = "symlink_to_real_file.txt"; + let symlink_path = path.join(symlink_file); + + // Create the real file + fs::write(&real_path, "Hello, world!").unwrap(); + + // Create the symmlink according to the platform + #[cfg(unix)] + std::os::unix::fs::symlink(&real_file, &symlink_path).unwrap(); + + #[cfg(windows)] + std::os::windows::fs::symlink_file(&real_file, &symlink_path).unwrap(); + + // Check that the symlink exists and points to the correct target + let exists = fs::exists(&symlink_path); + assert!( + exists.is_ok() && exists.unwrap(), + "Symlink should exist but was: {:?}", + fs::exists(&symlink_path) + ); + let symlink_metadata = fs::symlink_metadata(&symlink_path).unwrap(); + assert!(symlink_metadata.file_type().is_symlink()); + let target = fs::read_link(&symlink_path).unwrap(); + assert_eq!(target.to_str().unwrap(), real_file); + assert_eq!(fs::read(&symlink_path).unwrap(), b"Hello, world!"); + + // Confirm the symlink cannot be removed as a directory + let error = fs::remove_dir(&symlink_path); + assert!(error.is_err()); + assert_eq!(error.unwrap_err().kind(), std::io::ErrorKind::NotADirectory); + + // Remove the symlink + fs::remove_file(&symlink_path).unwrap(); + assert!(!fs::exists(&symlink_path).unwrap()); + }); +} + #[cfg(target_os = "redox")] #[test] fn mmap() { @@ -140,24 +256,6 @@ fn mmap() { }) } -#[test] -fn create_remove_should_not_increase_size() { - with_redoxfs(|mut fs| { - let initially_free = fs.allocator().free(); - - let tree_ptr = TreePtr::::root(); - let name = "test"; - let _ = fs - .tx(|tx| { - tx.create_node(tree_ptr, name, Node::MODE_FILE | 0644, 1, 0)?; - tx.remove_node(tree_ptr, name, Node::MODE_FILE) - }) - .unwrap(); - - assert_eq!(fs.allocator().free(), initially_free); - }); -} - #[test] fn many_create_remove_should_not_increase_size() { with_redoxfs(|mut fs| { @@ -190,3 +288,1072 @@ fn many_create_remove_should_not_increase_size() { assert_eq!(diff, 0); }); } + +#[test] +fn many_create_then_many_remove_should_not_increase_size() { + with_redoxfs(|mut fs| { + let tree_ptr = TreePtr::::root(); + let initially_free = fs.allocator().free(); + let initial_size = fs.tx(|tx| tx.read_tree(tree_ptr)).unwrap().data().size(); + + let end = 3000; + for i in 0..end { + let _ = fs + .tx(|tx| { + tx.create_node( + tree_ptr, + &format!("test{}", i), + Node::MODE_FILE | 0644, + 1, + 0, + ) + }) + .unwrap(); + } + + for i in 0..end { + let result = + fs.tx(|tx| tx.remove_node(tree_ptr, &format!("test{}", i), Node::MODE_FILE)); + if result.is_err() { + println!("Failed to delete on iteration {i}"); + } + result.unwrap(); + } + + let final_size = fs.tx(|tx| tx.read_tree(tree_ptr)).unwrap().data().size(); + assert_eq!(initial_size, final_size); + + // Any value greater than 0 indicates a storage leak + let _ = fs.tx(|tx| tx.sync(true)); + let diff = initially_free - fs.allocator().free(); + assert_eq!(diff, 0); + }); +} + +#[test] +fn empty_dir() { + with_redoxfs(|mut fs| { + let root_ptr = TreePtr::root(); + let empty_dir = fs + .tx(|tx| tx.create_node(root_ptr, "my_dir", Node::MODE_DIR, 1, 0)) + .unwrap(); + + // List + let mut children = Vec::::new(); + let _ = fs + .tx(|tx| tx.child_nodes(empty_dir.ptr(), &mut children)) + .unwrap(); + assert_eq!(children.len(), 0); + + // Find + let error = fs.tx(|tx| tx.find_node(empty_dir.ptr(), "does_not_exist")); + assert!(error.is_err()); + assert_eq!(error.unwrap_err().errno, syscall::error::ENOENT); + + // Remove + let error = fs.tx(|tx| tx.remove_node(empty_dir.ptr(), "does_not_exist", Node::MODE_FILE)); + assert!(error.is_err()); + assert_eq!(error.unwrap_err().errno, syscall::error::ENOENT); + }) +} + +// TODO: When increasing the total_count to 8000, the Allocator's deallocate() function surfaces as "slow" according to flamegraph. This +// appears to be the result of bulk deleting in this test, but I would bet that any filesystem that has lived for a long time would +// start to see degraded performance due to this. +#[test] +fn many_create_write_list_find_read_delete() { + let disk = DiskMemory::new(1024 * 1024 * 1024); + let ctime = time::SystemTime::now() + .duration_since(time::UNIX_EPOCH) + .unwrap(); + let mut fs = FileSystem::create(disk, None, ctime.as_secs(), ctime.subsec_nanos()).unwrap(); + let tree_ptr = TreePtr::::root(); + let total_count = 3000; + + // Create a bunch of files + for i in 0..total_count { + let result = fs.tx(|tx| { + tx.create_node( + tree_ptr, + &format!("file{i:05}"), + Node::MODE_FILE | 0644, + 1, + 0, + ) + }); + if result.is_err() { + println!("Failure on create iteration {i}"); + } + + let file_node = result.unwrap(); + let result = fs.tx(|tx| { + tx.write_node( + file_node.ptr(), + 0, + format!("Hello World! #{i}").as_bytes(), + ctime.as_secs(), + ctime.subsec_nanos(), + ) + }); + if result.is_err() { + println!("Failure on write iteration {i}"); + } + assert!(result.unwrap() > 0) + } + + // Confirm that they can be listed + { + let mut children = Vec::::with_capacity(total_count); + let _ = fs.tx(|tx| tx.child_nodes(tree_ptr, &mut children)).unwrap(); + assert_eq!( + children.len(), + total_count, + "The list of children should match the number of files created." + ); + let mut children: Vec = children + .iter() + .map(|entry| entry.name().unwrap_or_default().to_string()) + .collect(); + children.sort(); + + for i in 0..total_count { + let expected = format!("file{i:05}"); + let idx = children.binary_search(&expected); + assert!(idx.is_ok(), "Children did not contain '{}'", expected); + } + } + + // Find and read the files + for i in 0..total_count { + let result = fs.tx(|tx| tx.find_node(tree_ptr, &format!("file{i:05}"))); + if result.is_err() { + println!("Failure on find node iteration {i}"); + } + + let file_node = result.unwrap(); + let offset = 0; + let mut buf = [0_u8; 32]; + let result = fs.tx(|tx| { + tx.read_node( + file_node.ptr(), + offset, + &mut buf, + ctime.as_secs(), + ctime.subsec_nanos(), + ) + }); + if result.is_err() { + println!("Failure on read iteration {i}"); + } + let size = result.unwrap(); + let body = std::str::from_utf8(&buf[..size]).unwrap(); + assert_eq!(body, format!("Hello World! #{i}")); + } + + // Delete all the files + for i in 0..total_count { + let file_name = format!("file{i:05}"); + let result = fs.tx(|tx| tx.remove_node(tree_ptr, &file_name, Node::MODE_FILE)); + if result.is_err() { + println!("Failure on delete iteration {i}"); + result.unwrap(); + } + let result = fs.tx(|tx| tx.find_node(tree_ptr, &file_name)); + if !result.is_err() || result.err().unwrap().errno != syscall::error::ENOENT { + println!("Failure on delete verification iteration {i}"); + assert!(false, "Deletion appears to ahve failred"); + } + } +} + +#[test] +fn many_write_read_delete_mounted() { + with_mounted(|path| { + let total_count = 500; + + for i in 0..total_count { + fs::write( + &path.join(format!("file{}", i)), + format!("Hello, number {i}!"), + ) + .unwrap(); + } + + // Confirm each of the created files can be found and read + for i in 0..total_count { + let contents = fs::read_to_string(&path.join(format!("file{}", i))).unwrap(); + assert_eq!(contents, format!("Hello, number {i}!")); + } + + // Remove all the files + for i in 0..total_count { + let file_path = path.join(format!("file{i}")); + assert!(fs::exists(&file_path).unwrap()); + fs::remove_file(&file_path).unwrap(); + assert!(!fs::exists(&file_path).unwrap()); + } + }); +} + +// +// MARK: H-Tree tests +// +// Note that most of these tests use a test specific HTreeHash implementation that will simply parse the numeric +// value after two underscores in the name. So a name of `my_file__10` would have a HTreeHash value of 10. This +// allows for some explicit placement of test values into the H-tree. +// + +/// Create an unnaturally narrow but deep H-tree structure for efficient testing of the internal +/// algorithms used to change the H-tree state. +fn create_minimal_l2_htree( + child1_name: &str, + mut fs: FileSystem, +) -> (FileSystem, TreePtr) { + let parent_ptr = TreePtr::::root(); + let child_ptr = fs + .tx(|tx| { + let mut parent = tx.read_tree(parent_ptr).unwrap(); + + let child1_block_data = BlockData::new( + unsafe { tx.allocate(BlockLevel::default()) }.unwrap(), + Node::new( + Node::MODE_FILE, + parent.data().uid(), + parent.data().gid(), + 1, + 0, + ), + ); + let child1_block_ptr = unsafe { tx.write_block(child1_block_data) }.unwrap(); + let child1_ptr = tx.insert_tree(child1_block_ptr).unwrap(); + let child1_dir_entry = DirEntry::new(child1_ptr, child1_name); + let child1_htree_hash = HTreeHash::from_name(child1_name); + + let mut dir_list = BlockData::::empty(BlockAddr::default()).unwrap(); + dir_list.data_mut().append(&child1_dir_entry); + let dir_ptr = tx.sync_block(dir_list).unwrap(); + + let mut l1 = BlockData::>::empty(BlockAddr::default()).unwrap(); + l1.data_mut().ptrs[0] = HTreePtr::new(child1_htree_hash, dir_ptr); + let l1_ptr = tx.sync_block(l1).unwrap(); + + let mut l2 = + BlockData::>>::empty(BlockAddr::default()).unwrap(); + l2.data_mut().ptrs[0] = HTreePtr::new(child1_htree_hash, l1_ptr); + let l2_ptr = tx.sync_block(l2).unwrap(); + let l2_ptr = unsafe { l2_ptr.cast() }; + + parent.data_mut().level0[0] = BlockPtr::marker(2); + parent.data_mut().level0[1] = l2_ptr; + let size = parent.data().size() + BLOCK_SIZE * 4; + parent.data_mut().size = size.into(); + tx.sync_tree(parent).unwrap(); + Ok(child1_ptr) + }) + .unwrap(); + (fs, child_ptr) +} + +#[test] +fn insert_dir_entry_without_hash_change() { + with_redoxfs(|fs| { + let parent_ptr = TreePtr::::root(); + + // GIVEN a directory with H-Tree populated to level 2 and a new entry that lands + // in the last existing DirList, but the hash sorts lower than the max hash in the DirList + let child1_name = "child1__9"; + let child2_name = "child2__1"; + let child1_htree_hash = HTreeHash::from_name(child1_name); + let (mut fs, child1_ptr) = create_minimal_l2_htree(child1_name, fs); + + let _ = fs.tx(|tx| { + // WHEN the new child node is added to the parent directory + let child2_node = tx + .create_node(parent_ptr, child2_name, Node::MODE_FILE, 2, 0) + .unwrap(); + + // THEN the child node is added, but the H-Tree retains its structure, and the updated nodes retain + // the old HTreeHash value + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr = unsafe { parent.data().level0[1].cast() }; + let l2: BlockData>> = tx.read_block(l2_ptr).unwrap(); + + let l1_ptr = l2.data().ptrs[0]; + let l1 = tx.read_block(l1_ptr.ptr).unwrap(); + assert_eq!(l1_ptr.htree_hash, child1_htree_hash); + + let dir_list_ptr = l1.data().ptrs[0]; + let dir_list = tx.read_block(dir_list_ptr.ptr).unwrap(); + assert_eq!(dir_list_ptr.htree_hash, child1_htree_hash); + + let mut entries: Vec = dir_list + .data() + .entries() + .map(|e| e.name().unwrap().to_string()) + .collect(); + entries.sort(); + + assert_eq!(entries.len(), 2); + assert_eq!(entries, vec![child1_name, child2_name]); + + // Validate listing child_nodes works + let mut children = Vec::new(); + tx.child_nodes(parent_ptr, &mut children).unwrap(); + let mut children: Vec<&str> = children.iter().map(|e| e.name().unwrap()).collect(); + children.sort(); + assert_eq!(children, entries); + + // Validate find_node works + assert_eq!( + tx.find_node(parent_ptr, child1_name).unwrap().ptr().id(), + child1_ptr.id() + ); + assert_eq!( + tx.find_node(parent_ptr, child2_name).unwrap().ptr().id(), + child2_node.ptr().id() + ); + + // WHEN the new child node is removed from the parent directory + tx.remove_node(parent_ptr, child2_name, Node::MODE_FILE) + .unwrap(); + + // THEN the child node is removed, the H-Tree retains its structure, and the updated nodes retain + // the old HTreeHash value + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr = unsafe { parent.data().level0[1].cast() }; + let l2: BlockData>> = tx.read_block(l2_ptr).unwrap(); + + let l1_ptr = l2.data().ptrs[0]; + let l1 = tx.read_block(l1_ptr.ptr).unwrap(); + assert_eq!(l1_ptr.htree_hash, child1_htree_hash); + + let dir_list_ptr = l1.data().ptrs[0]; + let dir_list = tx.read_block(dir_list_ptr.ptr).unwrap(); + assert_eq!(dir_list_ptr.htree_hash, child1_htree_hash); + + let entries: Vec = dir_list + .data() + .entries() + .map(|e| e.name().unwrap().to_string()) + .collect(); + + assert_eq!(entries.len(), 1); + assert_eq!(entries, vec![child1_name]); + + // Validate listing child_nodes works + let mut children = Vec::new(); + tx.child_nodes(parent_ptr, &mut children).unwrap(); + let children: Vec<&str> = children.iter().map(|e| e.name().unwrap()).collect(); + assert_eq!(children, entries); + + // Validate find_node works + assert_eq!( + tx.find_node(parent_ptr, child1_name).unwrap().ptr().id(), + child1_ptr.id() + ); + assert_eq!( + tx.find_node(parent_ptr, child2_name).unwrap_err().errno, + syscall::error::ENOENT + ); + Ok(()) + }); + }); +} + +#[test] +fn insert_dir_entry_with_hash_change() { + with_redoxfs(|fs| { + let parent_ptr = TreePtr::::root(); + + // GIVEN a directory with H-Tree populated to level 2 and a new entry that lands + // in the last existing DirList, and the hash is sorted after the max hash in the DirList + let child1_name = "child1__1"; + let child2_name = "child2__9"; + let (mut fs, child1_ptr) = create_minimal_l2_htree(child1_name, fs); + + let _ = fs.tx(|tx| { + // WHEN the new child node is added to the parent directory + let child2_node = tx + .create_node(parent_ptr, child2_name, Node::MODE_FILE, 2, 0) + .unwrap(); + + // THEN the child node is added, the H-Tree retains its structure, and the updated nodes adopt + // the new HTreeHash value + let child2_htree_hash = HTreeHash::from_name(child2_name); + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr = unsafe { parent.data().level0[1].cast() }; + let l2: BlockData>> = tx.read_block(l2_ptr).unwrap(); + + let l1_ptr = l2.data().ptrs[0]; + let l1 = tx.read_block(l1_ptr.ptr).unwrap(); + assert_eq!(l1_ptr.htree_hash, child2_htree_hash); + + let dir_list_ptr = l1.data().ptrs[0]; + let dir_list = tx.read_block(dir_list_ptr.ptr).unwrap(); + assert_eq!(dir_list_ptr.htree_hash, child2_htree_hash); + + let mut entries: Vec = dir_list + .data() + .entries() + .map(|e| e.name().unwrap().to_string()) + .collect(); + entries.sort(); + + assert_eq!(entries.len(), 2); + assert_eq!(entries, vec![child1_name, child2_name]); + + // Validate listing child_nodes works + let mut children = Vec::new(); + tx.child_nodes(parent_ptr, &mut children).unwrap(); + let mut children: Vec<&str> = children.iter().map(|e| e.name().unwrap()).collect(); + children.sort(); + assert_eq!(children, entries); + + // Validate find_node works + assert_eq!( + tx.find_node(parent_ptr, child1_name).unwrap().ptr().id(), + child1_ptr.id() + ); + assert_eq!( + tx.find_node(parent_ptr, child2_name).unwrap().ptr().id(), + child2_node.ptr().id() + ); + + // WHEN the new child node is removed from the parent directory + tx.remove_node(parent_ptr, child2_name, Node::MODE_FILE) + .unwrap(); + + // THEN the child node is removed, the H-Tree retains its structure, and the updated nodes revert + // to child1's HTreeHash value + let child1_htree_hash = HTreeHash::from_name(child1_name); + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr = unsafe { parent.data().level0[1].cast() }; + let l2: BlockData>> = tx.read_block(l2_ptr).unwrap(); + + let l1_ptr = l2.data().ptrs[0]; + let l1 = tx.read_block(l1_ptr.ptr).unwrap(); + assert_eq!(l1_ptr.htree_hash, child1_htree_hash); + + let dir_list_ptr = l1.data().ptrs[0]; + let dir_list = tx.read_block(dir_list_ptr.ptr).unwrap(); + assert_eq!(dir_list_ptr.htree_hash, child1_htree_hash); + + let entries: Vec = dir_list + .data() + .entries() + .map(|e| e.name().unwrap().to_string()) + .collect(); + + assert_eq!(entries.len(), 1); + assert_eq!(entries, vec![child1_name]); + + // Validate listing child_nodes works + let mut children = Vec::new(); + tx.child_nodes(parent_ptr, &mut children).unwrap(); + let children: Vec<&str> = children.iter().map(|e| e.name().unwrap()).collect(); + assert_eq!(children, entries); + + // Validate find_node works + assert_eq!( + tx.find_node(parent_ptr, child1_name).unwrap().ptr().id(), + child1_ptr.id() + ); + assert_eq!( + tx.find_node(parent_ptr, child2_name).unwrap_err().errno, + syscall::error::ENOENT + ); + Ok(()) + }); + }); +} + +#[test] +fn delete_to_empty() { + with_redoxfs(|fs| { + let parent_ptr = TreePtr::::root(); + + // GIVEN a nearly empty tree + let child_name = "child1__9"; + let (mut fs, _child_ptr) = create_minimal_l2_htree(child_name, fs); + + // WHEN the last directory entry is removed + fs.tx(|tx| tx.remove_node(parent_ptr, child_name, Node::MODE_FILE)) + .unwrap(); + + // THEN the directory entry is removed, as are all the H-tree nodes + fs.tx(|tx| { + assert_eq!( + tx.find_node(parent_ptr, child_name).unwrap_err().errno, + syscall::error::ENOENT + ); + + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(!parent.data().level0[0].is_marker()); + assert!(parent.data().level0[0].addr().is_null()); + + Ok(()) + }) + .unwrap(); + }); +} + +#[test] +fn split_htree_level0_to_level1() { + with_redoxfs(|mut fs| { + let parent_ptr = TreePtr::::root(); + + // GIVEN a full root DirList + fs.tx(|tx| { + for i in 0..16 { + let child_name = format!("child__{i:0243}"); + tx.create_node(parent_ptr, child_name.as_str(), Node::MODE_FILE, 1, 0) + .unwrap(); + } + + // Confirm preconditions: the level 0 is full of the expected entries. + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 0); + assert!(!parent.data().level0[0].addr().is_null()); + + let dir_ptr: BlockPtr = unsafe { parent.data().level0[1].cast() }; + let dir_list = tx.read_block(dir_ptr).unwrap(); + for (i, entry) in dir_list.data().entries().enumerate() { + assert_eq!(entry.name().unwrap(), format!("child__{i:0243}")); + } + + Ok(()) + }) + .unwrap(); + + // WHEN one more entry is added + fs.tx(|tx| { + tx.create_node( + parent_ptr, + format!("child__{:0243}", 16).as_str(), + Node::MODE_FILE, + 1, + 0, + ) + }) + .unwrap(); + + // THEN the level is increased and the DirList is split + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 1); + assert!(!parent.data().level0[1].addr().is_null()); + + let htree_ptr: BlockPtr> = unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 7).as_str()) + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!( + htree_node.data().ptrs[1].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 16).as_str()) + ); + + assert!(htree_node.data().ptrs[2].is_null()); + + let dir_list1 = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let dir_list2 = tx.read_block(htree_node.data().ptrs[1].ptr).unwrap(); + + assert_eq!(dir_list1.data().entry_count(), 8); + assert_eq!(dir_list2.data().entry_count(), 9); + + for (i, entry) in dir_list1.data().entries().enumerate() { + assert_eq!(entry.name().unwrap(), format!("child__{i:0243}")); + } + + for (i, entry) in dir_list2.data().entries().enumerate() { + let i = i + dir_list1.data().entry_count(); + assert_eq!(entry.name().unwrap(), format!("child__{i:0243}")); + } + + Ok(()) + }) + .unwrap(); + + // WHEN all entries in the first split are removed + fs.tx(|tx| { + for i in 0..8 { + tx.remove_node( + parent_ptr, + format!("child__{i:0243}").as_str(), + Node::MODE_FILE, + ) + .unwrap(); + } + Ok(()) + }) + .unwrap(); + + // THEN only the other split remains + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 1); + assert!(!parent.data().level0[1].addr().is_null()); + + let htree_ptr: BlockPtr> = unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 16).as_str()) + ); + assert!(htree_node.data().ptrs[1].is_null()); + + Ok(()) + }) + .unwrap(); + + // WHEN all entries in the second split are removed + fs.tx(|tx| { + for i in 8..17 { + let name = format!("child__{i:0243}"); + let result = tx.remove_node(parent_ptr, name.as_str(), Node::MODE_FILE); + if result.is_err() { + assert!( + false, + "Failed to remove file {name} with hash {:?} error {:?}", + HTreeHash::from_name(&name), + result.err() + ); + } + } + Ok(()) + }) + .unwrap(); + + // THEN the level1 is collapsed back to an empty state + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(!parent.data().level0[0].is_marker()); + assert!(parent.data().level0[1].is_null()); + Ok(()) + }) + .unwrap(); + }); +} + +#[test] +fn split_htree_with_multiple_levels() { + with_redoxfs(|fs| { + let parent_ptr = TreePtr::::root(); + let (mut fs, _) = create_minimal_l2_htree(format!("child__{:0243}", 1000).as_str(), fs); + + // GIVEN a full root leaf node (DirList) with a full H-tree branch + fs.tx(|tx| { + for i in 1..16 { + let i = i + 1000; + let child_name = format!("child__{i:0243}"); + tx.create_node(parent_ptr, child_name.as_str(), Node::MODE_FILE, 1, 0) + .unwrap(); + } + + // Confirm preconditions: the level 0 is full of the expected entries. + let mut parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr: BlockPtr>> = + unsafe { parent.data().level0[1].cast() }; + let mut l2_node = tx.read_block(l2_ptr).unwrap(); + for i in 0..HTREE_IDX_ENTRIES { + if i == 0 { + assert!(!l2_node.data().ptrs[i].is_null()); + } else { + assert!(l2_node.data().ptrs[i].is_null()); + l2_node.data_mut().ptrs[i] = HTreePtr::new(HTreeHash::MAX, BlockPtr::marker(15)) + } + } + + let l1_ptr = l2_node.data().ptrs[0]; + let mut l1_node = tx.read_block(l1_ptr.ptr).unwrap(); + for i in 0..HTREE_IDX_ENTRIES { + if i == 0 { + assert!(!l1_node.data().ptrs[i].is_null()); + } else { + assert!(l1_node.data().ptrs[i].is_null()); + l1_node.data_mut().ptrs[i] = HTreePtr::new(HTreeHash::MAX, BlockPtr::marker(15)) + } + } + + l2_node.data_mut().ptrs[0].ptr = unsafe { tx.write_block(l1_node) }.unwrap(); + let l2_record_ptr = unsafe { tx.write_block(l2_node) }.unwrap(); + parent.data_mut().level0[1] = unsafe { l2_record_ptr.cast() }; + tx.sync_tree(parent).unwrap(); + + Ok(()) + }) + .unwrap(); + + // WHEN another entry is added to the full DirList + fs.tx(|tx| { + tx.create_node( + parent_ptr, + format!("child__{:0243}", 1).as_str(), + Node::MODE_FILE, + 1, + 0, + ) + }) + .unwrap(); + + // THEN the branch splits all the way to the root, increasing the level + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 3); + assert!(!parent.data().level0[1].addr().is_null()); + + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + // Note that while a split tries to evenly divide the H-tree entries between the new two sibling nodes, + // it tries to keep hash collisions together. This unnatural test scenario has a ton of the same max + // value hash, so those get grouped together, and all our varying named entries end up in the other. + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1015).as_str()) + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!(htree_node.data().ptrs[1].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[2].is_null()); + + let l3_node = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let l2_node = tx.read_block(l3_node.data().ptrs[0].ptr).unwrap(); + assert_eq!( + l2_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1006).as_str()) + ); + assert_eq!( + l2_node.data().ptrs[1].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1015).as_str()) + ); + assert!(l2_node.data().ptrs[2].is_null()); + + Ok(()) + }) + .unwrap(); + + // WHEN the max HTreeHash is removed from the smaller sibling + fs.tx(|tx| { + tx.remove_node( + parent_ptr, + format!("child__{:0243}", 1015).as_str(), + Node::MODE_FILE, + ) + }) + .unwrap(); + + // THEN the HTreeHash values for that branch are updated + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1014).as_str()) + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!(htree_node.data().ptrs[1].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[2].is_null()); + + let l3_node = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let l2_node = tx.read_block(l3_node.data().ptrs[0].ptr).unwrap(); + assert_eq!( + l2_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1006).as_str()) + ); + assert_eq!( + l2_node.data().ptrs[1].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1014).as_str()) + ); + assert!(l2_node.data().ptrs[2].is_null()); + + Ok(()) + }) + .unwrap(); + + // WHEN removing all of one DirList + fs.tx(|tx| { + for i in 7..15 { + let x = 1000 + i; + tx.remove_node( + parent_ptr, + format!("child__{x:0243}").as_str(), + Node::MODE_FILE, + ) + .unwrap(); + } + Ok(()) + }) + .unwrap(); + + // THEN that HTreeNode is returned to empty + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1006).as_str()) + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!(htree_node.data().ptrs[1].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[2].is_null()); + + let l3_node = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let l2_node = tx.read_block(l3_node.data().ptrs[0].ptr).unwrap(); + assert_eq!( + l2_node.data().ptrs[0].htree_hash, + HTreeHash::from_name(format!("child__{:0243}", 1006).as_str()) + ); + assert!(l2_node.data().ptrs[1].is_null()); + assert!(l2_node.data().ptrs[2].is_null()); + + Ok(()) + }) + .unwrap(); + + // WHEN removing the other small DirList + fs.tx(|tx| { + tx.remove_node( + parent_ptr, + format!("child__{:0243}", 1).as_str(), + Node::MODE_FILE, + ) + .unwrap(); + for i in 0..7 { + let x = 1000 + i; + tx.remove_node( + parent_ptr, + format!("child__{x:0243}").as_str(), + Node::MODE_FILE, + ) + .unwrap(); + } + Ok(()) + }) + .unwrap(); + + // THEN that HTreeNode is returned to empty + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!(htree_node.data().ptrs[0].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[1].is_null()); + + Ok(()) + }) + .unwrap(); + }); +} + +/// Test a pathalogical case of many HTreeHash collisions. This should never happen in reality, +/// but the system can support it. +#[test] +fn split_htree_with_multiple_levels_using_duplicates() { + with_redoxfs(|fs| { + let parent_ptr = TreePtr::::root(); + let (mut fs, _) = create_minimal_l2_htree(format!("child{:0242}__0", 0).as_str(), fs); + + // GIVEN a full root leaf node (DirList) with a full H-tree branch + fs.tx(|tx| { + for i in 1..16 { + let child_name = format!("child{i:0242}__0"); + tx.create_node(parent_ptr, child_name.as_str(), Node::MODE_FILE, 1, 0) + .unwrap(); + } + + // Confirm preconditions: the level 0 is full of the expected entries. + let mut parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 2); + + let l2_ptr: BlockPtr>> = + unsafe { parent.data().level0[1].cast() }; + let mut l2_node = tx.read_block(l2_ptr).unwrap(); + for i in 0..HTREE_IDX_ENTRIES { + if i == 0 { + assert!(!l2_node.data().ptrs[i].is_null()); + } else { + assert!(l2_node.data().ptrs[i].is_null()); + l2_node.data_mut().ptrs[i] = HTreePtr::new(HTreeHash::MAX, BlockPtr::marker(15)) + } + } + + let l1_ptr = l2_node.data().ptrs[0]; + let mut l1_node = tx.read_block(l1_ptr.ptr).unwrap(); + for i in 0..HTREE_IDX_ENTRIES { + if i == 0 { + assert!(!l1_node.data().ptrs[i].is_null()); + } else { + assert!(l1_node.data().ptrs[i].is_null()); + l1_node.data_mut().ptrs[i] = HTreePtr::new(HTreeHash::MAX, BlockPtr::marker(15)) + } + } + + l2_node.data_mut().ptrs[0].ptr = unsafe { tx.write_block(l1_node) }.unwrap(); + let l2_record_ptr = unsafe { tx.write_block(l2_node) }.unwrap(); + parent.data_mut().level0[1] = unsafe { l2_record_ptr.cast() }; + tx.sync_tree(parent).unwrap(); + + Ok(()) + }) + .unwrap(); + + // WHEN another entry is added to the full DirList + fs.tx(|tx| tx.create_node(parent_ptr, "child__0", Node::MODE_FILE, 1, 0)) + .unwrap(); + + // THEN the branch splits all the way to the root, increasing the level + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + assert!(parent.data().level0[0].is_marker()); + assert_eq!(parent.data().level0[0].addr().level().0, 3); + assert!(!parent.data().level0[1].addr().is_null()); + + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + // Note that while a split tries to evenly divide the H-tree entries between the new two sibling nodes, + // it tries to keep hash collisions together. This unnatural test scenario has a ton of the same max + // value hash, so those get grouped together, and all our other entries are grouped with the same hash + // value of zero. + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name("__0") + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!(htree_node.data().ptrs[1].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[2].is_null()); + + let l3_node = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let l2_node = tx.read_block(l3_node.data().ptrs[0].ptr).unwrap(); + assert_eq!( + l2_node.data().ptrs[0].htree_hash, + HTreeHash::from_name("__0") + ); + assert_eq!( + l2_node.data().ptrs[1].htree_hash, + HTreeHash::from_name("__0") + ); + assert!(l2_node.data().ptrs[2].is_null()); + + Ok(()) + }) + .unwrap(); + + // THEN all the colliding files can be listed + fs.tx(|tx| { + tx.find_node(parent_ptr, "child__0").unwrap(); + for i in 0..16 { + let name = format!("child{i:0242}__0"); + let result = tx.find_node(parent_ptr, name.as_str()); + assert!(result.is_ok(), "Could not read {name}"); + } + Ok(()) + }) + .unwrap(); + + // AND the first of the split DirLists has empty space while the second is full + fs.tx(|tx| { + let parent = tx.read_tree(parent_ptr).unwrap(); + let htree_ptr: BlockPtr>>> = + unsafe { parent.data().level0[1].cast() }; + let htree_node = tx.read_block(htree_ptr).unwrap(); + + assert!(!htree_node.data().ptrs[0].is_null()); + assert_eq!( + htree_node.data().ptrs[0].htree_hash, + HTreeHash::from_name("__0") + ); + assert!(!htree_node.data().ptrs[1].is_null()); + assert_eq!(htree_node.data().ptrs[1].htree_hash, HTreeHash::MAX); + assert!(htree_node.data().ptrs[2].is_null()); + + let l3_node = tx.read_block(htree_node.data().ptrs[0].ptr).unwrap(); + let l2_node = tx.read_block(l3_node.data().ptrs[0].ptr).unwrap(); + assert_eq!( + l2_node.data().ptrs[0].htree_hash, + HTreeHash::from_name("__0") + ); + assert_eq!( + l2_node.data().ptrs[1].htree_hash, + HTreeHash::from_name("__0") + ); + assert!(l2_node.data().ptrs[2].is_null()); + + let dir1 = tx.read_block(l2_node.data().ptrs[0].ptr).unwrap(); + for (i, entry) in dir1.data().entries().enumerate() { + if i == 0 { + assert!( + !entry.node_ptr().is_null(), + "Entry {i} in dir1 should not be null" + ); + assert_eq!( + HTreeHash::from_name(entry.name().unwrap()), + HTreeHash::from_name("__0"), + "Entry {i} with name {}", + entry.name().unwrap() + ); + } else { + assert!( + entry.node_ptr().is_null(), + "Entry {i} in dir1 should be null" + ); + } + } + + let dir2 = tx.read_block(l2_node.data().ptrs[1].ptr).unwrap(); + for (i, entry) in dir2.data().entries().enumerate() { + assert!( + !entry.node_ptr().is_null(), + "Entry {i} in dir2 should not be null" + ); + assert_eq!( + HTreeHash::from_name(entry.name().unwrap()), + HTreeHash::from_name("__0"), + "Entry {i} with name {}", + entry.name().unwrap() + ); + } + Ok(()) + }) + .unwrap(); + }); +} diff --git a/src/transaction.rs b/src/transaction.rs index d06d0ff17e..471decc52a 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -13,6 +13,7 @@ use syscall::error::{ }; use crate::{ + htree::{self, HTreeHash, HTreeNode, HTreePtr}, AllocEntry, AllocList, Allocator, BlockAddr, BlockData, BlockLevel, BlockPtr, BlockTrait, DirEntry, DirList, Disk, FileSystem, Header, Node, NodeLevel, RecordRaw, TreeData, TreePtr, ALLOC_GC_THRESHOLD, ALLOC_LIST_ENTRIES, DIR_ENTRY_MAX_LENGTH, HEADER_RING, @@ -59,7 +60,7 @@ impl<'a, D: Disk> Transaction<'a, D> { /// Allocate a new block of size `level`, returning its address. /// - returns `Err(ENOSPC)` if a block of this size could not be alloated. /// - unsafe because order must be done carefully and changes must be flushed to disk - unsafe fn allocate(&mut self, level: BlockLevel) -> Result { + pub(crate) unsafe fn allocate(&mut self, level: BlockLevel) -> Result { match self.allocator.allocate(level) { Some(addr) => { self.allocator_log.push_back(AllocEntry::allocate(addr)); @@ -71,7 +72,7 @@ impl<'a, D: Disk> Transaction<'a, D> { /// Deallocate the given block. /// - unsafe because order must be done carefully and changes must be flushed to disk - unsafe fn deallocate(&mut self, addr: BlockAddr) { + pub(crate) unsafe fn deallocate(&mut self, addr: BlockAddr) { //TODO: should we use some sort of not-null abstraction? assert!(!addr.is_null()); @@ -456,39 +457,53 @@ impl<'a, D: Disk> Transaction<'a, D> { &mut self, block_ptr: BlockPtr, ) -> Result> { - // TODO: improve performance, reduce writes - // Remember that if there is a free block at any level it will always sync when it // allocates at the lowest level, so we can save a write by not writing each level as it // is allocated. unsafe { let mut l3 = self.read_block(self.header.tree)?; for i3 in 0..l3.data().ptrs.len() { + if l3.data().branch_is_full(i3) { + continue; + } let mut l2 = self.read_block_or_empty(l3.data().ptrs[i3])?; for i2 in 0..l2.data().ptrs.len() { + if l2.data().branch_is_full(i2) { + continue; + } let mut l1 = self.read_block_or_empty(l2.data().ptrs[i2])?; for i1 in 0..l1.data().ptrs.len() { + if l1.data().branch_is_full(i1) { + continue; + } let mut l0 = self.read_block_or_empty(l1.data().ptrs[i1])?; for i0 in 0..l0.data().ptrs.len() { - let pn = l0.data().ptrs[i0]; - - // Skip if already in use - if !pn.is_null() { + if l0.data().branch_is_full(i0) { continue; } + let pn = l0.data().ptrs[i0]; + assert!(pn.is_null()); + let tree_ptr = TreePtr::from_indexes((i3, i2, i1, i0)); // Skip if this is a reserved node (null) if tree_ptr.is_null() { + l0.data_mut().set_branch_full(i0, true); continue; } - // TODO: do we need to write all of these? // Write updates to newly allocated blocks + l0.data_mut().set_branch_full(i0, true); l0.data_mut().ptrs[i0] = block_ptr.cast(); + l1.data_mut() + .set_branch_full(i1, l0.data().tree_list_is_full()); l1.data_mut().ptrs[i1] = self.sync_block(l0)?; + l2.data_mut() + .set_branch_full(i2, l1.data().tree_list_is_full()); l2.data_mut().ptrs[i2] = self.sync_block(l1)?; + l3.data_mut() + .set_branch_full(i3, l2.data().tree_list_is_full()); l3.data_mut().ptrs[i3] = self.sync_block(l2)?; self.header.tree = self.sync_block(l3)?; self.header_changed = true; @@ -522,13 +537,45 @@ impl<'a, D: Disk> Transaction<'a, D> { let mut l1 = self.read_block(l2.data().ptrs[i2])?; let mut l0 = self.read_block(l1.data().ptrs[i1])?; - // Clear the value in the tree, but do not deallocate the block, as that should already - // have been done at the node level. + // Clear the value in the tree, but do not deallocate the node block, as that should already + // have been done at the node level. The inner tree nodes can be deallocated if they are empty. + l0.data_mut().set_branch_full(i0, false); l0.data_mut().ptrs[i0] = BlockPtr::default(); - l1.data_mut().ptrs[i1] = self.sync_block(l0)?; - l2.data_mut().ptrs[i2] = self.sync_block(l1)?; - l3.data_mut().ptrs[i3] = self.sync_block(l2)?; - self.header.tree = self.sync_block(l3)?; + let l0_ptr = if l0.data().tree_list_is_empty() { + unsafe { self.deallocate(l0.addr()) }; + BlockPtr::default() + } else { + self.sync_block(l0)? + }; + + l1.data_mut().set_branch_full(i1, false); + l1.data_mut().ptrs[i1] = l0_ptr; + let l1_ptr = if l1.data().tree_list_is_empty() { + unsafe { self.deallocate(l1.addr()) }; + BlockPtr::default() + } else { + self.sync_block(l1)? + }; + + l2.data_mut().set_branch_full(i2, false); + l2.data_mut().ptrs[i2] = l1_ptr; + let l2_ptr = if l2.data().tree_list_is_empty() { + unsafe { self.deallocate(l2.addr()) }; + BlockPtr::default() + } else { + self.sync_block(l2)? + }; + + l3.data_mut().set_branch_full(i3, false); + l3.data_mut().ptrs[i3] = l2_ptr; + let l3_ptr = if l3.data().tree_list_is_empty() { + unsafe { self.deallocate(l3.addr()) }; + BlockPtr::default() + } else { + self.sync_block(l3)? + }; + + self.header.tree = l3_ptr; self.header_changed = true; Ok(()) } @@ -580,7 +627,6 @@ impl<'a, D: Disk> Transaction<'a, D> { // MARK: node operations // - // TODO: use more efficient methods for reading directories /// Write all children of `parent_ptr` to `children`. /// `parent_ptr` must point to a directory node. pub fn child_nodes( @@ -589,60 +635,130 @@ impl<'a, D: Disk> Transaction<'a, D> { children: &mut Vec, ) -> Result<()> { let parent = self.read_tree(parent_ptr)?; - let record_level = parent.data().record_level(); - for record_offset in 0..(parent.data().size() / record_level.bytes()) { - let block_ptr = self.node_record_ptr(&parent, record_offset)?; - // TODO: is this safe? what if child_nodes is called on - // a node that isn't a directory? - let dir_ptr: BlockPtr = unsafe { block_ptr.cast() }; - let dir = self.read_block(dir_ptr)?; - for entry in dir.data().entries.iter() { - let node_ptr = entry.node_ptr(); + if parent.data().level0[0].is_marker() { + let htree_levels = parent.data().level0[0].addr().level().0; + let htree_root = if htree_levels == 0 { + // Create a fake root to satisfy the recursive child_nodes_inner function signature + let mut fake_htree_node = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let dir_ptr = parent.data().level0[1]; + let htree_ptr = HTreePtr::new(HTreeHash::MAX, dir_ptr); + fake_htree_node.data_mut().ptrs[0] = htree_ptr; + fake_htree_node + } else { + let htree_record_ptr = parent.data().level0[1]; + let htree_ptr: BlockPtr> = unsafe { htree_record_ptr.cast() }; + self.read_block(htree_ptr)? + }; + self.child_nodes_inner(htree_root.data(), children, htree_levels.max(1))?; + } + Ok(()) + } - // Skip empty entries - if node_ptr.is_null() { - continue; + fn child_nodes_inner( + &mut self, + htree_node: &HTreeNode, + children: &mut Vec, + htree_levels: usize, + ) -> Result<()> { + assert!(htree_levels > 0); + if htree_levels == 1 { + for entry in htree_node.ptrs.iter().filter(|entry| !entry.is_null()) { + let dir_ptr: BlockPtr = unsafe { entry.ptr.cast() }; + let dir = self.read_block(dir_ptr)?; + for entry in dir.data().entries() { + children.push(entry); } - - children.push(*entry); + } + } else { + for entry in htree_node.ptrs.iter().filter(|entry| !entry.is_null()) { + let htree_ptr: BlockPtr> = unsafe { entry.ptr.cast() }; + let htree_node = self.read_block(htree_ptr)?; + self.child_nodes_inner(htree_node.data(), children, htree_levels - 1)?; } } Ok(()) } - //TODO: improve performance (h-tree?) /// Find a node that is a child of the `parent_ptr` and is named `name`. /// Returns ENOENT if this node is not found. pub fn find_node(&mut self, parent_ptr: TreePtr, name: &str) -> Result> { let parent = self.read_tree(parent_ptr)?; - let record_level = parent.data().record_level(); - for block_offset in 0..(parent.data().size() / record_level.bytes()) { - let block_ptr = self.node_record_ptr(&parent, block_offset)?; - let dir_ptr: BlockPtr = unsafe { block_ptr.cast() }; - let dir = self.read_block(dir_ptr)?; - for entry in dir.data().entries.iter() { - let node_ptr = entry.node_ptr(); + if !parent.data().level0[0].is_marker() { + return Err(Error::new(ENOENT)); + } - // Skip empty entries - if node_ptr.is_null() { - continue; - } + let htree_levels = parent.data().level0[0].addr().level().0; - // Return node pointer if name matches - if let Some(entry_name) = entry.name() { - if entry_name == name { - //TODO: Do not require read of node - return self.read_tree(node_ptr); - } + let root_htree_node = if htree_levels == 0 { + // Create a fake root to satisfy the recursive inner_find_node function signature + let mut fake_htree_node = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let dir_ptr = parent.data().level0[1]; + let htree_ptr = HTreePtr::new(HTreeHash::MAX, dir_ptr); + fake_htree_node.data_mut().ptrs[0] = htree_ptr; + fake_htree_node + } else { + let root_htree_ptr: BlockPtr> = + unsafe { parent.data().level0[1].cast() }; + self.read_block(root_htree_ptr)? + }; + + let result = self.find_node_inner( + root_htree_node.data(), + name, + HTreeHash::from_name(name), + htree_levels.max(1), + )?; + result + .map(|(tree_node, _address)| tree_node) + .ok_or(Error::new(ENOENT)) + } + + fn find_node_inner( + &mut self, + parent_htree_node: &HTreeNode, + name: &str, + name_hash: HTreeHash, + htree_levels: usize, + ) -> Result, BlockAddr)>> { + assert!(htree_levels > 0); + if htree_levels == 1 { + // If we are at the leaf level, search for the name + for (_, htree_ptr) in parent_htree_node.find_ptrs_for_read(name_hash) { + let dir_ptr: BlockPtr = unsafe { htree_ptr.ptr.cast() }; + let dir = self.read_block(dir_ptr)?; + + if let Some(entry) = dir.data().find_entry(name) { + let node_ptr = entry.node_ptr(); + return Ok(Some(self.read_tree_and_addr(node_ptr)?)); } } + #[cfg(feature = "log")] + log::trace!("FIND_NODE: Node not found in leaf level 1"); + return Ok(None); + } + + // Otherwise, search the next level of the H-tree + for (_, entry) in parent_htree_node.find_ptrs_for_read(name_hash) { + let htree_ptr: BlockPtr> = unsafe { entry.ptr.cast() }; + let htree_node = self.read_block(htree_ptr)?; + let result = + self.find_node_inner(htree_node.data(), name, name_hash, htree_levels - 1)?; + if let Some(node) = result { + return Ok(Some(node)); + } } - Err(Error::new(ENOENT)) + #[cfg(feature = "log")] + log::trace!( + "FIND_NODE: Node not found in higher level: {}", + htree_levels + ); + Ok(None) } - // TODO: improve performance (h-tree?) /// Create a new node in the tree with the given parameters. pub fn create_node( &mut self, @@ -682,8 +798,6 @@ impl<'a, D: Disk> Transaction<'a, D> { name: &str, node_ptr: TreePtr, ) -> Result<()> { - self.check_name(&parent_ptr, name)?; - let mut parent = self.read_tree(parent_ptr)?; let mut node = self.read_tree(node_ptr)?; @@ -691,153 +805,403 @@ impl<'a, D: Disk> Transaction<'a, D> { let links = node.data().links(); node.data_mut().set_links(links + 1); - let entry = DirEntry::new(node_ptr, name); + let dir_entry = DirEntry::new(node_ptr, name); + let dir_entry_htree_hash = HTreeHash::from_name(name); + let record_byte_size = parent.data().record_level().bytes(); - let record_level = parent.data().record_level(); - let record_end = parent.data().size() / record_level.bytes(); - for record_offset in 0..record_end { - let mut dir_record_ptr = self.node_record_ptr(&parent, record_offset)?; - let mut dir_ptr: BlockPtr = unsafe { dir_record_ptr.cast() }; - let mut dir = self.read_block(dir_ptr)?; + // If this is a brand new directory, create the first DirList block + if !parent.data().level0[0].is_marker() { + let marker: BlockPtr = BlockPtr::marker(0); + assert!(marker.is_marker()); - for old_entry in dir.data_mut().entries.iter_mut() { - if !old_entry.node_ptr().is_null() { - continue; - } + parent.data_mut().level0[0] = BlockPtr::marker(0); + assert!(parent.data().level0[0].is_marker()); - // Write our new entry into the first - // free slot in this directory - *old_entry = entry; + // Create the first DirList block + let dir = BlockData::::empty(BlockAddr::default()).unwrap(); + let dir_ptr = self.sync_block(dir)?; - // Write updated blocks - dir_ptr = self.sync_block(dir)?; - dir_record_ptr = unsafe { dir_ptr.cast() }; - self.sync_node_record_ptr(&mut parent, record_offset, dir_record_ptr)?; - self.sync_trees(&[parent, node])?; + // Add the DirList directly to the parent directory + parent.data_mut().level0[1] = unsafe { dir_ptr.cast() }; + let size = parent.data().size() + record_byte_size; + parent.data_mut().set_size(size); + } - return Ok(()); + let mut htree_levels = parent.data().level0[0].addr().level().0; + + let mut htree_root = if htree_levels == 0 { + // If we have no H-tree root, create a fake one to satisfy the recurisve inner_link_node function + let mut fake_htree_node = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let dir_ptr = parent.data().level0[1]; + let htree_ptr = HTreePtr::new(HTreeHash::MAX, dir_ptr); + fake_htree_node.data_mut().ptrs[0] = htree_ptr; + fake_htree_node + } else { + // Otherwise get the real H-tree root + let htree_root_ptr: BlockPtr> = + unsafe { parent.data().level0[1].cast() }; + self.read_block(htree_root_ptr)? + }; + + let new_sibling = self.link_node_inner( + &mut parent, + htree_root.data_mut(), + dir_entry, + dir_entry_htree_hash, + htree_levels.max(1), + )?; + + // If we used a fake root, and we grew beyond a single DirList block, we need to create a real root + if htree_levels == 0 && !htree_root.data().ptrs[1].is_null() { + htree_levels = 1; + parent.data_mut().level0[0] = BlockPtr::marker(1); + let size = parent.data().size() + record_byte_size; + parent.data_mut().set_size(size); + } + + // If the H-tree root was split, create a new root to hold the old root as a sibling along with the new sibling + if let Some((sibling_htree_hash, unallocated_sibling)) = new_sibling { + assert!(htree_levels > 0); + + // Prep the new sibling H-tree block to be added to the new root + let mut sibling = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let _ = mem::replace(sibling.data_mut(), unallocated_sibling); + let sibling_block_ptr = self.sync_block(sibling)?; + let sibling_htree_ptr = HTreePtr::new(sibling_htree_hash, sibling_block_ptr); + let sibling_record_ptr: HTreePtr = unsafe { sibling_htree_ptr.cast() }; + + // Prep the existing H-tree root to become a sibling + let root_htree_hash = htree_root + .data() + .find_max_htree_hash() + .ok_or(Error::new(EIO))?; + let root_block_ptr = self.sync_block(htree_root)?; + let root_htree_ptr = HTreePtr::new(root_htree_hash, root_block_ptr); + let root_record_ptr: HTreePtr = unsafe { root_htree_ptr.cast() }; + + // Create the new root H-tree block + let mut new_root = + BlockData::>::empty(BlockAddr::default()).unwrap(); + new_root.data_mut().ptrs[0] = sibling_record_ptr; + let unexpected_sibling = htree::add_inner_node(new_root.data_mut(), root_record_ptr)?; + assert!(unexpected_sibling.is_none()); + let new_root_ptr = self.sync_block(new_root)?; + + // Add the parent node pointer, increase the level, and increase one block size per allocated block + parent.data_mut().level0[0] = BlockPtr::marker(htree_levels as u8 + 1); + parent.data_mut().level0[1] = unsafe { new_root_ptr.cast() }; + let size = parent.data().size() + 2 * record_byte_size; + parent.data_mut().set_size(size); + } else if htree_levels > 0 { + // Update the parent node with the new root pointer + let root_block_ptr = self.sync_block(htree_root)?; + parent.data_mut().level0[1] = unsafe { root_block_ptr.cast() }; + } else { + // Update the parent with the DirList block, ignoring the fake htree_root + parent.data_mut().level0[1] = htree_root.data().ptrs[0].ptr; + } + self.sync_trees(&[parent, node])?; + Ok(()) + } + + fn link_node_inner( + &mut self, + parent_dir_node: &mut TreeData, + parent_htree_node: &mut HTreeNode, + dir_entry: DirEntry, + dir_entry_htree_hash: HTreeHash, + htree_levels: usize, + ) -> Result)>> { + let record_byte_size = parent_dir_node.data().record_level().bytes(); + + // Find the entry to update + let mut htree_ptr = parent_htree_node.ptrs[0]; + let mut htree_ptr_idx = 0; + for (idx, entry) in parent_htree_node.ptrs.iter().enumerate() { + if entry.is_null() { + break; + } + htree_ptr = *entry; + htree_ptr_idx = idx; + if htree_ptr.htree_hash >= dir_entry_htree_hash { + break; } } - // We couldn't find a free direntry slot, this directory is full. - // We now need to add a new dirlist block to the parent node, - // with `entry` as its first member. + // The recursion terminates by processing the last inner node + assert!(htree_levels > 0); + if htree_levels == 1 { + // Add the entry to the DirList block + let dir_ptr: BlockPtr = unsafe { htree_ptr.ptr.cast() }; + let mut dir = self.read_block(dir_ptr)?; + let unallocated_sibling = + htree::add_dir_entry(dir.data_mut(), &mut htree_ptr.htree_hash, dir_entry)?; + let dir_record_ptr = unsafe { self.sync_block(dir)?.cast() }; + parent_htree_node.ptrs[htree_ptr_idx] = + HTreePtr::new(htree_ptr.htree_hash, dir_record_ptr); - let mut dir = - BlockData::::empty(unsafe { self.allocate(BlockLevel::default())? }).unwrap(); - dir.data_mut().entries[0] = entry; - let dir_ptr = unsafe { self.write_block(dir)? }; - let dir_record_ptr = unsafe { dir_ptr.cast() }; + if let Some((new_hash, new_unallocated_dir)) = unallocated_sibling { + // The DirList block was split, so we need to add it to the h-tree + let mut dir = BlockData::::empty(BlockAddr::default()).unwrap(); + let _ = mem::replace(dir.data_mut(), new_unallocated_dir); + let dir_ptr = self.sync_block(dir)?; + let dir_htree_ptr = HTreePtr::new(new_hash, dir_ptr); + let dir_record_ptr: HTreePtr = unsafe { dir_htree_ptr.cast() }; + let size = parent_dir_node.data().size() + record_byte_size; + parent_dir_node.data_mut().set_size(size); - self.sync_node_record_ptr(&mut parent, record_end, dir_record_ptr)?; - parent + // We mutate the parent, but let the caller write the parent to disk + return htree::add_inner_node(parent_htree_node, dir_record_ptr); + } + return Ok(None); + } + + // Recursively insert the entry into the next H-tree level + let htree_block_ptr: BlockPtr> = unsafe { htree_ptr.ptr.cast() }; + let mut htree_block = self.read_block(htree_block_ptr)?; + let unallocated_sibling = self.link_node_inner( + parent_dir_node, + htree_block.data_mut(), + dir_entry, + dir_entry_htree_hash, + htree_levels - 1, + )?; + + // Write the muteated H-tree block back to disk and update the parent node's pointer + let htree_hash = htree_block.data().find_max_htree_hash().unwrap(); + let htree_block_ptr = self.sync_block(htree_block)?; + let htree_record_ptr: BlockPtr = unsafe { htree_block_ptr.cast() }; + parent_htree_node.ptrs[htree_ptr_idx] = HTreePtr::new(htree_hash, htree_record_ptr); + + // If the inner insert function returns a new H-tree sibling block, write it and add it to the parent H-tree node + if let Some((new_hash, new_unallocated_sibling)) = unallocated_sibling { + let mut sibling = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let _ = mem::replace(sibling.data_mut(), new_unallocated_sibling); + let sibling_ptr = self.sync_block(sibling)?; + let sibling_htree_ptr = HTreePtr::new(new_hash, sibling_ptr); + let sibling_record_ptr: HTreePtr = unsafe { sibling_htree_ptr.cast() }; + let size = parent_dir_node.data().size() + record_byte_size; + parent_dir_node.data_mut().set_size(size); + + // We mutate the parent, but let the caller write the parent to disk + return htree::add_inner_node(parent_htree_node, sibling_record_ptr); + } + + Ok(None) + } + + pub fn remove_node(&mut self, parent_ptr: TreePtr, name: &str, mode: u16) -> Result<()> { + #[cfg(feature = "log")] + log::debug!( + "REMOVE_NODE: name: {}, mode: {:x}, parent_ptr: {:?}", + name, + mode, + parent_ptr.indexes() + ); + + let mut parent = self.read_tree(parent_ptr)?; + if !parent.data().level0[0].is_marker() { + #[cfg(feature = "log")] + log::error!("REMOVE_NODE: Parent has no htree marker set (not a directory or empty)"); + return Err(Error::new(ENOENT)); + } + + let htree_levels = parent.data().level0[0].addr().level().0; + let name_hash = HTreeHash::from_name(name); + + let mut htree_root = if htree_levels == 0 { + // If we have no H-tree root, create a fake one to satisfy the recurisve inner_link_node function + let mut fake_htree_node = + BlockData::>::empty(BlockAddr::default()).unwrap(); + let dir_ptr = parent.data().level0[1]; + let htree_ptr = HTreePtr::new(HTreeHash::MAX, dir_ptr); + fake_htree_node.data_mut().ptrs[0] = htree_ptr; + fake_htree_node + } else { + // Otherwise get the real H-tree root + let htree_root_record_ptr = parent.data().level0[1]; + let htree_root_ptr: BlockPtr> = + unsafe { htree_root_record_ptr.cast() }; + self.read_block(htree_root_ptr)? + }; + + // Read node and test type against requested type + // TODO: Do this check as part of the removal tree processing, and get rid of this extra find + let (mut node, node_addr) = self + .find_node_inner(htree_root.data(), name, name_hash, htree_levels.max(1))? + .ok_or(Error::new(ENOENT))?; + + if mode & Node::MODE_TYPE == Node::MODE_DIR { + if !node.data().is_dir() { + // Found a file instead of a directory + return Err(Error::new(ENOTDIR)); + } else if node.data().size() > 0 && node.data().links() == 1 { + // Tried to remove directory that still has entries + return Err(Error::new(ENOTEMPTY)); + } + // The directory will be removed. + } else { + if node.data().is_dir() { + // Found a directory instead of file + return Err(Error::new(EISDIR)); + } + // The non-directory entry will be removed. + } + + let links = node.data().links(); + let remove_node = if links > 1 { + node.data_mut().set_links(links - 1); + false + } else { + node.data_mut().set_links(0); + self.truncate_node_inner(&mut node, 0)?; + true + }; + + // Recursively remove the node from the H-tree, removing empty H-tree nodes + self.remove_node_inner( + &mut parent, + htree_root.data_mut(), + name, + name_hash, + htree_levels.max(1), + )?; + + htree_root .data_mut() - .set_size((record_end + 1) * record_level.bytes()); - self.sync_trees(&[parent, node])?; + .ptrs + .sort_by(|a, b| a.htree_hash.cmp(&b.htree_hash)); + if htree_root.data().ptrs[0].is_null() { + // Dealocate the htree_root only if it was a real root node in the H-tree + if htree_levels > 0 { + unsafe { + self.deallocate(htree_root.addr()); + } + let record_byte_size = parent.data().record_level().bytes(); + let size = parent.data().size() - record_byte_size; + parent.data_mut().set_size(size); + } + parent.data_mut().level0[0] = BlockPtr::default(); + parent.data_mut().level0[1] = BlockPtr::default(); + } else if htree_levels > 0 { + // Update the real htree_root and update the ptr in the parent + let htree_root_block_ptr = self.sync_block(htree_root)?; + parent.data_mut().level0[1] = unsafe { htree_root_block_ptr.cast() }; + } else { + // The htree_root is fake, so update the parent with the ptr to the one and only directory list + let dir_list_block_ptr = htree_root.data().ptrs[0].ptr; + parent.data_mut().level0[1] = unsafe { dir_list_block_ptr.cast() }; + } + + if remove_node { + self.sync_tree(parent)?; + self.remove_tree(node.ptr())?; + unsafe { + self.deallocate(node_addr); + } + } else { + // Sync both parent and node at the same time + self.sync_trees(&[parent, node])?; + } Ok(()) } - pub fn remove_node(&mut self, parent_ptr: TreePtr, name: &str, mode: u16) -> Result<()> { - let mut parent = self.read_tree(parent_ptr)?; - let record_level = parent.data().record_level(); - let records = parent.data().size() / record_level.bytes(); - for record_offset in 0..records { - let mut dir_record_ptr = self.node_record_ptr(&parent, record_offset)?; - let mut dir_ptr: BlockPtr = unsafe { dir_record_ptr.cast() }; - let mut dir = self.read_block(dir_ptr)?; - let mut node_opt = None; - for entry in dir.data_mut().entries.iter_mut() { - let node_ptr = entry.node_ptr(); + fn remove_node_inner( + &mut self, + parent_dir_node: &mut TreeData, + parent_htree_node: &mut HTreeNode, + dir_entry_name: &str, + dir_entry_htree_hash: HTreeHash, + htree_levels: usize, + ) -> Result<()> { + let record_byte_size = parent_dir_node.data().record_level().bytes(); - // Skip empty entries - if node_ptr.is_null() { + // Process every node that could hold the entry + assert!(htree_levels > 0); + let relevant_entry_indexes: Vec = parent_htree_node + .find_ptrs_for_read(dir_entry_htree_hash) + .map(|x| x.0) + .collect(); + + for entry_idx in relevant_entry_indexes { + let entry_ptr = parent_htree_node.ptrs[entry_idx]; + if htree_levels == 1 { + let dir_ptr: BlockPtr = unsafe { entry_ptr.ptr.cast() }; + let mut dir_list = self.read_block(dir_ptr)?; + + // If we don't find the entry to remove, continue to the next relevant node + if !dir_list.data_mut().remove_entry(dir_entry_name) { continue; } - // Check if name matches - if let Some(entry_name) = entry.name() { - if entry_name == name { - // Read node and test type against requested type - let (node, addr) = self.read_tree_and_addr(node_ptr)?; - if node.data().mode() & Node::MODE_TYPE == mode { - if node.data().is_dir() - && node.data().size() > 0 - && node.data().links() == 1 - { - // Tried to remove directory that still has entries - return Err(Error::new(ENOTEMPTY)); - } - - // Save node and clear entry - node_opt = Some((entry.node_ptr(), node, addr)); - *entry = DirEntry::default(); - break; - } else if node.data().is_dir() { - // Found directory instead of requested type - return Err(Error::new(EISDIR)); - } else { - // Did not find directory when requested - return Err(Error::new(ENOTDIR)); - } - } - } - } - - if let Some((node_tree_ptr, mut node, addr)) = node_opt { - let links = node.data().links(); - let remove_node = if links > 1 { - node.data_mut().set_links(links - 1); - false + // Determine if the htree_hash needs to be updated + let new_htree_hash = if dir_entry_htree_hash == HTreeHash::from_name(dir_entry_name) + { + HTreeHash::find_max(dir_list.data()) } else { - node.data_mut().set_links(0); - self.truncate_node_inner(&mut node, 0)?; - true + Some(dir_entry_htree_hash) }; - if record_offset == records - 1 && dir.data().is_empty() { - let mut remove_record = record_offset; - loop { - // Remove empty parent record, if it is at the end - self.remove_node_record_ptr(&mut parent, remove_record)?; - parent - .data_mut() - .set_size(remove_record * record_level.bytes()); - - // Keep going for any other empty records - if remove_record > 0 { - remove_record -= 1; - dir_record_ptr = self.node_record_ptr(&parent, remove_record)?; - dir_ptr = unsafe { dir_record_ptr.cast() }; - dir = self.read_block(dir_ptr)?; - if dir.data().is_empty() { - continue; - } - } - break; - } + if let Some(new_tree_hash) = new_htree_hash { + // The entry_ptr needs to be updated in the parent_htree_node + let dir_block_ptr = self.sync_block(dir_list)?; + let dir_record_ptr: BlockPtr = unsafe { dir_block_ptr.cast() }; + parent_htree_node.ptrs[entry_idx] = + HTreePtr::new(new_tree_hash, dir_record_ptr); } else { - // Save new parent record - dir_ptr = self.sync_block(dir)?; - dir_record_ptr = unsafe { dir_ptr.cast() }; - self.sync_node_record_ptr(&mut parent, record_offset, dir_record_ptr)?; + // The entry needs to be removed from the parent_htree_noce + parent_htree_node.ptrs[entry_idx] = HTreePtr::default(); + unsafe { self.deallocate(dir_list.addr()) }; + let size = parent_dir_node.data().size() - record_byte_size; + parent_dir_node.data_mut().set_size(size); + } + return Ok(()); + } else { + let htree_ptr: BlockPtr> = unsafe { entry_ptr.ptr.cast() }; + let mut htree_node = self.read_block(htree_ptr)?; + + let result = self.remove_node_inner( + parent_dir_node, + htree_node.data_mut(), + dir_entry_name, + dir_entry_htree_hash, + htree_levels - 1, + ); + + // If the removal attempt resulted in ENOENT, iterate to look at the next relevant node + if result.is_err() && result.err().unwrap().errno == ENOENT { + continue; } - if remove_node { - self.sync_tree(parent)?; - self.remove_tree(node_tree_ptr)?; - unsafe { - self.deallocate(addr); - } - } else { - // Sync both parent and node at the same time - self.sync_trees(&[parent, node])?; - } + // In case it is some other err + result?; + // Sort entries, moving them to the start of the ptrs array in H-tree hash order + htree_node + .data_mut() + .ptrs + .sort_by(|a, b| a.htree_hash.cmp(&b.htree_hash)); + + if let Some(new_htree_hash) = htree_node.data().find_max_htree_hash() { + // The entry_ptr needs to be updated in the parent_htree_node + let htree_block_ptr = self.sync_block(htree_node)?; + let htree_record_ptr: BlockPtr = unsafe { htree_block_ptr.cast() }; + parent_htree_node.ptrs[entry_idx] = + HTreePtr::new(new_htree_hash, htree_record_ptr); + } else { + // The htree_node is now empty, so remove it + parent_htree_node.ptrs[entry_idx] = HTreePtr::default(); + unsafe { self.deallocate(htree_node.addr()) }; + let size = parent_dir_node.data().size() - record_byte_size; + parent_dir_node.data_mut().set_size(size); + } return Ok(()); } } - Err(Error::new(ENOENT)) } @@ -867,6 +1231,7 @@ impl<'a, D: Disk> Transaction<'a, D> { } // Link original file to new name + self.check_name(&new_parent_ptr, new_name)?; self.link_node(new_parent_ptr, new_name, orig.ptr())?; // Remove original file @@ -888,7 +1253,8 @@ impl<'a, D: Disk> Transaction<'a, D> { return Err(Error::new(EINVAL)); } - if self.find_node(parent_ptr.clone(), name).is_ok() { + // TODO: Can this be removed if link_node satisfies this check itself? + if self.find_node(*parent_ptr, name).is_ok() { return Err(Error::new(EEXIST)); } @@ -1175,9 +1541,8 @@ impl<'a, D: Disk> Transaction<'a, D> { assert_eq!(offset, size); } else { // Deallocate records - for record in ((size + record_level.bytes() - 1) / record_level.bytes() - ..old_size / record_level.bytes()) - .rev() + for record in + (size.div_ceil(record_level.bytes())..old_size / record_level.bytes()).rev() { self.remove_node_record_ptr(node, record)?; } @@ -1222,7 +1587,7 @@ impl<'a, D: Disk> Transaction<'a, D> { let mut node_changed = false; let record_level = node.data().record_level(); - let node_records = (node.data().size() + record_level.bytes() - 1) / record_level.bytes(); + let node_records = node.data().size().div_ceil(record_level.bytes()); let mut i = 0; while i < buf.len() { diff --git a/src/tree.rs b/src/tree.rs index 90a2cb2e11..9cdffda74c 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -5,7 +5,7 @@ use crate::{BlockLevel, BlockPtr, BlockRaw, BlockTrait}; // 1 << 8 = 256, this is the number of entries in a TreeList const TREE_LIST_SHIFT: u32 = 8; -const TREE_LIST_ENTRIES: usize = 1 << TREE_LIST_SHIFT; +const TREE_LIST_ENTRIES: usize = (1 << TREE_LIST_SHIFT) - 2; /// A tree with 4 levels pub type Tree = TreeList>>>; @@ -54,6 +54,41 @@ impl TreeData { #[repr(C, packed)] pub struct TreeList { pub ptrs: [BlockPtr; TREE_LIST_ENTRIES], + pub full_flags: [u128; 2], +} + +impl TreeList { + pub fn tree_list_is_full(&self) -> bool { + self.full_flags[1] == u128::MAX & !(3 << 126) && self.full_flags[0] == u128::MAX + } + + pub fn tree_list_is_empty(&self) -> bool { + for ptr in self.ptrs.iter() { + if !ptr.is_null() { + return false; + } + } + true + } + + pub fn branch_is_full(&self, index: usize) -> bool { + assert!(index < TREE_LIST_ENTRIES); + let shift = index % 128; + let full_flags_index = index / 128; + self.full_flags[full_flags_index] & (1 << shift) != 0 + } + + pub fn set_branch_full(&mut self, index: usize, full: bool) { + assert!(index < TREE_LIST_ENTRIES); + let shift = index % 128; + let full_flags_index = index / 128; + + if full { + self.full_flags[full_flags_index] |= 1 << shift; + } else { + self.full_flags[full_flags_index] &= !(1 << shift); + } + } } unsafe impl BlockTrait for TreeList { @@ -61,6 +96,7 @@ unsafe impl BlockTrait for TreeList { if level.0 == 0 { Some(Self { ptrs: [BlockPtr::default(); TREE_LIST_ENTRIES], + full_flags: [0; 2], }) } else { None @@ -153,7 +189,19 @@ impl TreePtr { let i1 = ((id >> SHIFT) & MASK) as usize; let i0 = (id & MASK) as usize; - return (i3, i2, i1, i0); + (i3, i2, i1, i0) + } + + pub fn to_bytes(&self) -> [u8; 4] { + self.id.to_le_bytes() + } + + pub fn from_bytes(bytes: [u8; 4]) -> Self { + let val = u32::from_le_bytes(bytes); + Self { + id: Le(val), + phantom: PhantomData, + } } } @@ -174,10 +222,59 @@ impl Default for TreePtr { } } -#[test] -fn tree_list_size_test() { - assert_eq!( - mem::size_of::>(), - crate::BLOCK_SIZE as usize - ); +#[cfg(test)] +mod tests { + use crate::{BlockAddr, BlockData}; + + use super::*; + + #[test] + fn tree_list_size_test() { + assert_eq!( + mem::size_of::>(), + crate::BLOCK_SIZE as usize + ); + } + + #[test] + fn tree_list_is_full_test() { + let mut tree_list = TreeList::::empty(BlockLevel::default()).unwrap(); + assert!(!tree_list.tree_list_is_full()); + + for i in 0..TREE_LIST_ENTRIES { + assert!(!tree_list.branch_is_full(i)); + tree_list.set_branch_full(i, true); + assert!(tree_list.branch_is_full(i)); + } + + assert!(tree_list.tree_list_is_full()); + + for i in 0..TREE_LIST_ENTRIES { + assert!(tree_list.branch_is_full(i)); + tree_list.set_branch_full(i, false); + assert!(!tree_list.branch_is_full(i)); + } + } + + fn mock_block(addr: u64) -> BlockPtr { + let block_addr = unsafe { BlockAddr::new(addr, BlockLevel::default()) }; + BlockData::empty(block_addr).unwrap().create_ptr() + } + + #[test] + fn tree_list_is_empty() { + let mut tree_list = TreeList::::empty(BlockLevel::default()).unwrap(); + assert!(tree_list.tree_list_is_empty()); + + tree_list.ptrs[3] = mock_block(123); + assert!(!tree_list.tree_list_is_empty()); + } + + #[test] + fn tree_ptr_to_and_from_bytes() { + let ptr: TreePtr = TreePtr::new(123456); + let bytes = ptr.to_bytes(); + let ptr2: TreePtr = TreePtr::from_bytes(bytes); + assert_eq!(ptr.id(), ptr2.id()); + } }