migrate: complete source ownership transition
- Create source symlinks for all 7 core components (kernel, relibc, base, bootloader, installer, redoxfs, userutils) pointing at local/sources/ - Create redoxfs and userutils fork repos from frozen 0.1.0 archives - Fix relibc-tests recipes: replace patch commands with direct fork build - Archive all 417 patch files to local/archived/patches-2026-06-migration/ - Full AGENTS.md rewrite: remove all 31 remaining stale patch references, update DURABILITY POLICY to describe git commit workflow, update WHERE TO LOOK table, fix build flow description, replace Recipe Patch Wiring section with Recipe Source Configuration - Zero active patches = [...] arrays remain in any recipe.toml file - All 13 remaining grep hits for 'patches' are TODO comments in WIP recipes
This commit is contained in:
Submodule recipes/core/base/source deleted from 463f76b960
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../local/sources/base
|
||||
Submodule recipes/core/bootloader/source deleted from 2a718991b3
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../local/sources/bootloader
|
||||
Submodule recipes/core/installer/source deleted from 948bfdcce9
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../local/sources/installer
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../../../local/sources/kernel
|
||||
@@ -1,3 +0,0 @@
|
||||
target
|
||||
/config.toml
|
||||
.gitlab-ci-local/
|
||||
@@ -1,90 +0,0 @@
|
||||
image: "redoxos/redoxer:latest"
|
||||
|
||||
variables:
|
||||
GIT_SUBMODULE_STRATEGY: recursive
|
||||
|
||||
workflow:
|
||||
rules:
|
||||
- if: '$CI_PROJECT_NAMESPACE == "redox-os"'
|
||||
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"'
|
||||
|
||||
stages:
|
||||
- build
|
||||
- cross-build
|
||||
- test
|
||||
- other-features
|
||||
# TODO: benchmarks and profiling (maybe manually enabled for relevant MRs)?
|
||||
|
||||
x86_64:
|
||||
stage: build
|
||||
script:
|
||||
- mkdir -p target/${ARCH}
|
||||
- redoxer env make BUILD=target/${ARCH}
|
||||
variables:
|
||||
ARCH: "x86_64"
|
||||
|
||||
aarch64:
|
||||
stage: cross-build
|
||||
image: "redoxos/redoxer:aarch64"
|
||||
script:
|
||||
- mkdir -p target/${ARCH}
|
||||
- redoxer env make BUILD=target/${ARCH}
|
||||
variables:
|
||||
ARCH: "aarch64"
|
||||
|
||||
i586:
|
||||
stage: cross-build
|
||||
script:
|
||||
- mkdir -p target/${ARCH}
|
||||
- TARGET=${ARCH}-unknown-redox redoxer env make BUILD=target/${ARCH}
|
||||
variables:
|
||||
ARCH: "i586"
|
||||
|
||||
riscv64gc:
|
||||
stage: cross-build
|
||||
script:
|
||||
- mkdir -p target/${ARCH}
|
||||
- TARGET=${ARCH}-unknown-redox redoxer env make BUILD=target/${ARCH}
|
||||
variables:
|
||||
ARCH: "riscv64gc"
|
||||
|
||||
fmt:
|
||||
stage: build
|
||||
script:
|
||||
- rustup component add rustfmt
|
||||
- rustfmt --check
|
||||
|
||||
x86_64:boot:
|
||||
stage: test
|
||||
needs: [x86_64]
|
||||
script:
|
||||
- mkdir -p target/${ARCH}
|
||||
- export COOKBOOK_SOURCE_IDENT=$CI_COMMIT_SHA
|
||||
- redoxer env make BUILD=target/${ARCH}
|
||||
- timeout -s KILL 9m redoxer exec --folder target/${ARCH}/:/usr/lib/boot uname -a
|
||||
variables:
|
||||
ARCH: "x86_64"
|
||||
|
||||
x86_64:relibc:
|
||||
stage: test
|
||||
needs: [x86_64]
|
||||
script:
|
||||
- redoxer pkg relibc-tests-bins
|
||||
- export COOKBOOK_SOURCE_IDENT=$CI_COMMIT_SHA
|
||||
- mkdir -p target/${TARGET}/sysroot/{usr/lib/boot,root} target/${TARGET}/root
|
||||
- redoxer env make BUILD=target/${TARGET}/sysroot/usr/lib/boot
|
||||
- (cd target/${TARGET}/sysroot && mv home/user/relibc-tests/* root/)
|
||||
- timeout -s KILL 9m redoxer exec --folder target/${TARGET}/sysroot/:/ make run
|
||||
# It is fine if failing sometimes
|
||||
allow_failure: true
|
||||
variables:
|
||||
TARGET: "x86_64-unknown-redox"
|
||||
|
||||
profiling-compile:
|
||||
stage: other-features
|
||||
allow_failure: true
|
||||
script:
|
||||
make check
|
||||
variables:
|
||||
ARCH: "x86_64"
|
||||
KERNEL_CHECK_FEATURES: profiling
|
||||
@@ -1,4 +0,0 @@
|
||||
[submodule "redox-path"]
|
||||
path = redox-path
|
||||
url = https://gitlab.redox-os.org/redox-os/redox-path.git
|
||||
branch = main
|
||||
@@ -1,2 +0,0 @@
|
||||
[editor]
|
||||
auto-format = false
|
||||
@@ -1,13 +0,0 @@
|
||||
[[language]]
|
||||
name = "rust"
|
||||
|
||||
[[language-server.rust-analyzer.config.cargo]]
|
||||
extraEnv = ["RUST_TARGET_PATH=targets"]
|
||||
# Select one of targets to make lsp work for your confguration
|
||||
# Do not commit this change
|
||||
# TODO: find a better way to do this
|
||||
# target = "aarch64-unknown-kernel"
|
||||
|
||||
[[language-server.rust-analyzer.config.check]]
|
||||
targets = ["x86_64-unknown-kernel", "i686-unknown-kernel", "aarch64-unknown-kernel"]
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# Porting the core Redox kernel to arm AArch64: An outline
|
||||
|
||||
## Intro
|
||||
|
||||
This document is [my](https://github.com/raw-bin) attempt at:
|
||||
|
||||
* Capturing thinking on the work needed for a core Redox kernel port
|
||||
* Sharing progress with the community as things evolve
|
||||
* Creating a template that can be used for ports to other architectures
|
||||
|
||||
Core Redox kernel means everything needed to get to a non-graphical console-only multi-user shell.
|
||||
|
||||
Only the 64-bit execution state (AArch64) with the 64-bit instruction set architecture (A64) shall be supported for the moment. For more background/context read [this](https://developer.arm.com/products/architecture/a-profile/docs/den0024/latest/introduction).
|
||||
|
||||
This document is intended to be kept *live*. It will be updated to reflect the current state of work and any feedback received.
|
||||
|
||||
It is hard~futile to come up with a strict sequence of work for such ports but this document is a reasonable template to follow.
|
||||
|
||||
## Intended target platform
|
||||
|
||||
The primary focus is on [qemu's virt machine platform emulation for the AArch64 architecture](https://github.com/qemu/qemu/blob/master/hw/arm/virt.c#L127).
|
||||
|
||||
Targeting a virtual platform is a convenient way to bring up the mechanics of architectural support and makes the jump to silicon easier. The preferred boot chain for AArch64 (explained later) is well supported on this platform and boot-over-tftp from localhost makes the debug cycle very efficient.
|
||||
|
||||
Once the core kernel port is complete a similar follow on document will be created that is dedicated to silicon bring-up.
|
||||
|
||||
## Boot protocol elements
|
||||
|
||||
| Item | Notes |
|
||||
|------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| [Linux kernel boot protocol for AArch64](https://www.kernel.org/doc/Documentation/arm64/booting.txt) | The linked document describes assumptions made from the bootloader which are field tested and worthwhile to have for Redox an AArch64. <br/> The intent is to consider most of the document except anything tied to the Linux kernel itself. |
|
||||
| [Flattened Device Tree](https://elinux.org/Device_Tree_Reference) | FDT binary blobs supplied by the bootloader shall provide the Redox kernel with misc platform \{memory, interrupt, devicemem} maps. Qemu's virt machine platform synthetically creates an FDT blob at a specific address which is very handy. |
|
||||
|
||||
## Boot flow elements
|
||||
|
||||
The following table lists the boot flow in order.
|
||||
|
||||
| Item | Notes |
|
||||
|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| [ARM Trusted Firmware (TF-A)](https://github.com/ARM-software/arm-trusted-firmware) | TF-A is a de-facto standard reference firmware implementation and proven in the field. <br/> TF-A runs post power-on on Armv8-A implementations and eventually hands off to further stages of the boot flow.<br />For qemu's virt machine platform, it is essentially absent but I mean to rely on it heavily for silicon bring up hence mentioning it here. |
|
||||
| [u-boot](https://www.denx.de/wiki/U-Boot) | u-boot will handle early console access, media access for fetching redox kernel images from non-volatile storage/misc disk subsystems/off the network. <br /> u-boot supports loading EFI applications. If EFI support to AArch64 Redox is added in the future that should essentially work out of the box. <br /> u-boot will load redox and FDT binary blobs into RAM and jump to the redox kernel. |
|
||||
| Redox early-init stub | For AArch64, the redox kernel will contain an A64 assembly stub that will setup the MMU from scratch. This is akin to the [x86_64 redox bootloader](https://github.com/redox-os/bootloader/blob/master/x86_64/startup-x86_64.asm). <br /> This stub sets up identity maps for MMU initialization, maps the kernel image itself as well as the device memory for the UART console. At present this stub shall be a part of the kernel itself for simplicity. |
|
||||
| Redox kstart entry | The early init stub hands off here. kstart will then re-init the MMU more comprehensively. |
|
||||
|
||||
## Supported devices
|
||||
|
||||
The following devices shall be supported. All necessary information specific to these devices will be provided to the redox kernel by the platform specific FDT binary blob.
|
||||
|
||||
| Device | Notes |
|
||||
|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| [Generic Interrupt Controller v2](https://developer.arm.com/products/architecture/a-profile/docs/ihi0048/b/arm-generic-interrupt-controller-architecture-version-20-architecture-specification) | The GIC is an Arm-v8A architectural element and is supported by all architecturally compliant processor implementations. GICv2 is supported by qemu's virt machine emulation and most subsequent GIC implementations are backward compatible to GICv2. |
|
||||
| [Generic Timer](http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500d/BGBBIJCB.html) | The Generic Timer Architecture is an Arm-v8A architectural element and is implemented by all compliant processor implementations. It is supported by qemu. |
|
||||
| [PrimeCell UART PL011](http://infocenter.arm.com/help/topic/com.arm.doc.ddi0183f/DDI0183.pdf) | The PL011 UART is supported by qemu and most ARM systems. |
|
||||
|
||||
## Intended development sequence and status
|
||||
|
||||
| Item | Description | Status | Notes |
|
||||
|--------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|-------------------------------------------------------------------------------|
|
||||
| Redox AArch64 toolchain | Create an usable redox AArch64 toolchain specification | Done | Using this JSON spec in isolated tests produces valid AArch64 soft float code |
|
||||
| Stubbed kernel image | Stub out AArch64 kernel support using the existing x86_64 arch code as a template <br /> Modify redox kernel build glue and work iteratively to get a linkable (non-functional) image | Not done yet | |
|
||||
| Boot flow | Create a self hosted u-boot -> redox kernel workflow <br /> Should obtain the stubbed image from a local TFTP server, load it into RAM and jump to it | Not done yet | |
|
||||
| GDB Debug flow | Create a debug workflow centered around qemu's GDB stub <br /> This should allow connecting to qemu's GDB stub and debug u-boot/redox stub via a GDB client and single stepping through code | Not done yet | |
|
||||
| Verify Redox entry | Verify that control reaches the redox kernel from u-boot | Not done yet | |
|
||||
| AArch64 early init stub | Add support for raw asm code for early AArch64 init in the redox kernel <br /> Verify that this code is located appropriately in the link map and that control reaches this code from u-boot | Not done yet | |
|
||||
| Basic DTB support | Integrate the [device_tree crate](https://mbr.github.io/device_tree-rs/device_tree/) <br /> Use the crate to access the qemu supplied DTB image and extract the memory map | Not done yet | |
|
||||
| Basic UART support | Use the device_tree crate to get the UART address from the DTB image and set up the initial console <br /> This is a polling mode only setup | Not done yet | |
|
||||
| Initial MMU support | Implement initial MMU support in the early init stub <br /> This forces the MMU into a clean state overriding any bootloader specific setup <br /> Create an identity map for MMU init <br /> Create a mapping for the kernel image <br /> Create a mapping for any devices needed at this stage (UART) | Not done yet | |
|
||||
| kmain entry | Verify that kmain entry works post early MMU init | Not done yet | |
|
||||
| Basic Redox MMU support | Get Redox to create a final set of mappings for everything <br /> Verify that this works as expected | Not done yet | |
|
||||
| Basic libc support | Flesh out a basic set of libc calls as required for simple user-land apps | Not done yet | |
|
||||
| userspace_init entry | Verify user-space entry and /sbin/init invocation | Not done yet | |
|
||||
| Basic Interrupt controller support | Add a GIC driver <br /> Verify functionality | Not done yet | |
|
||||
| Basic Timer support | Add a Generic Timer driver <br /> Verify functionality | Not done yet | |
|
||||
| UART interrupt support | Add support for UART interrupts | Not done yet | |
|
||||
| Task context switch support | Add context switching support <br /> Verify functionality | Not done yet | |
|
||||
| Login shell | Iteratively add and verify multi-user login shell support | Not done yet | |
|
||||
| Publish development branch on github | Work with the community to post work done after employer approval | Not done yet | |
|
||||
| Break out the Bubbly | Drink copious quantities of alcohol to celebrate | Not done yet | |
|
||||
| Silicon bring-up | Plan silicon bring-up | Not done yet | |
|
||||
@@ -1,121 +0,0 @@
|
||||
[workspace]
|
||||
resolver = "3"
|
||||
|
||||
[package]
|
||||
name = "kernel"
|
||||
version = "0.5.12"
|
||||
build = "build.rs"
|
||||
edition = "2024"
|
||||
|
||||
[build-dependencies]
|
||||
cc = "1.0"
|
||||
toml = "0.8"
|
||||
|
||||
[dependencies]
|
||||
acpi_ext = { package = "acpi", git = "https://gitlab.redox-os.org/redox-os/acpi.git", branch = "redox-6.x" }
|
||||
arrayvec = { version = "0.7.4", default-features = false }
|
||||
bitfield = "0.13.2"
|
||||
bitflags = "2"
|
||||
fdt = { git = "https://github.com/repnop/fdt.git", rev = "2fb1409edd1877c714a0aa36b6a7c5351004be54" }
|
||||
hashbrown = { version = "0.14.3", default-features = false, features = ["ahash", "inline-more"] }
|
||||
linked_list_allocator = "0.9.0"
|
||||
redox-path = "0.2.0"
|
||||
redox_syscall = { version = "0.7.4", default-features = false }
|
||||
rmm = { path = "rmm", default-features = false }
|
||||
slab = { version = "0.4", default-features = false }
|
||||
smallvec = { version = "1.15.1", default-features = false }
|
||||
spin = { version = "0.9.8" }
|
||||
|
||||
[dependencies.object]
|
||||
version = "0.37.1"
|
||||
default-features = false
|
||||
features = ["read_core", "elf"]
|
||||
|
||||
[dependencies.rustc-demangle]
|
||||
version = "0.1.16"
|
||||
default-features = false
|
||||
|
||||
[lints.clippy]
|
||||
# Overflows are very, very bad in kernel code as it may provide an attack vector for
|
||||
# userspace applications, and it is only checked in debug builds
|
||||
# TODO: address occurrences and then deny
|
||||
arithmetic_side_effects = "warn"
|
||||
cast_ptr_alignment = "warn" # TODO: address occurrences and then deny
|
||||
identity_op = "allow" # Used to allow stuff like 1 << 0 and 1 * 1024 * 1024
|
||||
if_same_then_else = "allow" # Useful for adding comments about different branches
|
||||
# Indexing a slice can cause panics and that is something we always want to avoid
|
||||
# in kernel code. Use .get and return an error instead
|
||||
# TODO: address occurrences and then deny
|
||||
indexing_slicing = "warn"
|
||||
many_single_char_names = "allow" # Useful in the syscall function
|
||||
module_inception = "allow" # Used for context::context
|
||||
# Not implementing default is sometimes useful in the case something has significant cost
|
||||
# to allocate. If you implement default, it can be allocated without evidence using the
|
||||
# ..Default::default() syntax. Not fun in kernel space
|
||||
new_without_default = "allow"
|
||||
not_unsafe_ptr_arg_deref = "deny"
|
||||
or_fun_call = "allow" # Used to make it nicer to return errors, for example, .ok_or(Error::new(ESRCH))
|
||||
precedence = "deny"
|
||||
ptr_cast_constness = "deny"
|
||||
too_many_arguments = "allow" # This is needed in some cases, like for syscall
|
||||
# Avoid panicking in the kernel without information about the panic. Use expect
|
||||
# TODO: address occurrences and then deny
|
||||
unwrap_used = "warn"
|
||||
|
||||
[lints.rust]
|
||||
static_mut_refs = "warn" # FIXME deny once all occurrences are fixed
|
||||
# This is usually a serious issue - a missing import of a define where it is interpreted
|
||||
# as a catch-all variable in a match, for example
|
||||
unreachable_patterns = "deny"
|
||||
unused_must_use = "deny" # Ensure that all must_use results are used
|
||||
|
||||
[target.'cfg(any(target_arch = "x86", target_arch = "x86_64"))'.dependencies]
|
||||
raw-cpuid = "10.2.0"
|
||||
x86 = { version = "0.47.0", default-features = false }
|
||||
|
||||
[target.'cfg(any(target_arch = "riscv64", target_arch = "riscv32"))'.dependencies]
|
||||
sbi-rt = "0.0.3"
|
||||
|
||||
[features]
|
||||
default = [
|
||||
"acpi",
|
||||
#"debugger",
|
||||
"multi_core",
|
||||
"serial_debug",
|
||||
"self_modifying",
|
||||
"x86_kvm_pv",
|
||||
#"busy_panic",
|
||||
#"drop_panic",
|
||||
#"syscall_debug"
|
||||
]
|
||||
|
||||
# Activates some limited code-overwriting optimizations, based on CPU features.
|
||||
self_modifying = []
|
||||
|
||||
acpi = []
|
||||
lpss_debug = []
|
||||
multi_core = ["acpi"]
|
||||
profiling = []
|
||||
#TODO: remove when threading issues are fixed
|
||||
pti = []
|
||||
drop_panic = []
|
||||
busy_panic = []
|
||||
qemu_debug = []
|
||||
serial_debug = []
|
||||
system76_ec_debug = []
|
||||
x86_kvm_pv = []
|
||||
|
||||
debugger = ["syscall_debug"]
|
||||
syscall_debug = []
|
||||
|
||||
sys_fdstat = []
|
||||
|
||||
[profile.dev]
|
||||
# Avoids having to define the eh_personality lang item and reduces kernel size
|
||||
panic = "abort"
|
||||
|
||||
[profile.release]
|
||||
# Avoids having to define the eh_personality lang item and reduces kernel size
|
||||
panic = "abort"
|
||||
#lto = true
|
||||
debug = "full"
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2017 Jeremy Soller
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -1,67 +0,0 @@
|
||||
# Red Bear OS kernel patches applied via individual patch files
|
||||
.PHONY: all check
|
||||
|
||||
SOURCE:=$(dir $(realpath $(lastword $(MAKEFILE_LIST))))
|
||||
BUILD?=$(CURDIR)
|
||||
export RUST_TARGET_PATH=$(SOURCE)/targets
|
||||
|
||||
ifeq ($(TARGET),)
|
||||
ARCH?=$(shell uname -m)
|
||||
else
|
||||
ARCH?=$(shell echo "$(TARGET)" | cut -d - -f1)
|
||||
endif
|
||||
|
||||
ifeq ($(ARCH),riscv64gc)
|
||||
override ARCH:=riscv64
|
||||
GNU_TARGET=riscv64-unknown-redox
|
||||
else ifeq ($(ARCH),i686)
|
||||
override ARCH:=i586
|
||||
GNU_TARGET=i686-unknown-redox
|
||||
else
|
||||
GNU_TARGET=$(ARCH)-unknown-redox
|
||||
endif
|
||||
|
||||
|
||||
all: $(BUILD)/kernel $(BUILD)/kernel.sym
|
||||
|
||||
LD_SCRIPT=$(SOURCE)/linkers/$(ARCH).ld
|
||||
LOCKFILE=$(SOURCE)/Cargo.lock
|
||||
MANIFEST=$(SOURCE)/Cargo.toml
|
||||
TARGET_SPEC=$(RUST_TARGET_PATH)/$(ARCH)-unknown-kernel.json
|
||||
|
||||
KERNEL_CARGO_FEATURES?=
|
||||
|
||||
$(BUILD)/kernel.all: $(LD_SCRIPT) $(LOCKFILE) $(MANIFEST) $(TARGET_SPEC) $(shell find $(SOURCE) -name "*.rs" -type f)
|
||||
cargo rustc \
|
||||
--bin kernel \
|
||||
--manifest-path "$(MANIFEST)" \
|
||||
--target "$(TARGET_SPEC)" \
|
||||
--release \
|
||||
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
|
||||
--features=$(KERNEL_CARGO_FEATURES) \
|
||||
-- \
|
||||
-C link-arg=-T -Clink-arg="$(LD_SCRIPT)" \
|
||||
-C link-arg=-z -Clink-arg=max-page-size=0x1000 \
|
||||
--emit link="$(BUILD)/kernel.all"
|
||||
|
||||
$(BUILD)/kernel.sym: $(BUILD)/kernel.all
|
||||
$(GNU_TARGET)-objcopy \
|
||||
--only-keep-debug \
|
||||
"$(BUILD)/kernel.all" \
|
||||
"$(BUILD)/kernel.sym"
|
||||
|
||||
$(BUILD)/kernel: $(BUILD)/kernel.all
|
||||
$(GNU_TARGET)-objcopy \
|
||||
--strip-debug \
|
||||
"$(BUILD)/kernel.all" \
|
||||
"$(BUILD)/kernel"
|
||||
|
||||
KERNEL_CHECK_FEATURES?=
|
||||
|
||||
check:
|
||||
cargo check \
|
||||
--bin kernel \
|
||||
--manifest-path "$(MANIFEST)" \
|
||||
--target "$(TARGET_SPEC)" \
|
||||
-Z build-std=core,alloc -Zbuild-std-features=compiler-builtins-mem \
|
||||
--features=$(KERNEL_CHECK_FEATURES)
|
||||
@@ -1,81 +0,0 @@
|
||||
# Kernel
|
||||
|
||||
Redox OS Microkernel
|
||||
|
||||
[](https://docs.rs/redox_syscall/latest/syscall/)
|
||||
[](https://github.com/XAMPPRocky/tokei)
|
||||
[](./LICENSE)
|
||||
|
||||
## Requirements
|
||||
|
||||
* [`nasm`](https://nasm.us/) needs to be available on the PATH at build time.
|
||||
|
||||
## Building The Documentation
|
||||
|
||||
Use this command:
|
||||
|
||||
```sh
|
||||
cargo doc --open --target x86_64-unknown-none
|
||||
```
|
||||
|
||||
## Debugging
|
||||
|
||||
### QEMU
|
||||
|
||||
Running [QEMU](https://www.qemu.org) with the `-s` flag will set up QEMU to listen on port `1234` for a GDB client to connect to it. To debug the redox kernel run.
|
||||
|
||||
```sh
|
||||
make qemu gdb=yes
|
||||
```
|
||||
|
||||
This will start a virtual machine with and listen on port `1234` for a GDB or LLDB client.
|
||||
|
||||
### GDB
|
||||
|
||||
If you are going to use [GDB](https://www.gnu.org/software/gdb/), run these commands to load debug symbols and connect to your running kernel:
|
||||
|
||||
```
|
||||
(gdb) symbol-file build/kernel.sym
|
||||
(gdb) target remote localhost:1234
|
||||
```
|
||||
|
||||
### LLDB
|
||||
|
||||
If you are going to use [LLDB](https://lldb.llvm.org/), run these commands to start debugging:
|
||||
|
||||
```
|
||||
(lldb) target create -s build/kernel.sym build/kernel
|
||||
(lldb) gdb-remote localhost:1234
|
||||
```
|
||||
|
||||
After connecting to your kernel you can set some interesting breakpoints and `continue`
|
||||
the process. See your debuggers man page for more information on useful commands to run.
|
||||
|
||||
## Notes
|
||||
|
||||
- Always use `foo.get(n)` instead of `foo[n]` and try to cover for the possibility of `Option::None`. Doing the regular way may work fine for applications, but never in the kernel. No possible panics should ever exist in kernel space, because then the whole OS would just stop working.
|
||||
|
||||
- If you receive a kernel panic in QEMU, use `pkill qemu-system` to kill the frozen QEMU process.
|
||||
|
||||
## How To Contribute
|
||||
|
||||
To learn how to contribute to this system component you need to read the following document:
|
||||
|
||||
- [CONTRIBUTING.md](https://gitlab.redox-os.org/redox-os/redox/-/blob/master/CONTRIBUTING.md)
|
||||
|
||||
## Development
|
||||
|
||||
To learn how to do development with this system component inside the Redox build system you need to read the [Build System](https://doc.redox-os.org/book/build-system-reference.html) and [Coding and Building](https://doc.redox-os.org/book/coding-and-building.html) pages.
|
||||
|
||||
### How To Build
|
||||
|
||||
To build this system component you need to download the Redox build system, you can learn how to do it on the [Building Redox](https://doc.redox-os.org/book/podman-build.html) page.
|
||||
|
||||
This is necessary because they only work with cross-compilation to a Redox virtual machine, but you can do some testing from Linux.
|
||||
|
||||
## Funding - _Unix-style Signals and Process Management_
|
||||
|
||||
This project is funded through [NGI Zero Core](https://nlnet.nl/core), a fund established by [NLnet](https://nlnet.nl) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu) program. Learn more at the [NLnet project page](https://nlnet.nl/project/RedoxOS-Signals).
|
||||
|
||||
[<img src="https://nlnet.nl/logo/banner.png" alt="NLnet foundation logo" width="20%" />](https://nlnet.nl)
|
||||
[<img src="https://nlnet.nl/image/logos/NGI0_tag.svg" alt="NGI Zero Logo" width="20%" />](https://nlnet.nl/core)
|
||||
@@ -1,113 +0,0 @@
|
||||
#![allow(clippy::unwrap_used)] // the build script can panic
|
||||
|
||||
use std::{env, path::Path, process::Command};
|
||||
use toml::Table;
|
||||
|
||||
fn parse_kconfig(arch: &str) -> Option<()> {
|
||||
println!("cargo:rerun-if-changed=config.toml");
|
||||
|
||||
assert!(Path::new("config.toml.example").try_exists().unwrap());
|
||||
if !Path::new("config.toml").try_exists().unwrap() {
|
||||
std::fs::copy("config.toml.example", "config.toml").unwrap();
|
||||
}
|
||||
let config_str = std::fs::read_to_string("config.toml").unwrap();
|
||||
let root: Table = toml::from_str(&config_str).unwrap();
|
||||
|
||||
let altfeatures = root
|
||||
.get("arch")?
|
||||
.as_table()
|
||||
.unwrap()
|
||||
.get(arch)?
|
||||
.as_table()
|
||||
.unwrap()
|
||||
.get("features")?
|
||||
.as_table()
|
||||
.unwrap();
|
||||
|
||||
#[expect(clippy::format_collect)] // TODO: remove once version is bumped
|
||||
let features_list = altfeatures
|
||||
.keys()
|
||||
.map(|feat| format!(", {feat:?}"))
|
||||
.collect::<String>();
|
||||
println!("cargo::rustc-check-cfg=cfg(cpu_feature_always, values(\"\"{features_list}))");
|
||||
println!("cargo::rustc-check-cfg=cfg(cpu_feature_auto, values(\"\"{features_list}))");
|
||||
println!("cargo::rustc-check-cfg=cfg(cpu_feature_never, values(\"\"{features_list}))");
|
||||
|
||||
let self_modifying = env::var("CARGO_FEATURE_SELF_MODIFYING").is_ok();
|
||||
|
||||
for (name, value) in altfeatures {
|
||||
let mut choice = value.as_str().unwrap();
|
||||
assert!(matches!(choice, "always" | "never" | "auto"));
|
||||
|
||||
if !self_modifying && choice == "auto" {
|
||||
choice = "never";
|
||||
}
|
||||
|
||||
println!("cargo:rustc-cfg=cpu_feature_{choice}=\"{name}\"");
|
||||
}
|
||||
|
||||
Some(())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("cargo::rustc-env=TARGET={}", env::var("TARGET").unwrap());
|
||||
println!("cargo::rustc-check-cfg=cfg(dtb)");
|
||||
|
||||
let out_dir = env::var("OUT_DIR").unwrap();
|
||||
let arch_str = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
|
||||
|
||||
match &*arch_str {
|
||||
"aarch64" => {
|
||||
println!("cargo::rustc-cfg=dtb");
|
||||
}
|
||||
"x86" => {
|
||||
println!("cargo::rerun-if-changed=src/asm/x86/trampoline.asm");
|
||||
|
||||
let status = Command::new("nasm")
|
||||
.arg("-f")
|
||||
.arg("bin")
|
||||
.arg("-o")
|
||||
.arg(format!("{}/trampoline", out_dir))
|
||||
.arg("src/asm/x86/trampoline.asm")
|
||||
.status()
|
||||
.expect("failed to run nasm");
|
||||
if !status.success() {
|
||||
panic!("nasm failed with exit status {}", status);
|
||||
}
|
||||
}
|
||||
"x86_64" => {
|
||||
println!("cargo::rerun-if-changed=src/asm/x86_64/trampoline.asm");
|
||||
println!("cargo::rerun-if-changed=src/asm/x86_64/s3_wakeup.asm");
|
||||
|
||||
let status = Command::new("nasm")
|
||||
.arg("-f")
|
||||
.arg("bin")
|
||||
.arg("-o")
|
||||
.arg(format!("{}/trampoline", out_dir))
|
||||
.arg("src/asm/x86_64/trampoline.asm")
|
||||
.status()
|
||||
.expect("failed to run nasm");
|
||||
if !status.success() {
|
||||
panic!("nasm failed with exit status {}", status);
|
||||
}
|
||||
|
||||
let status = Command::new("nasm")
|
||||
.arg("-f")
|
||||
.arg("bin")
|
||||
.arg("-o")
|
||||
.arg(format!("{}/s3_wakeup", out_dir))
|
||||
.arg("src/asm/x86_64/s3_wakeup.asm")
|
||||
.status()
|
||||
.expect("failed to run nasm");
|
||||
if !status.success() {
|
||||
panic!("nasm failed with exit status {}", status);
|
||||
}
|
||||
}
|
||||
"riscv64" => {
|
||||
println!("cargo::rustc-cfg=dtb");
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
let _ = parse_kconfig(&arch_str);
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
export RUST_TARGET_PATH="${PWD}/targets"
|
||||
export RUSTFLAGS="-C debuginfo=2"
|
||||
cargo clippy --lib --release --target x86_64-unknown-none "$@"
|
||||
@@ -1,7 +0,0 @@
|
||||
[arch.x86_64.features]
|
||||
smap = "auto"
|
||||
fsgsbase = "auto"
|
||||
xsave = "auto"
|
||||
xsaveopt = "auto"
|
||||
|
||||
# vim: ft=toml
|
||||
@@ -1,55 +0,0 @@
|
||||
ENTRY(kstart)
|
||||
OUTPUT_FORMAT("elf64-littleaarch64", "elf64-littleaarch64", "elf64-littleaarch64")
|
||||
|
||||
KERNEL_OFFSET = 0xFFFFFF0000000000;
|
||||
|
||||
SECTIONS {
|
||||
. = KERNEL_OFFSET;
|
||||
|
||||
. += SIZEOF_HEADERS;
|
||||
|
||||
/* Force the zero page to be part of a segment by creating a
|
||||
* dummy section in the zero page.
|
||||
* Limine will map the segment with the lowest vaddr value at
|
||||
* 0xFFFFFFFF80000000 even if the segment has a higher vaddr.
|
||||
* As such without the zero page being part of a segment, the
|
||||
* kernel would be loaded at an offset from the expected
|
||||
* location. As the redox kernel is not currently relocatable,
|
||||
* this would result in a crash. A similar issue likely exists
|
||||
* with multiboot/multiboot2 and the paddr of the segment.
|
||||
*/
|
||||
.dummy ALIGN(8) : AT(ADDR(.dummy) - KERNEL_OFFSET) {}
|
||||
|
||||
. = ALIGN(4096);
|
||||
|
||||
.text : AT(ADDR(.text) - KERNEL_OFFSET) {
|
||||
__text_start = .;
|
||||
*(.text*)
|
||||
. = ALIGN(4096);
|
||||
__text_end = .;
|
||||
}
|
||||
|
||||
.rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) {
|
||||
__rodata_start = .;
|
||||
*(.rodata*)
|
||||
. = ALIGN(4096);
|
||||
__rodata_end = .;
|
||||
}
|
||||
|
||||
.data : AT(ADDR(.data) - KERNEL_OFFSET) {
|
||||
*(.data*)
|
||||
. = ALIGN(4096);
|
||||
*(.bss*)
|
||||
. = ALIGN(4096);
|
||||
}
|
||||
|
||||
__end = .;
|
||||
|
||||
/DISCARD/ : {
|
||||
*(.comment*)
|
||||
*(.eh_frame*)
|
||||
*(.gcc_except_table*)
|
||||
*(.note*)
|
||||
*(.rel.eh_frame*)
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
ENTRY(kstart)
|
||||
OUTPUT_FORMAT(elf32-i386)
|
||||
|
||||
KERNEL_OFFSET = 0xC0000000;
|
||||
|
||||
SECTIONS {
|
||||
. = KERNEL_OFFSET;
|
||||
|
||||
. += SIZEOF_HEADERS;
|
||||
|
||||
/* Force the zero page to be part of a segment by creating a
|
||||
* dummy section in the zero page.
|
||||
* Limine will map the segment with the lowest vaddr value at
|
||||
* 0xFFFFFFFF80000000 even if the segment has a higher vaddr.
|
||||
* As such without the zero page being part of a segment, the
|
||||
* kernel would be loaded at an offset from the expected
|
||||
* location. As the redox kernel is not currently relocatable,
|
||||
* this would result in a crash. A similar issue likely exists
|
||||
* with multiboot/multiboot2 and the paddr of the segment.
|
||||
*/
|
||||
.dummy : AT(ADDR(.dummy) - KERNEL_OFFSET) {}
|
||||
|
||||
.text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) {
|
||||
__text_start = .;
|
||||
*(.text*)
|
||||
}
|
||||
|
||||
.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) {
|
||||
__text_end = .;
|
||||
__rodata_start = .;
|
||||
*(.rodata*)
|
||||
}
|
||||
|
||||
.data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) {
|
||||
__rodata_end = .;
|
||||
*(.data*)
|
||||
. = ALIGN(4K);
|
||||
*(.bss*)
|
||||
. = ALIGN(4K);
|
||||
}
|
||||
|
||||
__end = .;
|
||||
|
||||
/DISCARD/ : {
|
||||
*(.comment*)
|
||||
*(.eh_frame*)
|
||||
*(.gcc_except_table*)
|
||||
*(.note*)
|
||||
*(.rel.eh_frame*)
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
ENTRY(kstart)
|
||||
OUTPUT_FORMAT("elf64-littleriscv", "elf64-littleriscv", "elf64-littleriscv" )
|
||||
|
||||
KERNEL_OFFSET = 0xFFFFFFFF80000000;
|
||||
|
||||
SECTIONS {
|
||||
. = KERNEL_OFFSET;
|
||||
|
||||
. += SIZEOF_HEADERS;
|
||||
|
||||
/* Force the zero page to be part of a segment by creating a
|
||||
* dummy section in the zero page.
|
||||
* Linker will map the segment with the lowest vaddr value at
|
||||
* 0xFFFFFFFF80000000 even if the segment has a higher vaddr.
|
||||
* As such without the zero page being part of a segment, the
|
||||
* kernel would be loaded at an offset from the expected
|
||||
* location. As the redox kernel is not currently relocatable,
|
||||
* this would result in a crash. A similar issue likely exists
|
||||
* with multiboot/multiboot2 and the paddr of the segment.
|
||||
*/
|
||||
.dummy ALIGN(8) : AT(ADDR(.dummy) - KERNEL_OFFSET) {}
|
||||
|
||||
. = ALIGN(4096);
|
||||
|
||||
.text : AT(ADDR(.text) - KERNEL_OFFSET) {
|
||||
__text_start = .;
|
||||
*(.early_init.text*)
|
||||
. = ALIGN(4096);
|
||||
*(.text*)
|
||||
. = ALIGN(4096);
|
||||
__text_end = .;
|
||||
}
|
||||
|
||||
.rodata : AT(ADDR(.rodata) - KERNEL_OFFSET) {
|
||||
__rodata_start = .;
|
||||
*(.rodata*)
|
||||
. = ALIGN(4096);
|
||||
__rodata_end = .;
|
||||
}
|
||||
|
||||
.data : AT(ADDR(.data) - KERNEL_OFFSET) {
|
||||
*(.data*)
|
||||
*(.sdata*)
|
||||
. = ALIGN(4096);
|
||||
*(.got*)
|
||||
. = ALIGN(4096);
|
||||
*(.bss*)
|
||||
*(.sbss*)
|
||||
. = ALIGN(4096);
|
||||
}
|
||||
|
||||
__end = .;
|
||||
|
||||
/DISCARD/ : {
|
||||
*(.comment*)
|
||||
*(.eh_frame*)
|
||||
*(.gcc_except_table*)
|
||||
*(.note*)
|
||||
*(.rel.eh_frame*)
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
ENTRY(kstart)
|
||||
OUTPUT_FORMAT(elf64-x86-64)
|
||||
|
||||
KERNEL_OFFSET = 0xFFFFFFFF80000000;
|
||||
|
||||
SECTIONS {
|
||||
. = KERNEL_OFFSET;
|
||||
|
||||
. += SIZEOF_HEADERS;
|
||||
|
||||
/* Force the zero page to be part of a segment by creating a
|
||||
* dummy section in the zero page.
|
||||
* Limine will map the segment with the lowest vaddr value at
|
||||
* 0xFFFFFFFF80000000 even if the segment has a higher vaddr.
|
||||
* As such without the zero page being part of a segment, the
|
||||
* kernel would be loaded at an offset from the expected
|
||||
* location. As the redox kernel is not currently relocatable,
|
||||
* this would result in a crash. A similar issue likely exists
|
||||
* with multiboot/multiboot2 and the paddr of the segment.
|
||||
*/
|
||||
.dummy : AT(ADDR(.dummy) - KERNEL_OFFSET) {}
|
||||
|
||||
.text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) {
|
||||
__text_start = .;
|
||||
*(.text*)
|
||||
}
|
||||
|
||||
.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) {
|
||||
__text_end = .;
|
||||
__rodata_start = .;
|
||||
*(.rodata*)
|
||||
__altcode_start = .;
|
||||
KEEP(*(.altcode*))
|
||||
__altcode_end = .;
|
||||
. = ALIGN(8);
|
||||
__altrelocs_start = .;
|
||||
KEEP(*(.altrelocs*))
|
||||
__altrelocs_end = .;
|
||||
__altfeatures_start = .;
|
||||
KEEP(*(.altfeatures*))
|
||||
__altfeatures_end = .;
|
||||
}
|
||||
|
||||
.data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) {
|
||||
__rodata_end = .;
|
||||
*(.data*)
|
||||
. = ALIGN(4K);
|
||||
*(.bss*)
|
||||
}
|
||||
|
||||
__end = .;
|
||||
|
||||
/DISCARD/ : {
|
||||
*(.comment*)
|
||||
*(.eh_frame*)
|
||||
*(.gcc_except_table*)
|
||||
*(.note*)
|
||||
*(.rel.eh_frame*)
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -1,16 +0,0 @@
|
||||
[package]
|
||||
name = "rmm"
|
||||
version = "0.1.0"
|
||||
authors = ["Jeremy Soller <jeremy@system76.com>"]
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
bitflags = "2"
|
||||
|
||||
[features]
|
||||
std = []
|
||||
|
||||
[[bin]]
|
||||
name = "rmm"
|
||||
path = "src/main.rs"
|
||||
required-features = ["std"]
|
||||
@@ -1,4 +0,0 @@
|
||||
# Redox Memory Management
|
||||
|
||||
This is a Rust crate to provide abstractions for hardware memory management. It
|
||||
also contains a mechanism for testing memory management with software emulation.
|
||||
@@ -1,296 +0,0 @@
|
||||
use core::{marker::PhantomData, mem};
|
||||
|
||||
use crate::{
|
||||
Arch, BumpAllocator, FrameAllocator, FrameCount, FrameUsage, PhysicalAddress, VirtualAddress,
|
||||
};
|
||||
|
||||
#[repr(transparent)]
|
||||
struct BuddyUsage(u8);
|
||||
|
||||
#[repr(C, packed)]
|
||||
struct BuddyEntry<A> {
|
||||
base: PhysicalAddress,
|
||||
size: usize,
|
||||
// Number of first free page
|
||||
skip: usize,
|
||||
// Count of used pages
|
||||
used: usize,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A> Clone for BuddyEntry<A> {
|
||||
fn clone(&self) -> Self {
|
||||
*self
|
||||
}
|
||||
}
|
||||
impl<A> Copy for BuddyEntry<A> {}
|
||||
|
||||
impl<A: Arch> BuddyEntry<A> {
|
||||
fn empty() -> Self {
|
||||
Self {
|
||||
base: PhysicalAddress::new(0),
|
||||
size: 0,
|
||||
skip: 0,
|
||||
used: 0,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn pages(&self) -> usize {
|
||||
self.size >> A::PAGE_SHIFT
|
||||
}
|
||||
|
||||
fn usage_pages(&self) -> usize {
|
||||
let bytes = self.pages() * mem::size_of::<BuddyUsage>();
|
||||
// Round bytes used for usage to next page
|
||||
(bytes + A::PAGE_OFFSET_MASK) >> A::PAGE_SHIFT
|
||||
}
|
||||
|
||||
unsafe fn usage_addr(&self, page: usize) -> Option<VirtualAddress> {
|
||||
if page < self.pages() {
|
||||
let phys = self.base.add(page * mem::size_of::<BuddyUsage>());
|
||||
Some(A::phys_to_virt(phys))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn usage(&self, page: usize) -> Option<BuddyUsage> {
|
||||
unsafe {
|
||||
let addr = self.usage_addr(page)?;
|
||||
Some(A::read(addr))
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(clippy::unit_arg)]
|
||||
unsafe fn set_usage(&self, page: usize, usage: BuddyUsage) -> Option<()> {
|
||||
unsafe {
|
||||
let addr = self.usage_addr(page)?;
|
||||
Some(A::write(addr, usage))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct BuddyAllocator<A> {
|
||||
table_virt: VirtualAddress,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> BuddyAllocator<A> {
|
||||
const BUDDY_ENTRIES: usize = A::PAGE_SIZE / mem::size_of::<BuddyEntry<A>>();
|
||||
|
||||
pub unsafe fn new(mut bump_allocator: BumpAllocator<A>) -> Option<Self> {
|
||||
unsafe {
|
||||
// Allocate buddy table
|
||||
let table_phys = bump_allocator.allocate_one()?;
|
||||
let table_virt = A::phys_to_virt(table_phys);
|
||||
for i in 0..(A::PAGE_SIZE / mem::size_of::<BuddyEntry<A>>()) {
|
||||
let virt = table_virt.add(i * mem::size_of::<BuddyEntry<A>>());
|
||||
A::write(virt, BuddyEntry::<A>::empty());
|
||||
}
|
||||
|
||||
let allocator = Self {
|
||||
table_virt,
|
||||
phantom: PhantomData,
|
||||
};
|
||||
|
||||
// Add areas to buddy table, combining areas when possible, and skipping frames used
|
||||
// by the bump allocator
|
||||
let mut offset = bump_allocator.offset();
|
||||
for old_area in bump_allocator.areas().iter() {
|
||||
let mut area = *old_area;
|
||||
if offset >= area.size {
|
||||
offset -= area.size;
|
||||
continue;
|
||||
} else if offset > 0 {
|
||||
area.base = area.base.add(offset);
|
||||
area.size -= offset;
|
||||
offset = 0;
|
||||
}
|
||||
for i in 0..(A::PAGE_SIZE / mem::size_of::<BuddyEntry<A>>()) {
|
||||
let virt = table_virt.add(i * mem::size_of::<BuddyEntry<A>>());
|
||||
let mut entry = A::read::<BuddyEntry<A>>(virt);
|
||||
let inserted = if area.base.add(area.size) == { entry.base } {
|
||||
// Combine entry at start
|
||||
entry.base = area.base;
|
||||
entry.size += area.size;
|
||||
true
|
||||
} else if area.base == entry.base.add(entry.size) {
|
||||
// Combine entry at end
|
||||
entry.size += area.size;
|
||||
true
|
||||
} else if entry.size == 0 {
|
||||
// Create new entry
|
||||
entry.base = area.base;
|
||||
entry.size = area.size;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
};
|
||||
if inserted {
|
||||
A::write(virt, entry);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: sort areas?
|
||||
|
||||
// Allocate buddy maps
|
||||
for i in 0..Self::BUDDY_ENTRIES {
|
||||
let virt = table_virt.add(i * mem::size_of::<BuddyEntry<A>>());
|
||||
let mut entry = A::read::<BuddyEntry<A>>(virt);
|
||||
|
||||
// Only set up entries that have enough space for their own usage map
|
||||
let usage_pages = entry.usage_pages();
|
||||
if entry.pages() > usage_pages {
|
||||
// Mark all usage bytes as unused
|
||||
let usage_start = entry.usage_addr(0)?;
|
||||
for page in 0..usage_pages {
|
||||
A::write_bytes(usage_start.add(page << A::PAGE_SHIFT), 0, A::PAGE_SIZE);
|
||||
}
|
||||
|
||||
// Mark bytes used for usage as used
|
||||
for page in 0..usage_pages {
|
||||
entry.set_usage(page, BuddyUsage(1))?;
|
||||
}
|
||||
}
|
||||
|
||||
// Skip the pages used for usage
|
||||
entry.skip = usage_pages;
|
||||
|
||||
// Set used pages to pages used for usage
|
||||
entry.used = usage_pages;
|
||||
|
||||
// Write updated entry
|
||||
A::write(virt, entry);
|
||||
}
|
||||
|
||||
Some(allocator)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<A: Arch> FrameAllocator for BuddyAllocator<A> {
|
||||
fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress> {
|
||||
unsafe {
|
||||
if self.table_virt.data() == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
for entry_i in 0..Self::BUDDY_ENTRIES {
|
||||
let virt = self
|
||||
.table_virt
|
||||
.add(entry_i * mem::size_of::<BuddyEntry<A>>());
|
||||
let mut entry = A::read::<BuddyEntry<A>>(virt);
|
||||
|
||||
let mut free_page = entry.skip;
|
||||
let mut free_count = 0;
|
||||
for page in entry.skip..entry.pages() {
|
||||
let usage = entry.usage(page)?;
|
||||
if usage.0 == 0 {
|
||||
free_count += 1;
|
||||
|
||||
if free_count == count.data() {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
free_page = page + 1;
|
||||
free_count = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if free_count == count.data() {
|
||||
for page in free_page..free_page + free_count {
|
||||
// Update usage
|
||||
let mut usage = entry.usage(page)?;
|
||||
usage.0 += 1;
|
||||
entry.set_usage(page, usage);
|
||||
|
||||
// Zero page
|
||||
let page_phys = entry.base.add(page << A::PAGE_SHIFT);
|
||||
let page_virt = A::phys_to_virt(page_phys);
|
||||
A::write_bytes(page_virt, 0, A::PAGE_SIZE);
|
||||
}
|
||||
|
||||
// Update skip if necessary
|
||||
if entry.skip == free_page {
|
||||
entry.skip = free_page + free_count;
|
||||
}
|
||||
|
||||
// Update used page count
|
||||
entry.used += free_count;
|
||||
|
||||
// Write updated entry
|
||||
A::write(virt, entry);
|
||||
|
||||
return Some(entry.base.add(free_page << A::PAGE_SHIFT));
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn free(&mut self, base: PhysicalAddress, count: FrameCount) {
|
||||
unsafe {
|
||||
if self.table_virt.data() == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let size = count.data() * A::PAGE_SIZE;
|
||||
for i in 0..Self::BUDDY_ENTRIES {
|
||||
let virt = self.table_virt.add(i * mem::size_of::<BuddyEntry<A>>());
|
||||
let mut entry = A::read::<BuddyEntry<A>>(virt);
|
||||
|
||||
if base >= { entry.base } && base.add(size) <= entry.base.add(entry.size) {
|
||||
let start_page = (base.data() - { entry.base }.data()) >> A::PAGE_SHIFT;
|
||||
for page in start_page..start_page + count.data() {
|
||||
let mut usage = entry.usage(page).expect("failed to get usage during free");
|
||||
|
||||
if usage.0 > 0 {
|
||||
usage.0 -= 1;
|
||||
} else {
|
||||
panic!("tried to free already free frame");
|
||||
}
|
||||
|
||||
// If page was freed
|
||||
if usage.0 == 0 {
|
||||
// Update skip if necessary
|
||||
if page < entry.skip {
|
||||
entry.skip = page;
|
||||
}
|
||||
|
||||
// Update used page count
|
||||
entry.used -= 1;
|
||||
}
|
||||
|
||||
entry
|
||||
.set_usage(page, usage)
|
||||
.expect("failed to set usage during free");
|
||||
}
|
||||
|
||||
// Write updated entry
|
||||
A::write(virt, entry);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn usage(&self) -> FrameUsage {
|
||||
unsafe {
|
||||
let mut total = 0;
|
||||
let mut used = 0;
|
||||
for i in 0..Self::BUDDY_ENTRIES {
|
||||
let virt = self.table_virt.add(i * mem::size_of::<BuddyEntry<A>>());
|
||||
let entry = A::read::<BuddyEntry<A>>(virt);
|
||||
total += entry.size >> A::PAGE_SHIFT;
|
||||
used += entry.used;
|
||||
}
|
||||
FrameUsage::new(FrameCount::new(used), FrameCount::new(total))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
use core::marker::PhantomData;
|
||||
|
||||
use crate::{Arch, FrameAllocator, FrameCount, FrameUsage, MemoryArea, PhysicalAddress};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct BumpAllocator<A> {
|
||||
orig_areas: (&'static [MemoryArea], usize),
|
||||
cur_areas: (&'static [MemoryArea], usize),
|
||||
_marker: PhantomData<fn() -> A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> BumpAllocator<A> {
|
||||
pub fn new(mut areas: &'static [MemoryArea], mut offset: usize) -> Self {
|
||||
while let Some(first) = areas.first()
|
||||
&& first.size <= offset
|
||||
{
|
||||
offset -= first.size;
|
||||
areas = &areas[1..];
|
||||
}
|
||||
|
||||
Self {
|
||||
orig_areas: (areas, offset),
|
||||
cur_areas: (areas, offset),
|
||||
_marker: PhantomData,
|
||||
}
|
||||
}
|
||||
pub fn areas(&self) -> &'static [MemoryArea] {
|
||||
self.orig_areas.0
|
||||
}
|
||||
/// Returns one semifree and the fully free areas. The offset is the number of bytes after
|
||||
/// which the first area is free.
|
||||
pub fn free_areas(&self) -> (&'static [MemoryArea], usize) {
|
||||
self.cur_areas
|
||||
}
|
||||
pub fn abs_offset(&self) -> PhysicalAddress {
|
||||
let (areas, off) = self.cur_areas;
|
||||
areas
|
||||
.first()
|
||||
.map_or(PhysicalAddress::new(0), |a| a.base.add(off))
|
||||
}
|
||||
pub fn offset(&self) -> usize {
|
||||
(self.usage().total().data() - self.usage().free().data()) * A::PAGE_SIZE
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl<A: Arch> FrameAllocator for BumpAllocator<A> {
|
||||
fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress> {
|
||||
unsafe {
|
||||
let req_size = count.data() * A::PAGE_SIZE;
|
||||
|
||||
let block = loop {
|
||||
let area = self.cur_areas.0.first()?;
|
||||
let off = self.cur_areas.1;
|
||||
if area.size - off < req_size {
|
||||
self.cur_areas = (&self.cur_areas.0[1..], 0);
|
||||
continue;
|
||||
}
|
||||
self.cur_areas.1 += req_size;
|
||||
|
||||
break area.base.add(off);
|
||||
};
|
||||
A::write_bytes(A::phys_to_virt(block), 0, req_size);
|
||||
Some(block)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn free(&mut self, _address: PhysicalAddress, _count: FrameCount) {
|
||||
unimplemented!("BumpAllocator::free not implemented");
|
||||
}
|
||||
|
||||
fn usage(&self) -> FrameUsage {
|
||||
let total = self.orig_areas.0.iter().map(|a| a.size).sum::<usize>() - self.orig_areas.1;
|
||||
let free = self.cur_areas.0.iter().map(|a| a.size).sum::<usize>() - self.cur_areas.1;
|
||||
FrameUsage::new(
|
||||
FrameCount::new((total - free) / A::PAGE_SIZE),
|
||||
FrameCount::new(total / A::PAGE_SIZE),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
use crate::PhysicalAddress;
|
||||
|
||||
pub use self::{buddy::*, bump::*};
|
||||
|
||||
mod buddy;
|
||||
mod bump;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(transparent)]
|
||||
pub struct FrameCount(usize);
|
||||
|
||||
impl FrameCount {
|
||||
pub fn new(count: usize) -> Self {
|
||||
Self(count)
|
||||
}
|
||||
|
||||
pub fn data(&self) -> usize {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct FrameUsage {
|
||||
used: FrameCount,
|
||||
total: FrameCount,
|
||||
}
|
||||
|
||||
impl FrameUsage {
|
||||
pub fn new(used: FrameCount, total: FrameCount) -> Self {
|
||||
Self { used, total }
|
||||
}
|
||||
|
||||
pub fn used(&self) -> FrameCount {
|
||||
self.used
|
||||
}
|
||||
|
||||
pub fn free(&self) -> FrameCount {
|
||||
FrameCount(self.total.0 - self.used.0)
|
||||
}
|
||||
|
||||
pub fn total(&self) -> FrameCount {
|
||||
self.total
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe trait FrameAllocator {
|
||||
fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress>;
|
||||
|
||||
unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount);
|
||||
|
||||
fn allocate_one(&mut self) -> Option<PhysicalAddress> {
|
||||
self.allocate(FrameCount::new(1))
|
||||
}
|
||||
|
||||
unsafe fn free_one(&mut self, address: PhysicalAddress) {
|
||||
unsafe {
|
||||
self.free(address, FrameCount::new(1));
|
||||
}
|
||||
}
|
||||
|
||||
fn usage(&self) -> FrameUsage;
|
||||
}
|
||||
|
||||
unsafe impl<T> FrameAllocator for &mut T
|
||||
where
|
||||
T: FrameAllocator,
|
||||
{
|
||||
fn allocate(&mut self, count: FrameCount) -> Option<PhysicalAddress> {
|
||||
T::allocate(self, count)
|
||||
}
|
||||
unsafe fn free(&mut self, address: PhysicalAddress, count: FrameCount) {
|
||||
unsafe { T::free(self, address, count) }
|
||||
}
|
||||
fn allocate_one(&mut self) -> Option<PhysicalAddress> {
|
||||
T::allocate_one(self)
|
||||
}
|
||||
unsafe fn free_one(&mut self, address: PhysicalAddress) {
|
||||
unsafe { T::free_one(self, address) }
|
||||
}
|
||||
fn usage(&self) -> FrameUsage {
|
||||
T::usage(self)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
pub use self::frame::*;
|
||||
|
||||
mod frame;
|
||||
@@ -1,153 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct AArch64Arch;
|
||||
|
||||
impl Arch for AArch64Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = true;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each
|
||||
const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3
|
||||
|
||||
//TODO
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 40;
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT
|
||||
| 1 << 1 // Page flag
|
||||
| 1 << 10 // Access flag
|
||||
| Self::ENTRY_FLAG_NO_GLOBAL;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize
|
||||
= Self::ENTRY_FLAG_PRESENT
|
||||
| Self::ENTRY_FLAG_READWRITE
|
||||
| 1 << 1 // Table flag
|
||||
| 1 << 10 // Access flag
|
||||
;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 1 << 7;
|
||||
const ENTRY_FLAG_READWRITE: usize = 0;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 6;
|
||||
// This sets both userspace and privileged execute never
|
||||
//TODO: Separate the two?
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 0b11 << 53;
|
||||
const ENTRY_FLAG_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 1 << 11;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = MEM_ATTR_DEVICE_nGnRnE << 2;
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = MEM_ATTR_NC << 2;
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = MEM_ATTR_NC << 2;
|
||||
|
||||
const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000;
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe {
|
||||
asm!("
|
||||
dsb ishst
|
||||
tlbi vaae1is, {}
|
||||
dsb ish
|
||||
isb
|
||||
", in(reg) (address.data() >> Self::PAGE_SHIFT));
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe {
|
||||
asm!(
|
||||
"
|
||||
dsb ishst
|
||||
tlbi vmalle1is
|
||||
dsb ish
|
||||
isb
|
||||
"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(table_kind: TableKind) -> PhysicalAddress {
|
||||
let address: usize;
|
||||
match table_kind {
|
||||
TableKind::User => {
|
||||
unsafe { asm!("mrs {0}, ttbr0_el1", out(reg) address) };
|
||||
}
|
||||
TableKind::Kernel => {
|
||||
unsafe { asm!("mrs {0}, ttbr1_el1", out(reg) address) };
|
||||
}
|
||||
}
|
||||
PhysicalAddress::new(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress) {
|
||||
unsafe {
|
||||
match table_kind {
|
||||
TableKind::User => {
|
||||
asm!("msr ttbr0_el1, {0}", in(reg) address.data());
|
||||
}
|
||||
TableKind::Kernel => {
|
||||
asm!("msr ttbr1_el1, {0}", in(reg) address.data());
|
||||
}
|
||||
}
|
||||
Self::invalidate_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn virt_is_valid(_address: VirtualAddress) -> bool {
|
||||
//TODO: what makes an address valid on aarch64?
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(not(target_arch = "aarch64"), allow(unused))]
|
||||
const MEM_ATTR_WB: usize = 0;
|
||||
const MEM_ATTR_NC: usize = 1;
|
||||
#[allow(non_upper_case_globals)]
|
||||
const MEM_ATTR_DEVICE_nGnRnE: usize = 2;
|
||||
|
||||
/// Setup Memory Access Indirection Register
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
#[inline(always)]
|
||||
pub unsafe fn init_mair() {
|
||||
// https://github.com/freebsd/freebsd-src/blob/d15733065c4221dcd5bb3622d225760f271f6fc9/sys/arm64/include/armreg.h#L1986-L1991
|
||||
const fn mair_attr(attr: u64, idx: usize) -> u64 {
|
||||
attr << (idx * 8)
|
||||
}
|
||||
#[allow(non_upper_case_globals)]
|
||||
const MAIR_DEVICE_nGnRnE: u64 = 0x00;
|
||||
#[allow(non_upper_case_globals)]
|
||||
const _MAIR_DEVICE_nGnRE: u64 = 0x04;
|
||||
const MAIR_NORMAL_NC: u64 = 0x44;
|
||||
const _MAIR_NORMAL_WT: u64 = 0xbb;
|
||||
const MAIR_NORMAL_WB: u64 = 0xff;
|
||||
|
||||
unsafe {
|
||||
let val: u64 = const {
|
||||
mair_attr(MAIR_DEVICE_nGnRnE, MEM_ATTR_DEVICE_nGnRnE)
|
||||
| mair_attr(MAIR_NORMAL_NC, MEM_ATTR_NC)
|
||||
| mair_attr(MAIR_NORMAL_WB, MEM_ATTR_WB)
|
||||
};
|
||||
|
||||
asm!("msr mair_el1, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(AArch64Arch::PAGE_SIZE == 4096);
|
||||
assert!(AArch64Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(AArch64Arch::PAGE_ADDRESS_SHIFT == 48);
|
||||
assert!(AArch64Arch::PAGE_ADDRESS_SIZE == 0x0001_0000_0000_0000);
|
||||
assert!(AArch64Arch::PAGE_ADDRESS_MASK == 0x0000_FFFF_FFFF_F000);
|
||||
assert!(AArch64Arch::PAGE_ENTRY_SIZE == 8);
|
||||
assert!(AArch64Arch::PAGE_ENTRIES == 512);
|
||||
assert!(AArch64Arch::PAGE_ENTRY_MASK == 0x1FF);
|
||||
assert!(AArch64Arch::PAGE_NEGATIVE_MASK == 0xFFFF_0000_0000_0000);
|
||||
|
||||
assert!(AArch64Arch::ENTRY_ADDRESS_SIZE == 0x0000_0100_0000_0000);
|
||||
assert!(AArch64Arch::ENTRY_ADDRESS_MASK == 0x0000_00FF_FFFF_FFFF);
|
||||
assert!(AArch64Arch::ENTRY_FLAGS_MASK == 0xFFF0_0000_0000_0FFF);
|
||||
|
||||
assert!(AArch64Arch::PHYS_OFFSET == 0xFFFF_8000_0000_0000);
|
||||
};
|
||||
@@ -1,355 +0,0 @@
|
||||
extern crate std;
|
||||
|
||||
use std::{boxed::Box, collections::BTreeMap, marker::PhantomData, mem, ptr, sync::Mutex, vec};
|
||||
|
||||
use crate::{
|
||||
arch::x86_64::X8664Arch, page::PageFlags, Arch, MemoryArea, PageEntry, PhysicalAddress,
|
||||
TableKind, VirtualAddress, MEGABYTE,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct EmulateArch;
|
||||
|
||||
impl EmulateArch {
|
||||
pub unsafe fn init() -> &'static [MemoryArea] {
|
||||
unsafe {
|
||||
// Create machine with PAGE_ENTRIES pages offset mapped (2 MiB on x86_64)
|
||||
let mut machine = Machine::new(MEMORY_SIZE);
|
||||
|
||||
// PML4 index 256 (PHYS_OFFSET) link to PDP
|
||||
let pml4 = 0;
|
||||
let pdp = pml4 + Self::PAGE_SIZE;
|
||||
let flags = Self::ENTRY_FLAG_READWRITE | Self::ENTRY_FLAG_PRESENT;
|
||||
machine.write_phys::<usize>(
|
||||
PhysicalAddress::new(pml4 + 256 * Self::PAGE_ENTRY_SIZE),
|
||||
pdp | flags,
|
||||
);
|
||||
|
||||
// PDP link to PD
|
||||
let pd = pdp + Self::PAGE_SIZE;
|
||||
machine.write_phys::<usize>(PhysicalAddress::new(pdp), pd | flags);
|
||||
|
||||
// PD link to PT
|
||||
let pt = pd + Self::PAGE_SIZE;
|
||||
machine.write_phys::<usize>(PhysicalAddress::new(pd), pt | flags);
|
||||
|
||||
// PT links to frames
|
||||
for i in 0..Self::PAGE_ENTRIES {
|
||||
let page = i * Self::PAGE_SIZE;
|
||||
machine.write_phys::<usize>(
|
||||
PhysicalAddress::new(pt + i * Self::PAGE_ENTRY_SIZE),
|
||||
page | flags,
|
||||
);
|
||||
}
|
||||
|
||||
*MACHINE.lock().unwrap() = Some(machine);
|
||||
|
||||
// Set table to pml4
|
||||
EmulateArch::set_table(TableKind::Kernel, PhysicalAddress::new(pml4));
|
||||
|
||||
&MEMORY_AREAS
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Arch for EmulateArch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = X8664Arch::PAGE_SHIFT;
|
||||
const PAGE_ENTRY_SHIFT: usize = X8664Arch::PAGE_ENTRY_SHIFT;
|
||||
const PAGE_LEVELS: usize = X8664Arch::PAGE_LEVELS;
|
||||
|
||||
const ENTRY_ADDRESS_SHIFT: usize = X8664Arch::ENTRY_ADDRESS_SHIFT;
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_PAGE;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = X8664Arch::ENTRY_FLAG_DEFAULT_TABLE;
|
||||
const ENTRY_FLAG_PRESENT: usize = X8664Arch::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_READONLY: usize = X8664Arch::ENTRY_FLAG_READONLY;
|
||||
const ENTRY_FLAG_READWRITE: usize = X8664Arch::ENTRY_FLAG_READWRITE;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = X8664Arch::ENTRY_FLAG_PAGE_USER;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = X8664Arch::ENTRY_FLAG_NO_EXEC;
|
||||
const ENTRY_FLAG_EXEC: usize = X8664Arch::ENTRY_FLAG_EXEC;
|
||||
|
||||
const PHYS_OFFSET: usize = X8664Arch::PHYS_OFFSET;
|
||||
|
||||
const ENTRY_FLAG_GLOBAL: usize = X8664Arch::ENTRY_FLAG_GLOBAL;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = X8664Arch::ENTRY_FLAG_NO_GLOBAL;
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = X8664Arch::ENTRY_ADDRESS_WIDTH;
|
||||
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = X8664Arch::ENTRY_FLAG_DEVICE_MEMORY;
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = X8664Arch::ENTRY_FLAG_UNCACHEABLE;
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = X8664Arch::ENTRY_FLAG_WRITE_COMBINING;
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn read<T>(address: VirtualAddress) -> T {
|
||||
MACHINE.lock().unwrap().as_ref().unwrap().read(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn write<T>(address: VirtualAddress, value: T) {
|
||||
MACHINE
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write(address, value)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) {
|
||||
MACHINE
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_bytes(address, value, count)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
MACHINE
|
||||
.lock()
|
||||
.unwrap()
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.invalidate(address);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
MACHINE.lock().unwrap().as_mut().unwrap().invalidate_all();
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
MACHINE.lock().unwrap().as_mut().unwrap().get_table()
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
MACHINE.lock().unwrap().as_mut().unwrap().set_table(address);
|
||||
}
|
||||
fn virt_is_valid(_address: VirtualAddress) -> bool {
|
||||
// TODO: Don't see why an emulated arch would have any problems with canonicalness...
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
const MEMORY_SIZE: usize = 64 * MEGABYTE;
|
||||
static MEMORY_AREAS: [MemoryArea; 2] = [
|
||||
MemoryArea {
|
||||
base: PhysicalAddress::new(EmulateArch::PAGE_SIZE * 4), // Initial PML4, PDP, PD, and PT wasted
|
||||
size: MEMORY_SIZE / 2 - EmulateArch::PAGE_SIZE * 4,
|
||||
},
|
||||
// Second area for debugging
|
||||
MemoryArea {
|
||||
base: PhysicalAddress::new(MEMORY_SIZE / 2),
|
||||
size: MEMORY_SIZE / 2,
|
||||
},
|
||||
];
|
||||
|
||||
static MACHINE: Mutex<Option<Machine<EmulateArch>>> = Mutex::new(None);
|
||||
|
||||
struct Machine<A> {
|
||||
memory: Box<[u8]>,
|
||||
map: BTreeMap<VirtualAddress, PageEntry<A>>,
|
||||
table_addr: PhysicalAddress,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> Machine<A> {
|
||||
fn new(memory_size: usize) -> Self {
|
||||
Self {
|
||||
memory: vec![0; memory_size].into_boxed_slice(),
|
||||
map: BTreeMap::new(),
|
||||
table_addr: PhysicalAddress::new(0),
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
fn read_phys<T>(&self, phys: PhysicalAddress) -> T {
|
||||
let size = mem::size_of::<T>();
|
||||
if phys.add(size).data() <= self.memory.len() {
|
||||
unsafe { ptr::read(self.memory.as_ptr().add(phys.data()) as *const T) }
|
||||
} else {
|
||||
panic!(
|
||||
"read_phys: 0x{:X} size 0x{:X} outside of memory",
|
||||
phys.data(),
|
||||
size
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_phys<T>(&mut self, phys: PhysicalAddress, value: T) {
|
||||
let size = mem::size_of::<T>();
|
||||
if phys.add(size).data() <= self.memory.len() {
|
||||
unsafe {
|
||||
ptr::write(self.memory.as_mut_ptr().add(phys.data()) as *mut T, value);
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"write_phys: 0x{:X} size 0x{:X} outside of memory",
|
||||
phys.data(),
|
||||
size
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_phys_bytes(&mut self, phys: PhysicalAddress, value: u8, count: usize) {
|
||||
if phys.add(count).data() <= self.memory.len() {
|
||||
unsafe {
|
||||
ptr::write_bytes(self.memory.as_mut_ptr().add(phys.data()), value, count);
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"write_phys_bytes: 0x{:X} count 0x{:X} outside of memory",
|
||||
phys.data(),
|
||||
count
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags<A>)> {
|
||||
let virt_data = virt.data();
|
||||
let page = virt_data & A::PAGE_ADDRESS_MASK;
|
||||
let offset = virt_data & A::PAGE_OFFSET_MASK;
|
||||
let entry = self.map.get(&VirtualAddress::new(page))?;
|
||||
Some((entry.address().ok()?.add(offset), entry.flags()))
|
||||
}
|
||||
|
||||
fn read<T>(&self, virt: VirtualAddress) -> T {
|
||||
//TODO: allow reading past page boundaries
|
||||
let virt_data = virt.data();
|
||||
let size = mem::size_of::<T>();
|
||||
if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) {
|
||||
panic!(
|
||||
"read: 0x{:X} size 0x{:X} passes page boundary",
|
||||
virt_data, size
|
||||
);
|
||||
}
|
||||
|
||||
if let Some((phys, _flags)) = self.translate(virt) {
|
||||
self.read_phys(phys)
|
||||
} else {
|
||||
panic!("read: 0x{:X} size 0x{:X} not present", virt_data, size);
|
||||
}
|
||||
}
|
||||
|
||||
fn write<T>(&mut self, virt: VirtualAddress, value: T) {
|
||||
//TODO: allow writing past page boundaries
|
||||
let virt_data = virt.data();
|
||||
let size = mem::size_of::<T>();
|
||||
if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (size - 1)) & A::PAGE_ADDRESS_MASK) {
|
||||
panic!(
|
||||
"write: 0x{:X} size 0x{:X} passes page boundary",
|
||||
virt_data, size
|
||||
);
|
||||
}
|
||||
|
||||
if let Some((phys, flags)) = self.translate(virt) {
|
||||
if flags.has_write() {
|
||||
self.write_phys(phys, value);
|
||||
} else {
|
||||
panic!("write: 0x{:X} size 0x{:X} not writable", virt_data, size);
|
||||
}
|
||||
} else {
|
||||
panic!("write: 0x{:X} size 0x{:X} not present", virt_data, size);
|
||||
}
|
||||
}
|
||||
|
||||
fn write_bytes(&mut self, virt: VirtualAddress, value: u8, count: usize) {
|
||||
//TODO: allow writing past page boundaries
|
||||
let virt_data = virt.data();
|
||||
if (virt_data & A::PAGE_ADDRESS_MASK) != ((virt_data + (count - 1)) & A::PAGE_ADDRESS_MASK)
|
||||
{
|
||||
panic!(
|
||||
"write_bytes: 0x{:X} count 0x{:X} passes page boundary",
|
||||
virt_data, count
|
||||
);
|
||||
}
|
||||
|
||||
if let Some((phys, flags)) = self.translate(virt) {
|
||||
if flags.has_write() {
|
||||
self.write_phys_bytes(phys, value, count);
|
||||
} else {
|
||||
panic!(
|
||||
"write_bytes: 0x{:X} count 0x{:X} not writable",
|
||||
virt_data, count
|
||||
);
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"write_bytes: 0x{:X} count 0x{:X} not present",
|
||||
virt_data, count
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn invalidate(&mut self, _address: VirtualAddress) {
|
||||
unimplemented!("EmulateArch::invalidate not implemented");
|
||||
}
|
||||
|
||||
//TODO: cleanup
|
||||
fn invalidate_all(&mut self) {
|
||||
self.map.clear();
|
||||
|
||||
// PML4
|
||||
let a4 = self.table_addr.data();
|
||||
for i4 in 0..A::PAGE_ENTRIES {
|
||||
let e3 = self.read_phys::<usize>(PhysicalAddress::new(a4 + i4 * A::PAGE_ENTRY_SIZE));
|
||||
let f3 = e3 & A::ENTRY_FLAGS_MASK;
|
||||
if f3 & A::ENTRY_FLAG_PRESENT == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Page directory pointer
|
||||
let a3 = ((e3 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT;
|
||||
for i3 in 0..A::PAGE_ENTRIES {
|
||||
let e2 =
|
||||
self.read_phys::<usize>(PhysicalAddress::new(a3 + i3 * A::PAGE_ENTRY_SIZE));
|
||||
let f2 = e2 & A::ENTRY_FLAGS_MASK;
|
||||
if f2 & A::ENTRY_FLAG_PRESENT == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Page directory
|
||||
let a2 = ((e2 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT;
|
||||
for i2 in 0..A::PAGE_ENTRIES {
|
||||
let e1 =
|
||||
self.read_phys::<usize>(PhysicalAddress::new(a2 + i2 * A::PAGE_ENTRY_SIZE));
|
||||
let f1 = e1 & A::ENTRY_FLAGS_MASK;
|
||||
if f1 & A::ENTRY_FLAG_PRESENT == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Page table
|
||||
let a1 =
|
||||
((e1 >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT;
|
||||
for i1 in 0..A::PAGE_ENTRIES {
|
||||
let e = self
|
||||
.read_phys::<usize>(PhysicalAddress::new(a1 + i1 * A::PAGE_ENTRY_SIZE));
|
||||
let f = e & A::ENTRY_FLAGS_MASK;
|
||||
if f & A::ENTRY_FLAG_PRESENT == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Page
|
||||
let page = (i4 << 39) | (i3 << 30) | (i2 << 21) | (i1 << 12);
|
||||
//println!("map 0x{:X} to 0x{:X}, 0x{:X}", page, a, f);
|
||||
self.map
|
||||
.insert(VirtualAddress::new(page), PageEntry::from_data(e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_table(&self) -> PhysicalAddress {
|
||||
self.table_addr
|
||||
}
|
||||
|
||||
fn set_table(&mut self, address: PhysicalAddress) {
|
||||
self.table_addr = address;
|
||||
self.invalidate_all();
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
use core::ptr;
|
||||
|
||||
use crate::{PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
//TODO: Support having all page tables compile on all architectures
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
pub mod aarch64;
|
||||
#[cfg(all(feature = "std", target_pointer_width = "64"))]
|
||||
pub mod emulate;
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
pub mod riscv64;
|
||||
#[cfg(target_pointer_width = "32")]
|
||||
pub mod x86;
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
pub mod x86_64;
|
||||
mod x86_shared;
|
||||
|
||||
pub trait Arch: Clone + Copy {
|
||||
/// Does the architecture use a separate page table for the kernel.
|
||||
///
|
||||
/// If false, the page table entries corresponding to the top half of the
|
||||
/// address space will be copied into the top level of every page table
|
||||
/// and will never be unmapped when unmapping pages.
|
||||
const KERNEL_SEPARATE_TABLE: bool;
|
||||
|
||||
const PAGE_SHIFT: usize;
|
||||
const PAGE_ENTRY_SHIFT: usize;
|
||||
const PAGE_LEVELS: usize;
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize; // Number of bits of physical address in PTE
|
||||
const ENTRY_ADDRESS_SHIFT: usize = Self::PAGE_SHIFT; // Offset of physical address in PTE
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize;
|
||||
const ENTRY_FLAG_PRESENT: usize;
|
||||
const ENTRY_FLAG_READONLY: usize;
|
||||
const ENTRY_FLAG_READWRITE: usize;
|
||||
const ENTRY_FLAG_PAGE_USER: usize; // Leaf table user page flag
|
||||
const ENTRY_FLAG_TABLE_USER: usize = Self::ENTRY_FLAG_PAGE_USER; // Directory user page table flag
|
||||
const ENTRY_FLAG_NO_EXEC: usize;
|
||||
const ENTRY_FLAG_EXEC: usize;
|
||||
const ENTRY_FLAG_GLOBAL: usize;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize;
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize;
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize;
|
||||
|
||||
const PHYS_OFFSET: usize;
|
||||
|
||||
const PAGE_SIZE: usize = 1 << Self::PAGE_SHIFT;
|
||||
const PAGE_OFFSET_MASK: usize = Self::PAGE_SIZE - 1;
|
||||
const PAGE_ADDRESS_SHIFT: usize = Self::PAGE_LEVELS * Self::PAGE_ENTRY_SHIFT + Self::PAGE_SHIFT;
|
||||
const PAGE_ADDRESS_SIZE: u64 = 1 << (Self::PAGE_ADDRESS_SHIFT as u64);
|
||||
const PAGE_ADDRESS_MASK: usize = (Self::PAGE_ADDRESS_SIZE - (Self::PAGE_SIZE as u64)) as usize;
|
||||
const PAGE_ENTRY_SIZE: usize = 1 << (Self::PAGE_SHIFT - Self::PAGE_ENTRY_SHIFT);
|
||||
const PAGE_ENTRIES: usize = 1 << Self::PAGE_ENTRY_SHIFT;
|
||||
const PAGE_ENTRY_MASK: usize = Self::PAGE_ENTRIES - 1;
|
||||
const PAGE_NEGATIVE_MASK: usize = !(Self::PAGE_ADDRESS_SIZE - 1) as usize;
|
||||
|
||||
const ENTRY_ADDRESS_SIZE: usize = 1 << Self::ENTRY_ADDRESS_WIDTH; // size of addressable physical memory, in pages
|
||||
const ENTRY_ADDRESS_MASK: usize = Self::ENTRY_ADDRESS_SIZE - 1; // Mask of physical address, starting at 0th bit
|
||||
const ENTRY_FLAGS_MASK: usize = !(Self::ENTRY_ADDRESS_MASK << Self::ENTRY_ADDRESS_SHIFT);
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn read<T>(address: VirtualAddress) -> T {
|
||||
unsafe { ptr::read(address.data() as *const T) }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn write<T>(address: VirtualAddress, value: T) {
|
||||
unsafe { ptr::write(address.data() as *mut T, value) }
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn write_bytes(address: VirtualAddress, value: u8, count: usize) {
|
||||
unsafe { ptr::write_bytes(address.data() as *mut u8, value, count) }
|
||||
}
|
||||
|
||||
fn invalidate(address: VirtualAddress);
|
||||
fn invalidate_all();
|
||||
|
||||
fn table(table_kind: TableKind) -> PhysicalAddress;
|
||||
unsafe fn set_table(table_kind: TableKind, address: PhysicalAddress);
|
||||
|
||||
#[inline(always)]
|
||||
fn phys_to_virt(phys: PhysicalAddress) -> VirtualAddress {
|
||||
match phys.data().checked_add(Self::PHYS_OFFSET) {
|
||||
Some(some) => VirtualAddress::new(some),
|
||||
None => panic!("phys_to_virt({:#x}) overflow", phys.data()),
|
||||
}
|
||||
}
|
||||
|
||||
fn virt_is_valid(address: VirtualAddress) -> bool;
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
pub use sv39::RiscV64Sv39Arch;
|
||||
pub use sv48::RiscV64Sv48Arch;
|
||||
pub use sv57::RiscV64Sv57Arch;
|
||||
|
||||
mod sv39;
|
||||
mod sv48;
|
||||
mod sv57;
|
||||
@@ -1,124 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RiscV64Sv39Arch;
|
||||
|
||||
pub const ACCESSED: usize = 1 << 6;
|
||||
pub const DIRTY: usize = 1 << 7;
|
||||
|
||||
impl Arch for RiscV64Sv39Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each
|
||||
const PAGE_LEVELS: usize = 3; // L0, L1, L2
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 44;
|
||||
const ENTRY_ADDRESS_SHIFT: usize = 10;
|
||||
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize =
|
||||
Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY | ACCESSED | DIRTY;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 1 << 1;
|
||||
const ENTRY_FLAG_READWRITE: usize = 3 << 1;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 4;
|
||||
const ENTRY_FLAG_TABLE_USER: usize = 0;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_EXEC: usize = 1 << 3;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 1 << 5;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = 0; // FIXME use Svpbmt
|
||||
|
||||
const PHYS_OFFSET: usize = 0xFFFF_FFC0_0000_0000;
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe { asm!("sfence.vma {}", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe { asm!("sfence.vma") };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
let satp: usize;
|
||||
unsafe { asm!("csrr {0}, satp", out(reg) satp) };
|
||||
PhysicalAddress::new(
|
||||
(satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN
|
||||
)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
let satp = (8 << 60) | // Sv39 MODE
|
||||
(address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment)
|
||||
unsafe {
|
||||
asm!("csrw satp, {0}", in(reg) satp);
|
||||
Self::invalidate_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn virt_is_valid(address: VirtualAddress) -> bool {
|
||||
let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1);
|
||||
let masked = address.data() & mask;
|
||||
|
||||
masked == mask || masked == 0
|
||||
}
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(RiscV64Sv39Arch::PAGE_SIZE == 4096);
|
||||
assert!(RiscV64Sv39Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ADDRESS_SHIFT == 39);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ADDRESS_SIZE == 0x0000_0080_0000_0000);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ADDRESS_MASK == 0x0000_007F_FFFF_F000);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ENTRY_SIZE == 8);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ENTRIES == 512);
|
||||
assert!(RiscV64Sv39Arch::PAGE_ENTRY_MASK == 0x1FF);
|
||||
assert!(RiscV64Sv39Arch::PAGE_NEGATIVE_MASK == 0xFFFF_FF80_0000_0000);
|
||||
|
||||
assert!(RiscV64Sv39Arch::ENTRY_ADDRESS_SIZE == 0x0000_1000_0000_0000);
|
||||
assert!(RiscV64Sv39Arch::ENTRY_ADDRESS_MASK == 0x0000_0FFF_FFFF_FFFF);
|
||||
assert!(RiscV64Sv39Arch::ENTRY_FLAGS_MASK == 0xFFC0_0000_0000_03FF);
|
||||
|
||||
assert!(RiscV64Sv39Arch::PHYS_OFFSET == 0xFFFF_FFC0_0000_0000);
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RiscV64Sv39Arch;
|
||||
use crate::Arch;
|
||||
|
||||
#[test]
|
||||
fn is_canonical() {
|
||||
use super::VirtualAddress;
|
||||
|
||||
#[track_caller]
|
||||
fn yes(addr: usize) {
|
||||
assert!(RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr)));
|
||||
}
|
||||
#[track_caller]
|
||||
fn no(addr: usize) {
|
||||
assert!(!RiscV64Sv39Arch::virt_is_valid(VirtualAddress::new(addr)));
|
||||
}
|
||||
|
||||
yes(0xFFFF_FFFF_FFFF_FFFF);
|
||||
yes(0xFFFF_FFF0_1337_1337);
|
||||
no(0x0000_0F00_0000_0000);
|
||||
no(0x1337_0000_0000_0000);
|
||||
no(1 << 38);
|
||||
yes(1 << 37);
|
||||
|
||||
// Check for off-by-one errors.
|
||||
yes(0xFFFF_FFC0_0000_0000 | (1 << 37));
|
||||
yes(0xFFFF_FFE0_0000_0000 | (1 << 37));
|
||||
no(0xFFFF_FF80_0000_0000 | (1 << 37));
|
||||
}
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RiscV64Sv48Arch;
|
||||
|
||||
impl Arch for RiscV64Sv48Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each
|
||||
const PAGE_LEVELS: usize = 4; // L0, L1, L2, L3
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 44;
|
||||
const ENTRY_ADDRESS_SHIFT: usize = 10;
|
||||
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 1 << 1;
|
||||
const ENTRY_FLAG_READWRITE: usize = 3 << 1;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 4;
|
||||
const ENTRY_FLAG_TABLE_USER: usize = 0;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_EXEC: usize = 1 << 3;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 1 << 5;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = 0; // FIXME use Svpbmt
|
||||
|
||||
const PHYS_OFFSET: usize = 0xFFFF_8000_0000_0000;
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe { asm!("sfence.vma {}", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe { asm!("sfence.vma") };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
let satp: usize;
|
||||
unsafe { asm!("csrr {0}, satp", out(reg) satp) };
|
||||
PhysicalAddress::new(
|
||||
(satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN
|
||||
)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
let satp = (9 << 60) | // Sv48 MODE
|
||||
(address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment)
|
||||
unsafe {
|
||||
asm!("csrw satp, {0}", in(reg) satp);
|
||||
Self::invalidate_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn virt_is_valid(address: VirtualAddress) -> bool {
|
||||
// RISC-V SV48 uses 48-bit sign-extended addresses, identical to 4-level paging on x86_64.
|
||||
let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1);
|
||||
let masked = address.data() & mask;
|
||||
|
||||
masked == mask || masked == 0
|
||||
}
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(RiscV64Sv48Arch::PAGE_SIZE == 4096);
|
||||
assert!(RiscV64Sv48Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ADDRESS_SHIFT == 48);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ADDRESS_SIZE == 0x0001_0000_0000_0000);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ADDRESS_MASK == 0x0000_FFFF_FFFF_F000);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ENTRY_SIZE == 8);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ENTRIES == 512);
|
||||
assert!(RiscV64Sv48Arch::PAGE_ENTRY_MASK == 0x1FF);
|
||||
assert!(RiscV64Sv48Arch::PAGE_NEGATIVE_MASK == 0xFFFF_0000_0000_0000);
|
||||
|
||||
assert!(RiscV64Sv48Arch::ENTRY_ADDRESS_SIZE == 0x0000_1000_0000_0000);
|
||||
assert!(RiscV64Sv48Arch::ENTRY_ADDRESS_MASK == 0x0000_0FFF_FFFF_FFFF);
|
||||
assert!(RiscV64Sv48Arch::ENTRY_FLAGS_MASK == 0xFFC0_0000_0000_03FF);
|
||||
|
||||
assert!(RiscV64Sv48Arch::PHYS_OFFSET == 0xFFFF_8000_0000_0000);
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RiscV64Sv48Arch;
|
||||
use crate::Arch;
|
||||
|
||||
#[test]
|
||||
fn is_canonical() {
|
||||
use super::VirtualAddress;
|
||||
|
||||
// Close to identical when compared to x86_64 test.
|
||||
fn yes(address: usize) {
|
||||
assert!(RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(address)));
|
||||
}
|
||||
fn no(address: usize) {
|
||||
assert!(!RiscV64Sv48Arch::virt_is_valid(VirtualAddress::new(
|
||||
address
|
||||
)));
|
||||
}
|
||||
|
||||
yes(0xFFFF_8000_1337_1337);
|
||||
yes(0xFFFF_FFFF_FFFF_FFFF);
|
||||
yes(0x0000_0000_0000_0042);
|
||||
yes(0x0000_7FFF_FFFF_FFFF);
|
||||
no(0x1337_0000_0000_0000);
|
||||
no(0x1337_8000_0000_0000);
|
||||
no(0x0000_8000_0000_0000);
|
||||
}
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct RiscV64Sv57Arch;
|
||||
|
||||
impl Arch for RiscV64Sv57Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each
|
||||
const PAGE_LEVELS: usize = 5; // L0, L1, L2, L3, L4
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 44;
|
||||
const ENTRY_ADDRESS_SHIFT: usize = 10;
|
||||
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READONLY;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 1 << 1;
|
||||
const ENTRY_FLAG_READWRITE: usize = 3 << 1;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 4;
|
||||
const ENTRY_FLAG_TABLE_USER: usize = 0;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_EXEC: usize = 1 << 3;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 1 << 5;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = 0; // FIXME use Svpbmt
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = 0; // FIXME use Svpbmt
|
||||
|
||||
const PHYS_OFFSET: usize = 0xFF00_0000_0000_0000;
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe { asm!("sfence.vma {}", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe { asm!("sfence.vma") };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
let satp: usize;
|
||||
unsafe { asm!("csrr {0}, satp", out(reg) satp) };
|
||||
PhysicalAddress::new(
|
||||
(satp & Self::ENTRY_ADDRESS_MASK) << Self::PAGE_SHIFT, // Convert from PPN
|
||||
)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
let satp = (10 << 60) | // Sv57 MODE
|
||||
(address.data() >> Self::PAGE_SHIFT); // Convert to PPN (TODO: ensure alignment)
|
||||
unsafe {
|
||||
asm!("csrw satp, {0}", in(reg) satp);
|
||||
Self::invalidate_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn virt_is_valid(address: VirtualAddress) -> bool {
|
||||
let mask = !((Self::PAGE_ADDRESS_SIZE as usize - 1) >> 1);
|
||||
let masked = address.data() & mask;
|
||||
|
||||
masked == mask || masked == 0
|
||||
}
|
||||
}
|
||||
|
||||
const _: () = {
|
||||
assert!(RiscV64Sv57Arch::PAGE_SIZE == 4096);
|
||||
assert!(RiscV64Sv57Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ADDRESS_SHIFT == 57);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ADDRESS_SIZE == 0x0200_0000_0000_0000);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ADDRESS_MASK == 0x01FF_FFFF_FFFF_F000);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ENTRY_SIZE == 8);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ENTRIES == 512);
|
||||
assert!(RiscV64Sv57Arch::PAGE_ENTRY_MASK == 0x1FF);
|
||||
assert!(RiscV64Sv57Arch::PAGE_NEGATIVE_MASK == 0xFE00_0000_0000_0000);
|
||||
|
||||
assert!(RiscV64Sv57Arch::ENTRY_ADDRESS_SIZE == 0x0000_1000_0000_0000);
|
||||
assert!(RiscV64Sv57Arch::ENTRY_ADDRESS_MASK == 0x0000_0FFF_FFFF_FFFF);
|
||||
assert!(RiscV64Sv57Arch::ENTRY_FLAGS_MASK == 0xFFC0_0000_0000_03FF);
|
||||
|
||||
assert!(RiscV64Sv57Arch::PHYS_OFFSET == 0xFF00_0000_0000_0000);
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RiscV64Sv57Arch;
|
||||
use crate::Arch;
|
||||
|
||||
#[test]
|
||||
fn is_canonical() {
|
||||
use super::VirtualAddress;
|
||||
|
||||
fn yes(address: usize) {
|
||||
assert!(RiscV64Sv57Arch::virt_is_valid(VirtualAddress::new(address)));
|
||||
}
|
||||
fn no(address: usize) {
|
||||
assert!(!RiscV64Sv57Arch::virt_is_valid(VirtualAddress::new(
|
||||
address
|
||||
)));
|
||||
}
|
||||
|
||||
yes(0xFF00_0000_1337_1337);
|
||||
yes(0xFFFF_FFFF_FFFF_FFFF);
|
||||
yes(0x0000_0000_0000_0042);
|
||||
yes(0x00FF_FFFF_FFFF_FFFF);
|
||||
no(0x1337_0000_0000_0000);
|
||||
no(0x1337_8000_0000_0000);
|
||||
no(0x0F00_0000_0000_0000);
|
||||
}
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
//TODO: USE PAE
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct X86Arch;
|
||||
|
||||
impl Arch for X86Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 10; // 1024 entries, 4 bytes each
|
||||
const PAGE_LEVELS: usize = 2; // PD, PT
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 20;
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 0;
|
||||
const ENTRY_FLAG_READWRITE: usize = 1 << 1;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 2;
|
||||
// Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 1 << 8;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 0; // NOT AVAILABLE UNLESS PAE IS USED!
|
||||
const ENTRY_FLAG_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = PAT_UC_;
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = PAT_UC_;
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = PAT_WC;
|
||||
|
||||
const PHYS_OFFSET: usize = 0x8000_0000;
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe { asm!("invlpg [{0}]", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe { Self::set_table(TableKind::User, Self::table(TableKind::User)) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
let address: usize;
|
||||
unsafe { asm!("mov {0}, cr3", out(reg) address) };
|
||||
PhysicalAddress::new(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
unsafe { asm!("mov cr3, {0}", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
fn virt_is_valid(_address: VirtualAddress) -> bool {
|
||||
// On 32-bit x86, every virtual address is valid
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
pub use super::x86_shared::*;
|
||||
|
||||
const _: () = {
|
||||
assert!(X86Arch::PAGE_SIZE == 4096);
|
||||
assert!(X86Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(X86Arch::PAGE_ADDRESS_SHIFT == 32);
|
||||
assert!(X86Arch::PAGE_ADDRESS_SIZE == 0x0000_0001_0000_0000);
|
||||
assert!(X86Arch::PAGE_ADDRESS_MASK == 0xFFFF_F000);
|
||||
assert!(X86Arch::PAGE_ENTRY_SIZE == 4);
|
||||
assert!(X86Arch::PAGE_ENTRIES == 1024);
|
||||
assert!(X86Arch::PAGE_ENTRY_MASK == 0x3FF);
|
||||
assert!(X86Arch::PAGE_NEGATIVE_MASK == 0x0000_0000_0000);
|
||||
|
||||
assert!(X86Arch::ENTRY_ADDRESS_SIZE == 0x0000_0000_0010_0000);
|
||||
assert!(X86Arch::ENTRY_ADDRESS_MASK == 0x000F_FFFF);
|
||||
assert!(X86Arch::ENTRY_FLAGS_MASK == 0x0000_0FFF);
|
||||
|
||||
assert!(X86Arch::PHYS_OFFSET == 0x8000_0000);
|
||||
};
|
||||
@@ -1,107 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
use crate::{Arch, PhysicalAddress, TableKind, VirtualAddress};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct X8664Arch;
|
||||
|
||||
impl Arch for X8664Arch {
|
||||
const KERNEL_SEPARATE_TABLE: bool = false;
|
||||
|
||||
const PAGE_SHIFT: usize = 12; // 4096 bytes
|
||||
const PAGE_ENTRY_SHIFT: usize = 9; // 512 entries, 8 bytes each
|
||||
const PAGE_LEVELS: usize = 4; // PML4, PDP, PD, PT
|
||||
|
||||
const ENTRY_ADDRESS_WIDTH: usize = 40;
|
||||
const ENTRY_FLAG_DEFAULT_PAGE: usize = Self::ENTRY_FLAG_PRESENT;
|
||||
const ENTRY_FLAG_DEFAULT_TABLE: usize = Self::ENTRY_FLAG_PRESENT | Self::ENTRY_FLAG_READWRITE;
|
||||
const ENTRY_FLAG_PRESENT: usize = 1 << 0;
|
||||
const ENTRY_FLAG_READONLY: usize = 0;
|
||||
const ENTRY_FLAG_READWRITE: usize = 1 << 1;
|
||||
const ENTRY_FLAG_PAGE_USER: usize = 1 << 2;
|
||||
// Not used: const ENTRY_FLAG_HUGE: usize = 1 << 7;
|
||||
const ENTRY_FLAG_GLOBAL: usize = 1 << 8;
|
||||
const ENTRY_FLAG_NO_GLOBAL: usize = 0;
|
||||
const ENTRY_FLAG_NO_EXEC: usize = 1 << 63;
|
||||
const ENTRY_FLAG_EXEC: usize = 0;
|
||||
const ENTRY_FLAG_DEVICE_MEMORY: usize = PAT_UC_;
|
||||
const ENTRY_FLAG_UNCACHEABLE: usize = PAT_UC_;
|
||||
const ENTRY_FLAG_WRITE_COMBINING: usize = PAT_WC;
|
||||
|
||||
const PHYS_OFFSET: usize = Self::PAGE_NEGATIVE_MASK + (Self::PAGE_ADDRESS_SIZE >> 1) as usize; // PML4 slot 256 and onwards
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate(address: VirtualAddress) {
|
||||
unsafe { asm!("invlpg [{0}]", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn invalidate_all() {
|
||||
unsafe { Self::set_table(TableKind::User, Self::table(TableKind::User)) };
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn table(_table_kind: TableKind) -> PhysicalAddress {
|
||||
let address: usize;
|
||||
unsafe { asm!("mov {0}, cr3", out(reg) address) };
|
||||
PhysicalAddress::new(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn set_table(_table_kind: TableKind, address: PhysicalAddress) {
|
||||
unsafe { asm!("mov cr3, {0}", in(reg) address.data()) };
|
||||
}
|
||||
|
||||
fn virt_is_valid(address: VirtualAddress) -> bool {
|
||||
// On x86_64, an address is valid if and only if it is canonical. It may still point to
|
||||
// unmapped memory, but will always be valid once translated via the page table has
|
||||
// suceeded.
|
||||
let masked = address.data() & 0xFFFF_8000_0000_0000;
|
||||
// TODO: 5-level paging
|
||||
masked == 0xFFFF_8000_0000_0000 || masked == 0
|
||||
}
|
||||
}
|
||||
|
||||
pub use super::x86_shared::*;
|
||||
|
||||
const _: () = {
|
||||
assert!(X8664Arch::PAGE_SIZE == 4096);
|
||||
assert!(X8664Arch::PAGE_OFFSET_MASK == 0xFFF);
|
||||
assert!(X8664Arch::PAGE_ADDRESS_SHIFT == 48);
|
||||
assert!(X8664Arch::PAGE_ADDRESS_SIZE == 0x0001_0000_0000_0000);
|
||||
assert!(X8664Arch::PAGE_ADDRESS_MASK == 0x0000_FFFF_FFFF_F000);
|
||||
assert!(X8664Arch::PAGE_ENTRY_SIZE == 8);
|
||||
assert!(X8664Arch::PAGE_ENTRIES == 512);
|
||||
assert!(X8664Arch::PAGE_ENTRY_MASK == 0x1FF);
|
||||
assert!(X8664Arch::PAGE_NEGATIVE_MASK == 0xFFFF_0000_0000_0000);
|
||||
|
||||
assert!(X8664Arch::ENTRY_ADDRESS_SIZE == 0x0000_0100_0000_0000);
|
||||
assert!(X8664Arch::ENTRY_ADDRESS_MASK == 0x0000_00FF_FFFF_FFFF);
|
||||
assert!(X8664Arch::ENTRY_FLAGS_MASK == 0xFFF0_0000_0000_0FFF);
|
||||
|
||||
assert!(X8664Arch::PHYS_OFFSET == 0xFFFF_8000_0000_0000);
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{VirtualAddress, X8664Arch};
|
||||
use crate::Arch;
|
||||
|
||||
#[test]
|
||||
fn is_canonical() {
|
||||
fn yes(address: usize) {
|
||||
assert!(X8664Arch::virt_is_valid(VirtualAddress::new(address)));
|
||||
}
|
||||
fn no(address: usize) {
|
||||
assert!(!X8664Arch::virt_is_valid(VirtualAddress::new(address)));
|
||||
}
|
||||
|
||||
yes(0xFFFF_8000_1337_1337);
|
||||
yes(0xFFFF_FFFF_FFFF_FFFF);
|
||||
yes(0x0000_0000_0000_0042);
|
||||
yes(0x0000_7FFF_FFFF_FFFF);
|
||||
no(0x1337_0000_0000_0000);
|
||||
no(0x1337_8000_0000_0000);
|
||||
no(0x0000_8000_0000_0000);
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
#![expect(clippy::identity_op)]
|
||||
|
||||
// Page attribute table is indexed by PAT(7) PCD(4) PWT(3)
|
||||
pub(crate) const _PAT_WB: usize = (0b0 << 7) + (0b00 << 3);
|
||||
pub(crate) const _PAT_WT: usize = (0b0 << 7) + (0b01 << 3);
|
||||
pub(crate) const PAT_UC_: usize = (0b0 << 7) + (0b10 << 3); // UC-
|
||||
pub(crate) const _PAT_UC: usize = (0b0 << 7) + (0b11 << 3); // UC
|
||||
pub(crate) const PAT_WC: usize = (0b1 << 7) + (0b00 << 3);
|
||||
|
||||
/// Setup page attribute table
|
||||
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
||||
#[inline(always)]
|
||||
pub unsafe fn init_pat() {
|
||||
unsafe {
|
||||
let uncacheable = 0; // UC
|
||||
let write_combining = 1; // WC
|
||||
let write_through = 4; // WT
|
||||
let _write_protected = 5; // WP
|
||||
let write_back = 6; // WB
|
||||
let uncached = 7; // UC- (overridable by WC MTRR)
|
||||
|
||||
let pat0 = write_back;
|
||||
let pat1 = write_through;
|
||||
let pat2 = uncached;
|
||||
let pat3 = uncacheable;
|
||||
|
||||
let pat4 = write_combining;
|
||||
let pat5 = pat1;
|
||||
let pat6 = pat2;
|
||||
let pat7 = pat3;
|
||||
|
||||
let msr = 631; // IA32_PAT
|
||||
let low = u32::from_be_bytes([pat3, pat2, pat1, pat0]);
|
||||
let high = u32::from_be_bytes([pat7, pat6, pat5, pat4]);
|
||||
core::arch::asm!("wrmsr", in("ecx") msr, in("eax") low, in("edx") high);
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
#![no_std]
|
||||
#![allow(clippy::new_without_default)]
|
||||
|
||||
pub use crate::{allocator::*, arch::*, page::*};
|
||||
|
||||
mod allocator;
|
||||
mod arch;
|
||||
mod page;
|
||||
|
||||
pub const KILOBYTE: usize = 1024;
|
||||
pub const MEGABYTE: usize = KILOBYTE * 1024;
|
||||
pub const GIGABYTE: usize = MEGABYTE * 1024;
|
||||
#[cfg(target_pointer_width = "64")]
|
||||
pub const TERABYTE: usize = GIGABYTE * 1024;
|
||||
|
||||
/// Specific table to be used, needed on some architectures
|
||||
//TODO: Use this throughout the code
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub enum TableKind {
|
||||
/// Userspace page table
|
||||
User,
|
||||
/// Kernel page table
|
||||
Kernel,
|
||||
}
|
||||
|
||||
/// Physical memory address
|
||||
#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
#[repr(transparent)]
|
||||
pub struct PhysicalAddress(usize);
|
||||
|
||||
impl PhysicalAddress {
|
||||
#[inline(always)]
|
||||
pub const fn new(address: usize) -> Self {
|
||||
Self(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn data(&self) -> usize {
|
||||
self.0
|
||||
}
|
||||
|
||||
#[expect(clippy::should_implement_trait)]
|
||||
#[inline(always)]
|
||||
pub fn add(self, offset: usize) -> Self {
|
||||
Self(self.0 + offset)
|
||||
}
|
||||
}
|
||||
|
||||
impl core::fmt::Debug for PhysicalAddress {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
write!(f, "[phys {:#0x}]", self.data())
|
||||
}
|
||||
}
|
||||
|
||||
/// Virtual memory address
|
||||
#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)]
|
||||
#[repr(transparent)]
|
||||
pub struct VirtualAddress(usize);
|
||||
|
||||
impl VirtualAddress {
|
||||
#[inline(always)]
|
||||
pub const fn new(address: usize) -> Self {
|
||||
Self(address)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn data(&self) -> usize {
|
||||
self.0
|
||||
}
|
||||
|
||||
#[expect(clippy::should_implement_trait)]
|
||||
#[inline(always)]
|
||||
pub fn add(self, offset: usize) -> Self {
|
||||
Self(self.0 + offset)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn kind(&self) -> TableKind {
|
||||
if (self.0 as isize) < 0 {
|
||||
TableKind::Kernel
|
||||
} else {
|
||||
TableKind::User
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl core::fmt::Debug for VirtualAddress {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
write!(f, "[virt {:#0x}]", self.data())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct MemoryArea {
|
||||
pub base: PhysicalAddress,
|
||||
pub size: usize,
|
||||
}
|
||||
@@ -1,309 +0,0 @@
|
||||
#![cfg(target_pointer_width = "64")]
|
||||
|
||||
use rmm::{
|
||||
emulate::EmulateArch, Arch, BuddyAllocator, BumpAllocator, Flusher, FrameAllocator, FrameCount,
|
||||
MemoryArea, PageFlags, PageFlushAll, PageMapper, PageTable, PhysicalAddress, TableKind,
|
||||
VirtualAddress, GIGABYTE, KILOBYTE, MEGABYTE, TERABYTE,
|
||||
};
|
||||
use std::marker::PhantomData;
|
||||
|
||||
pub fn format_size(size: usize) -> String {
|
||||
if size >= 2 * TERABYTE {
|
||||
format!("{} TB", size / TERABYTE)
|
||||
} else if size >= 2 * GIGABYTE {
|
||||
format!("{} GB", size / GIGABYTE)
|
||||
} else if size >= 2 * MEGABYTE {
|
||||
format!("{} MB", size / MEGABYTE)
|
||||
} else if size >= 2 * KILOBYTE {
|
||||
format!("{} KB", size / KILOBYTE)
|
||||
} else {
|
||||
format!("{} B", size)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
unsafe fn dump_tables<A: Arch>(table: PageTable<A>) {
|
||||
unsafe {
|
||||
let level = table.level();
|
||||
for i in 0..A::PAGE_ENTRIES {
|
||||
if level == 0 {
|
||||
if let Some(entry) = table.entry(i) {
|
||||
if entry.present() {
|
||||
let base = table.entry_base(i).unwrap();
|
||||
println!(
|
||||
"0x{:X}: 0x{:X}",
|
||||
base.data(),
|
||||
entry.address().unwrap().data()
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if let Some(next) = table.next(i) {
|
||||
dump_tables(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SlabNode<A> {
|
||||
next: PhysicalAddress,
|
||||
count: usize,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> SlabNode<A> {
|
||||
pub fn new(next: PhysicalAddress, count: usize) -> Self {
|
||||
Self {
|
||||
next,
|
||||
count,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn empty() -> Self {
|
||||
Self::new(PhysicalAddress::new(0), 0)
|
||||
}
|
||||
|
||||
pub unsafe fn insert(&mut self, phys: PhysicalAddress) {
|
||||
unsafe {
|
||||
let virt = A::phys_to_virt(phys);
|
||||
A::write(virt, self.next);
|
||||
self.next = phys;
|
||||
self.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn remove(&mut self) -> Option<PhysicalAddress> {
|
||||
unsafe {
|
||||
if self.count > 0 {
|
||||
let phys = self.next;
|
||||
let virt = A::phys_to_virt(phys);
|
||||
self.next = A::read(virt);
|
||||
self.count -= 1;
|
||||
Some(phys)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SlabAllocator<A> {
|
||||
//TODO: Allow allocations up to maximum pageable size
|
||||
nodes: [SlabNode<A>; 4],
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> SlabAllocator<A> {
|
||||
pub unsafe fn new(areas: &'static [MemoryArea], offset: usize) -> Self {
|
||||
unsafe {
|
||||
let mut allocator = Self {
|
||||
nodes: [
|
||||
SlabNode::empty(),
|
||||
SlabNode::empty(),
|
||||
SlabNode::empty(),
|
||||
SlabNode::empty(),
|
||||
],
|
||||
phantom: PhantomData,
|
||||
};
|
||||
|
||||
// Add unused areas to free lists
|
||||
let mut area_offset = offset;
|
||||
for area in areas.iter() {
|
||||
if area_offset < area.size {
|
||||
let area_base = area.base.add(area_offset);
|
||||
let area_size = area.size - area_offset;
|
||||
allocator.free(area_base, area_size);
|
||||
area_offset = 0;
|
||||
} else {
|
||||
area_offset -= area.size;
|
||||
}
|
||||
}
|
||||
|
||||
allocator
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn allocate(&mut self, size: usize) -> Option<PhysicalAddress> {
|
||||
unsafe {
|
||||
for level in 0..A::PAGE_LEVELS - 1 {
|
||||
let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT;
|
||||
let level_size = 1 << level_shift;
|
||||
if size <= level_size {
|
||||
if let Some(base) = self.nodes[level].remove() {
|
||||
self.free(base.add(size), level_size - size);
|
||||
return Some(base);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: This causes fragmentation, since neighbors are not identified
|
||||
//TODO: remainders less than PAGE_SIZE will be lost
|
||||
pub unsafe fn free(&mut self, mut base: PhysicalAddress, mut size: usize) {
|
||||
unsafe {
|
||||
for level in (0..A::PAGE_LEVELS - 1).rev() {
|
||||
let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT;
|
||||
let level_size = 1 << level_shift;
|
||||
while size >= level_size {
|
||||
println!("Add {:X} {}", base.data(), format_size(level_size));
|
||||
self.nodes[level].insert(base);
|
||||
base = base.add(level_size);
|
||||
size -= level_size;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn remaining(&mut self) -> usize {
|
||||
let mut remaining = 0;
|
||||
for level in (0..A::PAGE_LEVELS - 1).rev() {
|
||||
let level_shift = level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT;
|
||||
let level_size = 1 << level_shift;
|
||||
remaining += self.nodes[level].count * level_size;
|
||||
}
|
||||
remaining
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn new_tables<A: Arch>(areas: &'static [MemoryArea]) {
|
||||
unsafe {
|
||||
// First, calculate how much memory we have
|
||||
let mut size = 0;
|
||||
for area in areas.iter() {
|
||||
size += area.size;
|
||||
}
|
||||
|
||||
println!("Memory: {}", format_size(size));
|
||||
|
||||
// Create a basic allocator for the first pages
|
||||
let mut bump_allocator = BumpAllocator::<A>::new(areas, 0);
|
||||
|
||||
{
|
||||
// Map all physical areas at PHYS_OFFSET
|
||||
let mut mapper = PageMapper::<A, _>::create(TableKind::Kernel, &mut bump_allocator)
|
||||
.expect("failed to create Mapper");
|
||||
for area in areas.iter() {
|
||||
for i in 0..area.size / A::PAGE_SIZE {
|
||||
let phys = area.base.add(i * A::PAGE_SIZE);
|
||||
let (_, flush) = mapper
|
||||
.map_linearly(phys, PageFlags::<A>::new().write(true))
|
||||
.expect("failed to map page to frame");
|
||||
flush.ignore(); // Not the active table
|
||||
}
|
||||
}
|
||||
|
||||
// Use the new table
|
||||
mapper.make_current();
|
||||
}
|
||||
|
||||
// Create the physical memory map
|
||||
let offset = bump_allocator.offset();
|
||||
println!("Permanently used: {}", format_size(offset));
|
||||
|
||||
let mut allocator = BuddyAllocator::<A>::new(bump_allocator).unwrap();
|
||||
|
||||
for i in 0..16 {
|
||||
{
|
||||
let phys_opt = allocator.allocate_one();
|
||||
println!("page {}: {:X?}", i, phys_opt);
|
||||
if i % 3 == 0 {
|
||||
if let Some(phys) = phys_opt {
|
||||
println!("free {}: {:X?}", i, phys_opt);
|
||||
allocator.free_one(phys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let phys_opt = allocator.allocate(FrameCount::new(16));
|
||||
println!("page*16 {}: {:X?}", i, phys_opt);
|
||||
if i % 2 == 0 {
|
||||
if let Some(phys) = phys_opt {
|
||||
println!("free*16 {}: {:X?}", i, phys_opt);
|
||||
allocator.free(phys, FrameCount::new(16));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut mapper = PageMapper::<A, _>::current(TableKind::Kernel, &mut allocator);
|
||||
let mut flush_all = PageFlushAll::new();
|
||||
for i in 0..16 {
|
||||
let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE);
|
||||
let phys = mapper
|
||||
.allocator_mut()
|
||||
.allocate_one()
|
||||
.expect("failed to map page");
|
||||
let flush = mapper
|
||||
.map_phys(virt, phys, PageFlags::<A>::new().user(true).write(true))
|
||||
.expect("failed to map page");
|
||||
flush_all.consume(flush);
|
||||
}
|
||||
flush_all.flush();
|
||||
|
||||
let mut flush_all = PageFlushAll::new();
|
||||
for i in 0..16 {
|
||||
let virt = VirtualAddress::new(MEGABYTE + i * A::PAGE_SIZE);
|
||||
let (old, _, flush) = mapper.unmap_phys(virt).expect("failed to unmap page");
|
||||
mapper.allocator_mut().free_one(old);
|
||||
flush_all.consume(flush);
|
||||
}
|
||||
flush_all.flush();
|
||||
|
||||
let usage = allocator.usage();
|
||||
println!("Allocator usage:");
|
||||
println!(
|
||||
" Used: {}",
|
||||
format_size(usage.used().data() * A::PAGE_SIZE)
|
||||
);
|
||||
println!(
|
||||
" Free: {}",
|
||||
format_size(usage.free().data() * A::PAGE_SIZE)
|
||||
);
|
||||
println!(
|
||||
" Total: {}",
|
||||
format_size(usage.total().data() * A::PAGE_SIZE)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn main() {
|
||||
unsafe {
|
||||
let areas = EmulateArch::init();
|
||||
|
||||
// Debug table
|
||||
//dump_tables(PageTable::<A>::top());
|
||||
|
||||
new_tables::<EmulateArch>(areas);
|
||||
|
||||
//dump_tables(PageTable::<A>::top());
|
||||
|
||||
for i in &[1, 2, 4, 8, 16, 32] {
|
||||
let phys = PhysicalAddress::new(i * MEGABYTE);
|
||||
let virt = EmulateArch::phys_to_virt(phys);
|
||||
|
||||
// Test read
|
||||
println!(
|
||||
"0x{:X} (0x{:X}) = 0x{:X}",
|
||||
virt.data(),
|
||||
phys.data(),
|
||||
EmulateArch::read::<u8>(virt)
|
||||
);
|
||||
|
||||
// Test write
|
||||
EmulateArch::write::<u8>(virt, 0x5A);
|
||||
|
||||
// Test read
|
||||
println!(
|
||||
"0x{:X} (0x{:X}) = 0x{:X}",
|
||||
virt.data(),
|
||||
phys.data(),
|
||||
EmulateArch::read::<u8>(virt)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
use core::marker::PhantomData;
|
||||
|
||||
use crate::{Arch, PageFlags, PhysicalAddress};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct PageEntry<A> {
|
||||
data: usize,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> PageEntry<A> {
|
||||
#[inline(always)]
|
||||
pub fn new(address: usize, flags: usize) -> Self {
|
||||
let data = (((address >> A::PAGE_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::ENTRY_ADDRESS_SHIFT)
|
||||
| flags;
|
||||
Self::from_data(data)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn from_data(data: usize) -> Self {
|
||||
Self {
|
||||
data,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn data(&self) -> usize {
|
||||
self.data
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn address(&self) -> Result<PhysicalAddress, PhysicalAddress> {
|
||||
let addr = PhysicalAddress(
|
||||
((self.data >> A::ENTRY_ADDRESS_SHIFT) & A::ENTRY_ADDRESS_MASK) << A::PAGE_SHIFT,
|
||||
);
|
||||
|
||||
if self.present() {
|
||||
Ok(addr)
|
||||
} else {
|
||||
Err(addr)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn flags(&self) -> PageFlags<A> {
|
||||
unsafe { PageFlags::from_data(self.data & A::ENTRY_FLAGS_MASK) }
|
||||
}
|
||||
#[inline(always)]
|
||||
pub fn set_flags(&mut self, flags: PageFlags<A>) {
|
||||
self.data &= !A::ENTRY_FLAGS_MASK;
|
||||
self.data |= flags.data();
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn present(&self) -> bool {
|
||||
self.data & A::ENTRY_FLAG_PRESENT != 0
|
||||
}
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
use core::{fmt, marker::PhantomData};
|
||||
|
||||
use crate::Arch;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct PageFlags<A> {
|
||||
data: usize,
|
||||
arch: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> PageFlags<A> {
|
||||
#[inline(always)]
|
||||
pub fn new() -> Self {
|
||||
unsafe {
|
||||
Self::from_data(
|
||||
// Flags set to present, kernel space, read-only, no-execute by default
|
||||
A::ENTRY_FLAG_DEFAULT_PAGE
|
||||
| A::ENTRY_FLAG_READONLY
|
||||
| A::ENTRY_FLAG_NO_EXEC
|
||||
| A::ENTRY_FLAG_NO_GLOBAL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn new_table() -> Self {
|
||||
unsafe {
|
||||
Self::from_data(
|
||||
// Flags set to present, kernel space, read-only, no-execute by default
|
||||
A::ENTRY_FLAG_DEFAULT_TABLE | A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_NO_GLOBAL,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn from_data(data: usize) -> Self {
|
||||
Self {
|
||||
data,
|
||||
arch: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn data(&self) -> usize {
|
||||
self.data
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn custom_flag(mut self, flag: usize, value: bool) -> Self {
|
||||
if value {
|
||||
self.data |= flag;
|
||||
} else {
|
||||
self.data &= !flag;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn device_memory(self, value: bool) -> Self {
|
||||
self.custom_flag(A::ENTRY_FLAG_DEVICE_MEMORY, value)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn uncacheable(self, value: bool) -> Self {
|
||||
self.custom_flag(A::ENTRY_FLAG_UNCACHEABLE, value)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn write_combining(self, value: bool) -> Self {
|
||||
self.custom_flag(A::ENTRY_FLAG_WRITE_COMBINING, value)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn has_flag(&self, flag: usize) -> bool {
|
||||
self.data & flag == flag
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn has_present(&self) -> bool {
|
||||
self.has_flag(A::ENTRY_FLAG_PRESENT)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn user(self, value: bool) -> Self {
|
||||
self.custom_flag(A::ENTRY_FLAG_PAGE_USER, value)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn has_user(&self) -> bool {
|
||||
self.has_flag(A::ENTRY_FLAG_PAGE_USER)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn write(self, value: bool) -> Self {
|
||||
// Architecture may use readonly or readwrite, or both, support either
|
||||
if value {
|
||||
self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false)
|
||||
.custom_flag(A::ENTRY_FLAG_READWRITE, true)
|
||||
} else {
|
||||
self.custom_flag(A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE, false)
|
||||
.custom_flag(A::ENTRY_FLAG_READONLY, true)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn has_write(&self) -> bool {
|
||||
// Architecture may use readonly or readwrite, or both, support either
|
||||
self.data & (A::ENTRY_FLAG_READONLY | A::ENTRY_FLAG_READWRITE) == A::ENTRY_FLAG_READWRITE
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn execute(self, value: bool) -> Self {
|
||||
//TODO: write xor execute?
|
||||
// Architecture may use no exec or exec, support either
|
||||
self.custom_flag(A::ENTRY_FLAG_NO_EXEC, !value)
|
||||
.custom_flag(A::ENTRY_FLAG_EXEC, value)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn has_execute(&self) -> bool {
|
||||
// Architecture may use no exec or exec, support either
|
||||
self.data & (A::ENTRY_FLAG_NO_EXEC | A::ENTRY_FLAG_EXEC) == A::ENTRY_FLAG_EXEC
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[inline(always)]
|
||||
pub fn global(self, value: bool) -> Self {
|
||||
// Architecture may use global or non global, support either
|
||||
self.custom_flag(A::ENTRY_FLAG_NO_GLOBAL, !value)
|
||||
.custom_flag(A::ENTRY_FLAG_GLOBAL, value)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn is_global(&self) -> bool {
|
||||
// Architecture may use global or non global, support either
|
||||
self.data & (A::ENTRY_FLAG_GLOBAL | A::ENTRY_FLAG_NO_GLOBAL) == A::ENTRY_FLAG_GLOBAL
|
||||
}
|
||||
}
|
||||
|
||||
impl<A: Arch> fmt::Debug for PageFlags<A> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("PageFlags")
|
||||
.field("present", &self.has_present())
|
||||
.field("write", &self.has_write())
|
||||
.field("executable", &self.has_execute())
|
||||
.field("user", &self.has_user())
|
||||
.field("bits", &format_args!("{:#0x}", self.data))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
use core::{marker::PhantomData, mem};
|
||||
|
||||
use crate::{Arch, VirtualAddress};
|
||||
|
||||
pub trait Flusher<A> {
|
||||
fn consume(&mut self, flush: PageFlush<A>);
|
||||
}
|
||||
|
||||
#[must_use = "The page table must be flushed, or the changes unsafely ignored"]
|
||||
pub struct PageFlush<A> {
|
||||
virt: VirtualAddress,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> PageFlush<A> {
|
||||
pub fn new(virt: VirtualAddress) -> Self {
|
||||
Self {
|
||||
virt,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flush(self) {
|
||||
A::invalidate(self.virt);
|
||||
}
|
||||
|
||||
#[expect(clippy::forget_non_drop)]
|
||||
pub unsafe fn ignore(self) {
|
||||
mem::forget(self);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Might remove Drop and add #[must_use] again, but ergonomically I prefer being able to pass
|
||||
// a flusher, and have it dropped by the end of the function it is passed to, in order to flush.
|
||||
pub struct PageFlushAll<A: Arch> {
|
||||
phantom: PhantomData<fn() -> A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> PageFlushAll<A> {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn flush(self) {}
|
||||
|
||||
pub unsafe fn ignore(self) {
|
||||
mem::forget(self);
|
||||
}
|
||||
}
|
||||
impl<A: Arch> Drop for PageFlushAll<A> {
|
||||
fn drop(&mut self) {
|
||||
A::invalidate_all();
|
||||
}
|
||||
}
|
||||
impl<A: Arch> Flusher<A> for PageFlushAll<A> {
|
||||
fn consume(&mut self, flush: PageFlush<A>) {
|
||||
unsafe {
|
||||
flush.ignore();
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<A: Arch, T: Flusher<A> + ?Sized> Flusher<A> for &mut T {
|
||||
fn consume(&mut self, flush: PageFlush<A>) {
|
||||
<T as Flusher<A>>::consume(self, flush)
|
||||
}
|
||||
}
|
||||
impl<A: Arch> Flusher<A> for () {
|
||||
fn consume(&mut self, _: PageFlush<A>) {}
|
||||
}
|
||||
@@ -1,269 +0,0 @@
|
||||
use core::marker::PhantomData;
|
||||
|
||||
use crate::{
|
||||
Arch, FrameAllocator, PageEntry, PageFlags, PageFlush, PageTable, PhysicalAddress, TableKind,
|
||||
VirtualAddress,
|
||||
};
|
||||
|
||||
pub struct PageMapper<A, F> {
|
||||
table_kind: TableKind,
|
||||
table_addr: PhysicalAddress,
|
||||
allocator: F,
|
||||
_phantom: PhantomData<fn() -> A>,
|
||||
}
|
||||
|
||||
impl<A: Arch, F> PageMapper<A, F> {
|
||||
unsafe fn new(table_kind: TableKind, table_addr: PhysicalAddress, allocator: F) -> Self {
|
||||
Self {
|
||||
table_kind,
|
||||
table_addr,
|
||||
allocator,
|
||||
_phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn current(table_kind: TableKind, allocator: F) -> Self {
|
||||
unsafe {
|
||||
let table_addr = A::table(table_kind);
|
||||
Self::new(table_kind, table_addr, allocator)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_current(&self) -> bool {
|
||||
self.table().phys() == A::table(self.table_kind)
|
||||
}
|
||||
|
||||
pub unsafe fn make_current(&self) {
|
||||
unsafe {
|
||||
A::set_table(self.table_kind, self.table_addr);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn table(&self) -> PageTable<A> {
|
||||
// SAFETY: The only way to initialize a PageMapper is via new(), and we assume it upholds
|
||||
// all necessary invariants for this to be safe.
|
||||
unsafe { PageTable::new(VirtualAddress::new(0), self.table_addr, A::PAGE_LEVELS - 1) }
|
||||
}
|
||||
|
||||
pub fn allocator(&self) -> &F {
|
||||
&self.allocator
|
||||
}
|
||||
|
||||
pub fn allocator_mut(&mut self) -> &mut F {
|
||||
&mut self.allocator
|
||||
}
|
||||
|
||||
fn visit<T>(
|
||||
&self,
|
||||
virt: VirtualAddress,
|
||||
f: impl FnOnce(&mut PageTable<A>, usize) -> T,
|
||||
) -> Option<T> {
|
||||
let mut table = self.table();
|
||||
loop {
|
||||
let i = table.index_of(virt)?;
|
||||
if table.level() == 0 {
|
||||
return Some(f(&mut table, i));
|
||||
} else {
|
||||
table = unsafe { table.next(i)? };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn translate(&self, virt: VirtualAddress) -> Option<(PhysicalAddress, PageFlags<A>)> {
|
||||
let entry = self.visit(virt, |p1, i| unsafe { p1.entry(i) })??;
|
||||
Some((entry.address().ok()?, entry.flags()))
|
||||
}
|
||||
|
||||
pub unsafe fn remap_with_full(
|
||||
&mut self,
|
||||
virt: VirtualAddress,
|
||||
f: impl FnOnce(PhysicalAddress, PageFlags<A>) -> Option<(PhysicalAddress, PageFlags<A>)>,
|
||||
) -> Option<(PageFlags<A>, PhysicalAddress, PageFlush<A>)> {
|
||||
unsafe {
|
||||
self.visit(virt, |p1, i| {
|
||||
let old_entry = p1.entry(i)?;
|
||||
let old_phys = old_entry.address().ok()?;
|
||||
let old_flags = old_entry.flags();
|
||||
let (new_phys, new_flags) = f(old_phys, old_flags)?;
|
||||
// TODO: Higher-level PageEntry::new interface?
|
||||
let new_entry = PageEntry::new(new_phys.data(), new_flags.data());
|
||||
p1.set_entry(i, new_entry);
|
||||
Some((old_flags, old_phys, PageFlush::new(virt)))
|
||||
})
|
||||
.flatten()
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn remap_with(
|
||||
&mut self,
|
||||
virt: VirtualAddress,
|
||||
map_flags: impl FnOnce(PageFlags<A>) -> PageFlags<A>,
|
||||
) -> Option<(PageFlags<A>, PhysicalAddress, PageFlush<A>)> {
|
||||
unsafe {
|
||||
self.remap_with_full(virt, |same_phys, old_flags| {
|
||||
Some((same_phys, map_flags(old_flags)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn remap(
|
||||
&mut self,
|
||||
virt: VirtualAddress,
|
||||
flags: PageFlags<A>,
|
||||
) -> Option<PageFlush<A>> {
|
||||
unsafe { self.remap_with(virt, |_| flags).map(|(_, _, flush)| flush) }
|
||||
}
|
||||
}
|
||||
|
||||
impl<A: Arch, F: FrameAllocator> PageMapper<A, F> {
|
||||
pub unsafe fn create(table_kind: TableKind, mut allocator: F) -> Option<Self> {
|
||||
unsafe {
|
||||
let table_addr = allocator.allocate_one()?;
|
||||
let mut table = Self::new(table_kind, table_addr, allocator);
|
||||
|
||||
match (table_kind, A::KERNEL_SEPARATE_TABLE) {
|
||||
(TableKind::Kernel, false) => {
|
||||
// Pre-allocate all kernel top-level page table entries so that when
|
||||
// the page table is copied, these entries are synced between processes.
|
||||
for i in A::PAGE_ENTRIES / 2..A::PAGE_ENTRIES {
|
||||
let phys = table
|
||||
.allocator
|
||||
.allocate_one()
|
||||
.expect("failed to map page table");
|
||||
let flags = A::ENTRY_FLAG_DEFAULT_TABLE;
|
||||
table
|
||||
.table()
|
||||
.set_entry(i, PageEntry::new(phys.data(), flags));
|
||||
}
|
||||
}
|
||||
(TableKind::User, false) => {
|
||||
// Copy higher half (kernel) mappings
|
||||
let active_ktable = PageMapper::current(TableKind::Kernel, ());
|
||||
for i in A::PAGE_ENTRIES / 2..A::PAGE_ENTRIES {
|
||||
if let Some(entry) = active_ktable.table().entry(i) {
|
||||
table.table().set_entry(i, entry);
|
||||
}
|
||||
}
|
||||
}
|
||||
(_, true) => {
|
||||
// There is a separate page table for the kernel. No need to copy the kernel
|
||||
// mappings to the user page table.
|
||||
}
|
||||
}
|
||||
|
||||
Some(table)
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn map_phys(
|
||||
&mut self,
|
||||
virt: VirtualAddress,
|
||||
phys: PhysicalAddress,
|
||||
flags: PageFlags<A>,
|
||||
) -> Option<PageFlush<A>> {
|
||||
unsafe {
|
||||
//TODO: verify virt and phys are aligned
|
||||
//TODO: verify flags have correct bits
|
||||
let entry = PageEntry::new(phys.data(), flags.data());
|
||||
let mut table = self.table();
|
||||
loop {
|
||||
let i = table.index_of(virt)?;
|
||||
if table.level() == 0 {
|
||||
//TODO: check for overwriting entry
|
||||
table.set_entry(i, entry);
|
||||
return Some(PageFlush::new(virt));
|
||||
}
|
||||
|
||||
let next = match table.next(i) {
|
||||
Some(some) => some,
|
||||
None => {
|
||||
let next_phys = self.allocator.allocate_one()?;
|
||||
//TODO: correct flags?
|
||||
let flags = A::ENTRY_FLAG_DEFAULT_TABLE
|
||||
| if virt.kind() == TableKind::User {
|
||||
A::ENTRY_FLAG_TABLE_USER
|
||||
} else {
|
||||
0
|
||||
};
|
||||
table.set_entry(i, PageEntry::new(next_phys.data(), flags));
|
||||
table.next(i)?
|
||||
}
|
||||
};
|
||||
table = next;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn map_linearly(
|
||||
&mut self,
|
||||
phys: PhysicalAddress,
|
||||
flags: PageFlags<A>,
|
||||
) -> Option<(VirtualAddress, PageFlush<A>)> {
|
||||
unsafe {
|
||||
let virt = A::phys_to_virt(phys);
|
||||
self.map_phys(virt, phys, flags).map(|flush| (virt, flush))
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn unmap_phys(
|
||||
&mut self,
|
||||
virt: VirtualAddress,
|
||||
) -> Option<(PhysicalAddress, PageFlags<A>, PageFlush<A>)> {
|
||||
//TODO: verify virt is aligned
|
||||
let mut table = self.table();
|
||||
|
||||
let unmap_parents = A::KERNEL_SEPARATE_TABLE || table.index_of(virt)? < A::PAGE_ENTRIES / 2; // Is a userspace mapping
|
||||
|
||||
unsafe {
|
||||
unmap_phys_inner(virt, &mut table, unmap_parents, &mut self.allocator)
|
||||
.map(|(pa, pf)| (pa, pf, PageFlush::new(virt)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn unmap_phys_inner<A: Arch>(
|
||||
virt: VirtualAddress,
|
||||
table: &mut PageTable<A>,
|
||||
unmap_parents: bool,
|
||||
allocator: &mut impl FrameAllocator,
|
||||
) -> Option<(PhysicalAddress, PageFlags<A>)> {
|
||||
unsafe {
|
||||
let i = table.index_of(virt)?;
|
||||
|
||||
if table.level() == 0 {
|
||||
let entry_opt = table.entry(i);
|
||||
table.set_entry(i, PageEntry::new(0, 0));
|
||||
let entry = entry_opt?;
|
||||
|
||||
return Some((entry.address().ok()?, entry.flags()));
|
||||
}
|
||||
|
||||
let mut subtable = table.next(i)?;
|
||||
|
||||
let res = unmap_phys_inner(virt, &mut subtable, unmap_parents, allocator)?;
|
||||
|
||||
if unmap_parents {
|
||||
// TODO: Use a counter? This would reduce the remaining number of available bits, but could be
|
||||
// faster (benchmark is needed).
|
||||
let is_still_populated = (0..A::PAGE_ENTRIES)
|
||||
.map(|j| subtable.entry(j).expect("must be within bounds"))
|
||||
.any(|e| e.present());
|
||||
|
||||
if !is_still_populated {
|
||||
allocator.free_one(subtable.phys());
|
||||
table.set_entry(i, PageEntry::new(0, 0));
|
||||
}
|
||||
}
|
||||
|
||||
Some(res)
|
||||
}
|
||||
}
|
||||
|
||||
impl<A, F: core::fmt::Debug> core::fmt::Debug for PageMapper<A, F> {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.debug_struct("PageMapper")
|
||||
.field("frame", &self.table_addr)
|
||||
.field("allocator", &self.allocator)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
pub use self::{entry::*, flags::*, flush::*, mapper::*, table::*};
|
||||
|
||||
mod entry;
|
||||
mod flags;
|
||||
mod flush;
|
||||
mod mapper;
|
||||
mod table;
|
||||
@@ -1,105 +0,0 @@
|
||||
use core::{fmt, marker::PhantomData};
|
||||
|
||||
use crate::{page::PageEntry, Arch, PhysicalAddress, VirtualAddress};
|
||||
|
||||
pub struct PageTable<A> {
|
||||
base: VirtualAddress,
|
||||
phys: PhysicalAddress,
|
||||
level: usize,
|
||||
phantom: PhantomData<A>,
|
||||
}
|
||||
|
||||
impl<A: Arch> PageTable<A> {
|
||||
pub(super) unsafe fn new(base: VirtualAddress, phys: PhysicalAddress, level: usize) -> Self {
|
||||
Self {
|
||||
base,
|
||||
phys,
|
||||
level,
|
||||
phantom: PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn base(&self) -> VirtualAddress {
|
||||
self.base
|
||||
}
|
||||
|
||||
pub fn phys(&self) -> PhysicalAddress {
|
||||
self.phys
|
||||
}
|
||||
|
||||
pub fn level(&self) -> usize {
|
||||
self.level
|
||||
}
|
||||
|
||||
pub fn entry_base(&self, i: usize) -> Option<VirtualAddress> {
|
||||
if i < A::PAGE_ENTRIES {
|
||||
let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT;
|
||||
Some(self.base.add(i << level_shift))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn entry_virt(&self, i: usize) -> Option<VirtualAddress> {
|
||||
if i < A::PAGE_ENTRIES {
|
||||
Some(A::phys_to_virt(self.phys).add(i * A::PAGE_ENTRY_SIZE))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn entry(&self, i: usize) -> Option<PageEntry<A>> {
|
||||
unsafe {
|
||||
let addr = self.entry_virt(i)?;
|
||||
Some(PageEntry::from_data(A::read::<usize>(addr)))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) unsafe fn set_entry(&mut self, i: usize, entry: PageEntry<A>) -> Option<()> {
|
||||
unsafe {
|
||||
let addr = self.entry_virt(i)?;
|
||||
A::write::<usize>(addr, entry.data());
|
||||
Some(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn index_of(&self, address: VirtualAddress) -> Option<usize> {
|
||||
// Canonicalize address first
|
||||
let address = VirtualAddress::new(address.data() & A::PAGE_ADDRESS_MASK);
|
||||
let level_shift = self.level * A::PAGE_ENTRY_SHIFT + A::PAGE_SHIFT;
|
||||
// Intentionally wraps around at last-level table to get all-ones mask on architectures
|
||||
// where addressable physical address space covers entire usized space (e.g. x86)
|
||||
let level_mask = A::PAGE_ENTRIES
|
||||
.wrapping_shl(level_shift as u32)
|
||||
.wrapping_sub(1);
|
||||
if address >= self.base && address <= self.base.add(level_mask) {
|
||||
Some((address.data() >> level_shift) & A::PAGE_ENTRY_MASK)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn next(&self, i: usize) -> Option<Self> {
|
||||
if self.level == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
unsafe {
|
||||
Some(PageTable::new(
|
||||
self.entry_base(i)?,
|
||||
self.entry(i)?.address().ok()?,
|
||||
self.level - 1,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn debug_entries(&self, f: impl Fn(fmt::Arguments<'_>)) {
|
||||
for i in 0..A::PAGE_ENTRIES {
|
||||
if let Some(entry) = unsafe { self.entry(i) }
|
||||
&& entry.present()
|
||||
{
|
||||
f(format_args!("{}: {:X}", i, entry.data()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +0,0 @@
|
||||
[toolchain]
|
||||
channel = "nightly-2025-10-03"
|
||||
components = ["rust-src"]
|
||||
@@ -1,22 +0,0 @@
|
||||
blank_lines_lower_bound = 0 # default
|
||||
blank_lines_upper_bound = 1 # default
|
||||
brace_style = "SameLineWhere" # default
|
||||
disable_all_formatting = false # default
|
||||
edition = "2024"
|
||||
style_edition = "2015"
|
||||
empty_item_single_line = true # default
|
||||
fn_single_line = false # default
|
||||
force_explicit_abi = true # default
|
||||
format_strings = false # default
|
||||
hard_tabs = false # default
|
||||
show_parse_errors = true # default
|
||||
imports_granularity = "Crate" # default = Preserve
|
||||
imports_indent = "Block" # default
|
||||
imports_layout = "Mixed" # default
|
||||
indent_style = "Block" # default
|
||||
max_width = 100 # default
|
||||
newline_style = "Unix" # default = Auto
|
||||
skip_children = false # default
|
||||
tab_spaces = 4 # default
|
||||
trailing_comma = "Vertical" # default
|
||||
where_single_line = false # default
|
||||
@@ -1,64 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
|
||||
use super::{find_sdt, sdt::Sdt};
|
||||
use crate::{
|
||||
arch::device::generic_timer::GenericTimer,
|
||||
dtb::irqchip::{register_irq, IRQ_CHIP},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct Gtdt {
|
||||
pub header: Sdt,
|
||||
pub cnt_control_base: u64,
|
||||
_reserved: u32,
|
||||
pub secure_el1_timer_gsiv: u32,
|
||||
pub secure_el1_timer_flags: u32,
|
||||
pub non_secure_el1_timer_gsiv: u32,
|
||||
pub non_secure_el1_timer_flags: u32,
|
||||
pub virtual_el1_timer_gsiv: u32,
|
||||
pub virtual_el1_timer_flags: u32,
|
||||
pub el2_timer_gsiv: u32,
|
||||
pub el2_timer_flags: u32,
|
||||
pub cnt_read_base: u64,
|
||||
pub platform_timer_count: u32,
|
||||
pub platform_timer_offset: u32,
|
||||
/*TODO: we don't need these yet, and they cause short tables to fail parsing
|
||||
pub virtual_el2_timer_gsiv: u32,
|
||||
pub virtual_el2_timer_flags: u32,
|
||||
*/
|
||||
//TODO: platform timer structure (at platform timer offset, with platform timer count)
|
||||
}
|
||||
|
||||
impl Gtdt {
|
||||
pub fn init() {
|
||||
let gtdt_sdt = find_sdt("GTDT");
|
||||
let gtdt = if gtdt_sdt.len() == 1 {
|
||||
match Gtdt::new(gtdt_sdt[0]) {
|
||||
Some(gtdt) => gtdt,
|
||||
None => {
|
||||
warn!("Failed to parse GTDT");
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Unable to find GTDT");
|
||||
return;
|
||||
};
|
||||
|
||||
let gsiv = gtdt.non_secure_el1_timer_gsiv;
|
||||
info!("generic_timer gsiv = {}", gsiv);
|
||||
let mut timer = GenericTimer::new();
|
||||
timer.init();
|
||||
register_irq(gsiv, Box::new(timer));
|
||||
unsafe { IRQ_CHIP.irq_enable(gsiv as u32) };
|
||||
}
|
||||
|
||||
pub fn new(sdt: &'static Sdt) -> Option<&'static Gtdt> {
|
||||
if &sdt.signature == b"GTDT" && sdt.length as usize >= size_of::<Gtdt>() {
|
||||
Some(unsafe { &*((sdt as *const Sdt) as *const Gtdt) })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
use core::ptr::{self, read_volatile, write_volatile};
|
||||
|
||||
#[cfg(not(target_arch = "x86"))]
|
||||
use crate::memory::{RmmA, RmmArch};
|
||||
use crate::{find_one_sdt, memory::PhysicalAddress};
|
||||
|
||||
use super::{sdt::Sdt, GenericAddressStructure, ACPI_TABLE};
|
||||
|
||||
#[repr(C, packed)]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Hpet {
|
||||
pub header: Sdt,
|
||||
|
||||
pub hw_rev_id: u8,
|
||||
pub comparator_descriptor: u8,
|
||||
pub pci_vendor_id: u16,
|
||||
|
||||
pub base_address: GenericAddressStructure,
|
||||
|
||||
pub hpet_number: u8,
|
||||
pub min_periodic_clk_tick: u16,
|
||||
pub oem_attribute: u8,
|
||||
}
|
||||
|
||||
impl Hpet {
|
||||
pub fn init() {
|
||||
let hpet = Hpet::new(find_one_sdt!("HPET"));
|
||||
|
||||
if let Some(hpet) = hpet {
|
||||
debug!(" HPET: {:X}", hpet.hpet_number);
|
||||
|
||||
let mut hpet_t = ACPI_TABLE.hpet.write();
|
||||
*hpet_t = Some(hpet);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(sdt: &'static Sdt) -> Option<Hpet> {
|
||||
if &sdt.signature == b"HPET" && sdt.length as usize >= size_of::<Hpet>() {
|
||||
let s = unsafe { ptr::read((sdt as *const Sdt) as *const Hpet) };
|
||||
if s.base_address.address_space == 0 {
|
||||
unsafe { s.map() };
|
||||
Some(s)
|
||||
} else {
|
||||
warn!(
|
||||
"HPET has unsupported address space {}",
|
||||
s.base_address.address_space
|
||||
);
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//TODO: x86 use assumes only one HPET and only one GenericAddressStructure
|
||||
#[cfg(target_arch = "x86")]
|
||||
impl Hpet {
|
||||
pub unsafe fn map(&self) {
|
||||
unsafe {
|
||||
use crate::memory::{Frame, KernelMapper, Page, PageFlags, VirtualAddress};
|
||||
|
||||
let frame = Frame::containing(PhysicalAddress::new(self.base_address.address as usize));
|
||||
let page = Page::containing_address(VirtualAddress::new(crate::HPET_OFFSET));
|
||||
|
||||
KernelMapper::lock_rw()
|
||||
.map_phys(
|
||||
page.start_address(),
|
||||
frame.base(),
|
||||
PageFlags::new().write(true).device_memory(true),
|
||||
)
|
||||
.expect("failed to map memory for GenericAddressStructure")
|
||||
.flush();
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn read_u64(&self, offset: usize) -> u64 {
|
||||
unsafe { read_volatile((crate::HPET_OFFSET + offset) as *const u64) }
|
||||
}
|
||||
|
||||
pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
|
||||
unsafe {
|
||||
write_volatile((crate::HPET_OFFSET + offset) as *mut u64, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "x86"))]
|
||||
impl Hpet {
|
||||
pub unsafe fn map(&self) {
|
||||
unsafe {
|
||||
crate::memory::map_device_memory(
|
||||
PhysicalAddress::new(self.base_address.address as usize),
|
||||
crate::memory::PAGE_SIZE,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn read_u64(&self, offset: usize) -> u64 {
|
||||
unsafe {
|
||||
read_volatile(
|
||||
RmmA::phys_to_virt(PhysicalAddress::new(
|
||||
self.base_address.address as usize + offset,
|
||||
))
|
||||
.data() as *const u64,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn write_u64(&mut self, offset: usize, value: u64) {
|
||||
unsafe {
|
||||
write_volatile(
|
||||
RmmA::phys_to_virt(PhysicalAddress::new(
|
||||
self.base_address.address as usize + offset,
|
||||
))
|
||||
.data() as *mut u64,
|
||||
value,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
use alloc::{boxed::Box, vec::Vec};
|
||||
|
||||
use super::{Madt, MadtEntry};
|
||||
use crate::{
|
||||
arch::device::irqchip::{
|
||||
gic::{GenericInterruptController, GicCpuIf, GicDistIf},
|
||||
gicv3::{GicV3, GicV3CpuIf},
|
||||
},
|
||||
dtb::irqchip::{IrqChipItem, IRQ_CHIP},
|
||||
memory::{map_device_memory, PhysicalAddress, PAGE_SIZE},
|
||||
};
|
||||
|
||||
pub(super) fn init(madt: Madt) {
|
||||
let mut gicd_opt = None;
|
||||
let mut giccs = Vec::new();
|
||||
for madt_entry in madt.iter() {
|
||||
debug!(" {:#x?}", madt_entry);
|
||||
match madt_entry {
|
||||
MadtEntry::Gicc(gicc) => {
|
||||
giccs.push(gicc);
|
||||
}
|
||||
MadtEntry::Gicd(gicd) => {
|
||||
if gicd_opt.is_some() {
|
||||
warn!("Only one GICD should be present on a system, ignoring this one");
|
||||
} else {
|
||||
gicd_opt = Some(gicd);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let Some(gicd) = gicd_opt else {
|
||||
warn!("No GICD found");
|
||||
return;
|
||||
};
|
||||
let mut gic_dist_if = GicDistIf::default();
|
||||
unsafe {
|
||||
let phys = PhysicalAddress::new(gicd.physical_base_address as usize);
|
||||
let virt = map_device_memory(phys, PAGE_SIZE);
|
||||
gic_dist_if.init(virt.data());
|
||||
};
|
||||
info!("{:#x?}", gic_dist_if);
|
||||
match gicd.gic_version {
|
||||
1 | 2 => {
|
||||
for gicc in giccs {
|
||||
let mut gic_cpu_if = GicCpuIf::default();
|
||||
unsafe {
|
||||
let phys = PhysicalAddress::new(gicc.physical_base_address as usize);
|
||||
let virt = map_device_memory(phys, PAGE_SIZE);
|
||||
gic_cpu_if.init(virt.data())
|
||||
};
|
||||
info!("{:#x?}", gic_cpu_if);
|
||||
let gic = GenericInterruptController {
|
||||
gic_dist_if,
|
||||
gic_cpu_if,
|
||||
irq_range: (0, 0),
|
||||
};
|
||||
let chip = IrqChipItem {
|
||||
phandle: 0,
|
||||
parents: Vec::new(),
|
||||
children: Vec::new(),
|
||||
ic: Box::new(gic),
|
||||
};
|
||||
unsafe { IRQ_CHIP.irq_chip_list.chips.push(chip) };
|
||||
//TODO: support more GICCs
|
||||
break;
|
||||
}
|
||||
}
|
||||
3 => {
|
||||
for gicc in giccs {
|
||||
let mut gic_cpu_if = GicV3CpuIf;
|
||||
unsafe { gic_cpu_if.init() };
|
||||
info!("{:#x?}", gic_cpu_if);
|
||||
let gic = GicV3 {
|
||||
gic_dist_if,
|
||||
gic_cpu_if,
|
||||
//TODO: get GICRs
|
||||
gicrs: Vec::new(),
|
||||
irq_range: (0, 0),
|
||||
};
|
||||
let chip = IrqChipItem {
|
||||
phandle: 0,
|
||||
parents: Vec::new(),
|
||||
children: Vec::new(),
|
||||
ic: Box::new(gic),
|
||||
};
|
||||
unsafe { IRQ_CHIP.irq_chip_list.chips.push(chip) };
|
||||
//TODO: support more GICCs
|
||||
break;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
warn!("unsupported GIC version {}", gicd.gic_version);
|
||||
}
|
||||
}
|
||||
unsafe { IRQ_CHIP.init(None) };
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
use super::Madt;
|
||||
|
||||
pub(super) fn init(madt: Madt) {
|
||||
for madt_entry in madt.iter() {
|
||||
debug!(" {:#x?}", madt_entry);
|
||||
}
|
||||
|
||||
warn!("MADT not yet handled on this platform");
|
||||
}
|
||||
@@ -1,760 +0,0 @@
|
||||
use core::{
|
||||
hint,
|
||||
sync::atomic::{AtomicU8, Ordering},
|
||||
};
|
||||
|
||||
use x86::time::rdtsc;
|
||||
|
||||
use crate::{
|
||||
arch::{
|
||||
device::local_apic::the_local_apic,
|
||||
start::{kstart_ap, KernelArgsAp},
|
||||
},
|
||||
cpu_set::LogicalCpuId,
|
||||
memory::{
|
||||
allocate_p2frame, map_device_memory, Frame, KernelMapper, Page, PageFlags,
|
||||
PhysicalAddress, RmmA, RmmArch, VirtualAddress, PAGE_SIZE,
|
||||
},
|
||||
startup::AP_READY,
|
||||
};
|
||||
|
||||
use super::{Madt, MadtEntry};
|
||||
|
||||
use alloc::collections::BTreeSet;
|
||||
use alloc::vec::Vec;
|
||||
|
||||
/// Maximum number of APIC→CPU mappings we track for NUMA topology.
|
||||
const MAX_APIC_MAPPINGS: usize = 256;
|
||||
|
||||
struct ApicMapping {
|
||||
apic_id: u32,
|
||||
cpu_id: LogicalCpuId,
|
||||
}
|
||||
|
||||
const UNINIT_MAPPING: ApicMapping = ApicMapping { apic_id: u32::MAX, cpu_id: LogicalCpuId::new(0) };
|
||||
|
||||
static mut APIC_MAPPINGS: [ApicMapping; MAX_APIC_MAPPINGS] = [UNINIT_MAPPING; MAX_APIC_MAPPINGS];
|
||||
static mut APIC_MAPPING_COUNT: usize = 0;
|
||||
|
||||
unsafe fn record_apic_mapping(apic_id: u32, cpu_id: LogicalCpuId) {
|
||||
let count = APIC_MAPPING_COUNT;
|
||||
if count < MAX_APIC_MAPPINGS {
|
||||
APIC_MAPPINGS[count] = ApicMapping { apic_id, cpu_id };
|
||||
APIC_MAPPING_COUNT = count + 1;
|
||||
}
|
||||
}
|
||||
|
||||
const AP_SPIN_LIMIT: u32 = 1_000_000;
|
||||
const TRAMPOLINE: usize = 0x8000;
|
||||
static TRAMPOLINE_DATA: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/trampoline"));
|
||||
|
||||
/// Estimate TSC frequency in MHz from CPUID.
|
||||
///
|
||||
/// Tries CPUID leaf 0x16 (Processor Frequency Information) first,
|
||||
/// then CPUID leaf 0x15 (TSC/Core Crystal Clock Ratio).
|
||||
/// Returns None if frequency cannot be determined.
|
||||
fn tsc_freq_mhz_cpuid() -> Option<u64> {
|
||||
let max_leaf = unsafe { core::arch::x86_64::__cpuid(0).eax as u32 };
|
||||
|
||||
// CPUID leaf 0x16: EAX = Core Base Frequency in MHz (Intel)
|
||||
if max_leaf >= 0x16 {
|
||||
let mhz = unsafe { core::arch::x86_64::__cpuid(0x16) }.eax as u64;
|
||||
if mhz > 0 {
|
||||
return Some(mhz);
|
||||
}
|
||||
}
|
||||
|
||||
// CPUID leaf 0x15: EAX = denominator, EBX = numerator, ECX = crystal Hz
|
||||
if max_leaf >= 0x15 {
|
||||
let res = unsafe { core::arch::x86_64::__cpuid(0x15) };
|
||||
let denom = res.eax as u64;
|
||||
let numer = res.ebx as u64;
|
||||
let crystal_hz = res.ecx as u64;
|
||||
if denom > 0 && numer > 0 && crystal_hz > 0 {
|
||||
// TSC freq = crystal_hz * numer / denom
|
||||
let tsc_hz = crystal_hz * numer / denom;
|
||||
return Some(tsc_hz / 1_000_000); // Hz → MHz
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Early-boot microsecond delay using the Time Stamp Counter.
|
||||
///
|
||||
/// Uses CPUID-based TSC frequency estimation when available.
|
||||
/// Falls back to a conservative spin loop calibrated for the
|
||||
/// minimum expected CPU speed (1 GHz).
|
||||
///
|
||||
/// # Safety
|
||||
/// Must only be called after the BSP TSC is running (always true
|
||||
/// after CPU reset on x86).
|
||||
fn early_udelay(us: u64) {
|
||||
if let Some(mhz) = tsc_freq_mhz_cpuid() {
|
||||
// TSC-based delay: precise on invariant TSC (all modern x86).
|
||||
// MHz = cycles per µs.
|
||||
let target = unsafe { rdtsc() } + us * mhz;
|
||||
while unsafe { rdtsc() } < target {
|
||||
hint::spin_loop();
|
||||
}
|
||||
} else {
|
||||
// Fallback: conservative spin loop.
|
||||
// spin_loop() (PAUSE) is ~40 cycles on modern Intel, ~1 on AMD.
|
||||
// At 1 GHz minimum: 1000 cycles/µs ÷ 40 cycles/iter = 25 iters/µs.
|
||||
// Use 50 iters/µs for safety margin on slower/variable CPUs.
|
||||
let iters = us.saturating_mul(50);
|
||||
for _ in 0..iters {
|
||||
hint::spin_loop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn current_x2apic_processor_uid(madt: &Madt, apic_id: u32) -> Option<u32> {
|
||||
madt.iter().find_map(|entry| match entry {
|
||||
MadtEntry::LocalX2Apic(x2apic) if x2apic.x2apic_id == apic_id => Some(x2apic.processor_uid),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_lapic_address_override(
|
||||
local_apic: &mut crate::arch::device::local_apic::LocalApic,
|
||||
address: u64,
|
||||
) {
|
||||
if local_apic.x2 || address == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok(physaddr) = usize::try_from(address) else {
|
||||
warn!(
|
||||
"Ignoring LAPIC address override {:#x}: does not fit host usize",
|
||||
address
|
||||
);
|
||||
return;
|
||||
};
|
||||
|
||||
let mapped = unsafe { map_device_memory(PhysicalAddress::new(physaddr), 4096) }.data();
|
||||
local_apic.address = mapped;
|
||||
debug!("Applied LAPIC address override: {:#x}", address);
|
||||
}
|
||||
|
||||
pub(super) fn init(madt: Madt) {
|
||||
let local_apic = unsafe { the_local_apic() };
|
||||
let me = local_apic.id();
|
||||
|
||||
if local_apic.x2 {
|
||||
debug!(" X2APIC {}", me.get());
|
||||
} else {
|
||||
debug!(" XAPIC {}: {:>08X}", me.get(), local_apic.address);
|
||||
}
|
||||
|
||||
if cfg!(not(feature = "multi_core")) {
|
||||
unsafe {
|
||||
record_apic_mapping(me.get(), LogicalCpuId::new(0));
|
||||
}
|
||||
crate::numa::init_default();
|
||||
return;
|
||||
}
|
||||
|
||||
// Map trampoline
|
||||
let trampoline_frame = Frame::containing(PhysicalAddress::new(TRAMPOLINE));
|
||||
let trampoline_page = Page::containing_address(VirtualAddress::new(TRAMPOLINE));
|
||||
let (result, page_table_physaddr) = unsafe {
|
||||
//TODO: do not have writable and executable!
|
||||
let mut mapper = KernelMapper::lock_rw();
|
||||
|
||||
let result = match mapper.map_phys(
|
||||
trampoline_page.start_address(),
|
||||
trampoline_frame.base(),
|
||||
PageFlags::new().execute(true).write(true),
|
||||
) {
|
||||
Some(result) => result,
|
||||
None => {
|
||||
println!("KERNEL AP: failed to map trampoline page, AP bring-up disabled");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
(result, mapper.table().phys().data())
|
||||
};
|
||||
result.flush();
|
||||
|
||||
// Write trampoline, make sure TRAMPOLINE page is free for use
|
||||
for (i, val) in TRAMPOLINE_DATA.iter().enumerate() {
|
||||
unsafe {
|
||||
(*((TRAMPOLINE as *mut u8).add(i) as *const AtomicU8)).store(*val, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
// Detect whether MADT contains any LocalX2Apic entries.
|
||||
// Some firmware (notably QEMU and some older BIOS) provides only 8-bit
|
||||
// LocalApic entries even when the CPU supports x2APIC. In that case we must
|
||||
// fall back to processing LocalApic entries with zero-extended IDs.
|
||||
let has_x2apic_entries = madt.iter().any(|e| matches!(e, MadtEntry::LocalX2Apic(_)));
|
||||
let x2apic_fallback = local_apic.x2 && !has_x2apic_entries;
|
||||
if x2apic_fallback {
|
||||
warn!("MADT: x2APIC mode active but no LocalX2Apic entries found; falling back to LocalApic entries with zero-extended IDs");
|
||||
}
|
||||
|
||||
unsafe {
|
||||
let preliminary_cpu_count = madt
|
||||
.iter()
|
||||
.filter(|entry| match entry {
|
||||
// When x2APIC is active, LocalApic entries use 8-bit IDs that don't
|
||||
// match the BSP's 32-bit x2APIC ID. Use LocalX2Apic entries instead.
|
||||
MadtEntry::LocalApic(local) if !local_apic.x2 => {
|
||||
u32::from(local.id) == me.get() || local.flags & 1 == 1
|
||||
}
|
||||
MadtEntry::LocalApic(local) if local_apic.x2 && x2apic_fallback => {
|
||||
u32::from(local.id) == me.get() || local.flags & 1 == 1
|
||||
}
|
||||
MadtEntry::LocalApic(_) => false,
|
||||
// xAPIC mode: cannot use 32-bit x2APIC IDs via 8-bit ICR.
|
||||
// Skip LocalX2Apic entries and use LocalApic exclusively.
|
||||
MadtEntry::LocalX2Apic(local) if local_apic.x2 => {
|
||||
local.x2apic_id == me.get() || local.flags & 1 == 1
|
||||
}
|
||||
MadtEntry::LocalX2Apic(_) => false,
|
||||
_ => false,
|
||||
})
|
||||
.count();
|
||||
crate::profiling::allocate(preliminary_cpu_count as u32);
|
||||
}
|
||||
|
||||
// Firmware bug detection: check for duplicate APIC IDs in MADT.
|
||||
// Some firmware (especially on early BIOS/UEFI) may list the same
|
||||
// processor multiple times. Keep first occurrence, warn on duplicates.
|
||||
let mut seen_apic_ids: BTreeSet<u32> = BTreeSet::new();
|
||||
{
|
||||
let _ = seen_apic_ids.insert(me.get()); // BSP
|
||||
for entry in madt.iter() {
|
||||
match entry {
|
||||
MadtEntry::LocalApic(local) if local.flags & 1 == 1 && !local_apic.x2 => {
|
||||
let id = u32::from(local.id);
|
||||
if !seen_apic_ids.insert(id) {
|
||||
warn!("MADT: duplicate APIC ID {} in LocalApic entry, firmware bug", id);
|
||||
}
|
||||
}
|
||||
MadtEntry::LocalApic(local) if local.flags & 1 == 1 && local_apic.x2 => {
|
||||
if x2apic_fallback {
|
||||
let id = u32::from(local.id);
|
||||
if !seen_apic_ids.insert(id) {
|
||||
warn!("MADT: duplicate APIC ID {} in LocalApic entry (x2APIC fallback), firmware bug", id);
|
||||
}
|
||||
} else {
|
||||
debug!("MADT: ignoring 8-bit LocalApic ID {} in x2APIC mode", local.id);
|
||||
}
|
||||
}
|
||||
MadtEntry::LocalX2Apic(local) if local.flags & 1 == 1 && local_apic.x2 => {
|
||||
let id = local.x2apic_id;
|
||||
if !seen_apic_ids.insert(id) {
|
||||
warn!("MADT: duplicate x2APIC ID {} in LocalX2Apic entry, firmware bug", id);
|
||||
}
|
||||
}
|
||||
MadtEntry::LocalX2Apic(local) if local.flags & 1 == 1 && !local_apic.x2 => {
|
||||
// xAPIC mode: skip 32-bit x2APIC IDs; dedup only among LocalApic entries.
|
||||
let id = local.x2apic_id; // Copy from packed struct
|
||||
debug!("MADT: ignoring 32-bit x2APIC ID {} in xAPIC mode", id);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for madt_entry in madt.iter() {
|
||||
debug!(" {:x?}", madt_entry);
|
||||
if let MadtEntry::LocalApic(ap_local_apic) = madt_entry {
|
||||
// x2APIC mode: LocalApic entries have 8-bit IDs that don't match
|
||||
// the BSP's 32-bit x2APIC ID. All entries would be treated as APs,
|
||||
// and SIPI would target the wrong processors. Skip them and rely
|
||||
// on LocalX2Apic entries exclusively.
|
||||
if local_apic.x2 && !x2apic_fallback {
|
||||
debug!(
|
||||
" Skipping 8-bit LocalApic id={} (x2APIC active, using LocalX2Apic entries)",
|
||||
ap_local_apic.id
|
||||
);
|
||||
} else if local_apic.x2 && x2apic_fallback {
|
||||
let apic_id = u32::from(ap_local_apic.id);
|
||||
if apic_id == me.get() {
|
||||
debug!(" This is my local APIC (x2APIC fallback, id={})", apic_id);
|
||||
} else if ap_local_apic.flags & 1 == 1 {
|
||||
let alloc = match allocate_p2frame(4) {
|
||||
Some(frame) => frame,
|
||||
None => {
|
||||
println!("KERNEL AP: CPU {} no memory for stack, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let stack_start = RmmA::phys_to_virt(alloc.base()).data();
|
||||
let stack_end = stack_start + (PAGE_SIZE << 4);
|
||||
|
||||
let cpu_id = LogicalCpuId::new(crate::CPU_COUNT.fetch_add(1, Ordering::SeqCst));
|
||||
if cpu_id.get() >= crate::cpu_set::MAX_CPU_COUNT {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} exceeds logical CPU limit, skipping",
|
||||
apic_id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
|
||||
let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id);
|
||||
|
||||
let args = KernelArgsAp {
|
||||
stack_end: stack_end as *mut u8,
|
||||
cpu_id,
|
||||
pcr_ptr,
|
||||
idt_ptr,
|
||||
};
|
||||
|
||||
let ap_ready = (TRAMPOLINE + 8) as *mut u64;
|
||||
let ap_args_ptr = unsafe { ap_ready.add(1) };
|
||||
let ap_page_table = unsafe { ap_ready.add(2) };
|
||||
let ap_code = unsafe { ap_ready.add(3) };
|
||||
|
||||
unsafe {
|
||||
ap_ready.write(0);
|
||||
ap_args_ptr.write(&args as *const _ as u64);
|
||||
ap_page_table.write(page_table_physaddr as u64);
|
||||
#[expect(clippy::fn_to_numeric_cast)]
|
||||
ap_code.write(kstart_ap as u64);
|
||||
|
||||
core::sync::atomic::fence(Ordering::SeqCst);
|
||||
};
|
||||
AP_READY.store(false, Ordering::SeqCst);
|
||||
|
||||
// Clear APIC Error Status Register before starting AP.
|
||||
unsafe { local_apic.esr(); }
|
||||
|
||||
// Send INIT IPI (Assert) — x2APIC uses 64-bit ICR format.
|
||||
{
|
||||
let mut icr = 0x4500u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Intel SDM Vol 3A §8.4.4: wait 10ms after INIT deassert
|
||||
early_udelay(10_000);
|
||||
|
||||
// Send START IPI #1
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
let mut icr = 0x0600 | ap_segment as u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
early_udelay(200);
|
||||
|
||||
// Send START IPI #2 (recommended for compatibility)
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
let mut icr = 0x0600 | ap_segment as u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
early_udelay(200);
|
||||
|
||||
// Check ESR for delivery errors after SIPI sequence.
|
||||
let esr_val = unsafe { local_apic.esr() };
|
||||
if esr_val != 0 {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} SIPI delivery error (ESR={:#x}), continuing",
|
||||
apic_id, esr_val
|
||||
);
|
||||
}
|
||||
|
||||
let mut trampoline_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } != 0 {
|
||||
trampoline_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !trampoline_ready {
|
||||
println!("KERNEL AP: CPU {} trampoline timeout, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut kernel_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if AP_READY.load(Ordering::SeqCst) {
|
||||
kernel_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !kernel_ready {
|
||||
println!("KERNEL AP: CPU {} AP_READY timeout, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Record APIC→CPU mapping for NUMA topology.
|
||||
unsafe {
|
||||
record_apic_mapping(apic_id, cpu_id);
|
||||
}
|
||||
// Set NUMA node from SRAT data.
|
||||
if let Some(percpu) = crate::percpu::get_for_cpu(cpu_id) {
|
||||
if let Some(node) = crate::acpi::srat::numa_node_for_apic(apic_id) {
|
||||
percpu.numa_node.set(node);
|
||||
}
|
||||
}
|
||||
|
||||
RmmA::invalidate_all();
|
||||
}
|
||||
} else if u32::from(ap_local_apic.id) == me.get() {
|
||||
debug!(" This is my local APIC");
|
||||
} else if ap_local_apic.flags & 1 == 1 {
|
||||
// Allocate a stack
|
||||
let alloc = match allocate_p2frame(4) {
|
||||
Some(frame) => frame,
|
||||
None => {
|
||||
println!("KERNEL AP: CPU {} no memory for stack, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let stack_start = RmmA::phys_to_virt(alloc.base()).data();
|
||||
let stack_end = stack_start + (PAGE_SIZE << 4);
|
||||
|
||||
// Atomically allocate a CPU ID — fetch_add is SeqCst so that
|
||||
// all later stores (PercpuBlock, NUMA node) are ordered after.
|
||||
let cpu_id = LogicalCpuId::new(crate::CPU_COUNT.fetch_add(1, Ordering::SeqCst));
|
||||
if cpu_id.get() >= crate::cpu_set::MAX_CPU_COUNT {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} exceeds logical CPU limit, skipping",
|
||||
ap_local_apic.id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
|
||||
|
||||
let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id);
|
||||
|
||||
let args = KernelArgsAp {
|
||||
stack_end: stack_end as *mut u8,
|
||||
cpu_id,
|
||||
pcr_ptr,
|
||||
idt_ptr,
|
||||
};
|
||||
|
||||
let ap_ready = (TRAMPOLINE + 8) as *mut u64;
|
||||
let ap_args_ptr = unsafe { ap_ready.add(1) };
|
||||
let ap_page_table = unsafe { ap_ready.add(2) };
|
||||
let ap_code = unsafe { ap_ready.add(3) };
|
||||
|
||||
// Set the ap_ready to 0, volatile
|
||||
unsafe {
|
||||
ap_ready.write(0);
|
||||
ap_args_ptr.write(&args as *const _ as u64);
|
||||
ap_page_table.write(page_table_physaddr as u64);
|
||||
#[expect(clippy::fn_to_numeric_cast)]
|
||||
ap_code.write(kstart_ap as u64);
|
||||
|
||||
// Ensure all trampoline writes are visible to the AP before
|
||||
// it starts executing. asm!("") is only a compiler barrier;
|
||||
// fence(SeqCst) is a full hardware memory barrier.
|
||||
core::sync::atomic::fence(Ordering::SeqCst);
|
||||
};
|
||||
AP_READY.store(false, Ordering::SeqCst);
|
||||
|
||||
// Clear APIC Error Status Register before starting AP.
|
||||
// Intel SDM §8.4.4: ESR should be cleared before sending SIPI.
|
||||
unsafe { local_apic.esr(); }
|
||||
|
||||
// Send INIT IPI (Assert)
|
||||
{
|
||||
// ICR: Delivery Mode=INIT(101), Level=Assert, Trigger=Edge
|
||||
let mut icr = 0x4500u64;
|
||||
if local_apic.x2 {
|
||||
icr |= u64::from(ap_local_apic.id) << 32;
|
||||
} else {
|
||||
icr |= u64::from(ap_local_apic.id) << 56;
|
||||
}
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Intel SDM Vol 3A §8.4.4: wait 10ms after INIT deassert
|
||||
// before sending first SIPI. Modern CPUs may need less,
|
||||
// but 10ms is the safe specification-compliant value.
|
||||
early_udelay(10_000);
|
||||
|
||||
// Send START IPI #1
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
// ICR: Delivery Mode=StartUp(110), Vector=ap_segment
|
||||
// Note: bit 14 (Level) must be 0 for SIPI per Intel SDM.
|
||||
let mut icr = 0x0600 | ap_segment as u64;
|
||||
if local_apic.x2 {
|
||||
icr |= u64::from(ap_local_apic.id) << 32;
|
||||
} else {
|
||||
icr |= u64::from(ap_local_apic.id) << 56;
|
||||
}
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Intel SDM: wait 200µs between SIPIs
|
||||
early_udelay(200);
|
||||
|
||||
// Send START IPI #2 (recommended for compatibility)
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
let mut icr = 0x0600 | ap_segment as u64;
|
||||
if local_apic.x2 {
|
||||
icr |= u64::from(ap_local_apic.id) << 32;
|
||||
} else {
|
||||
icr |= u64::from(ap_local_apic.id) << 56;
|
||||
}
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Wait briefly for SIPI to be accepted
|
||||
early_udelay(200);
|
||||
|
||||
// Check ESR for delivery errors after SIPI sequence.
|
||||
// Bit 5 = Send Accept Error, Bit 6 = Send Illegal Vector.
|
||||
let esr_val = unsafe { local_apic.esr() };
|
||||
if esr_val != 0 {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} SIPI delivery error (ESR={:#x}), continuing",
|
||||
ap_local_apic.id, esr_val
|
||||
);
|
||||
}
|
||||
|
||||
// Wait for trampoline ready with timeout
|
||||
let mut trampoline_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } != 0 {
|
||||
trampoline_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !trampoline_ready {
|
||||
println!("KERNEL AP: CPU {} trampoline timeout, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut kernel_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if AP_READY.load(Ordering::SeqCst) {
|
||||
kernel_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !kernel_ready {
|
||||
println!("KERNEL AP: CPU {} AP_READY timeout, skipping", ap_local_apic.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Record APIC→CPU mapping for NUMA topology.
|
||||
unsafe {
|
||||
record_apic_mapping(u32::from(ap_local_apic.id), cpu_id);
|
||||
}
|
||||
// Set NUMA node from SRAT data.
|
||||
if let Some(percpu) = crate::percpu::get_for_cpu(cpu_id) {
|
||||
if let Some(node) = crate::acpi::srat::numa_node_for_apic(u32::from(ap_local_apic.id)) {
|
||||
percpu.numa_node.set(node);
|
||||
}
|
||||
}
|
||||
|
||||
RmmA::invalidate_all();
|
||||
}
|
||||
} else if let MadtEntry::LocalX2Apic(ap_x2apic) = madt_entry {
|
||||
let apic_id = ap_x2apic.x2apic_id;
|
||||
let flags = ap_x2apic.flags;
|
||||
|
||||
// xAPIC mode: cannot target 32-bit x2APIC IDs via 8-bit ICR.
|
||||
// Skip LocalX2Apic entries; use LocalApic entries exclusively.
|
||||
if !local_apic.x2 {
|
||||
debug!(
|
||||
" Skipping 32-bit x2APIC id={} (xAPIC mode, using LocalApic entries)",
|
||||
apic_id
|
||||
);
|
||||
} else if apic_id == me.get() {
|
||||
debug!(" This is my local x2APIC");
|
||||
} else if flags & 1 == 1 {
|
||||
let alloc = match allocate_p2frame(4) {
|
||||
Some(frame) => frame,
|
||||
None => {
|
||||
println!("KERNEL AP: CPU {} no memory for stack, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let stack_start = RmmA::phys_to_virt(alloc.base()).data();
|
||||
let stack_end = stack_start + (PAGE_SIZE << 4);
|
||||
|
||||
// Atomically allocate a CPU ID — fetch_add is SeqCst so that
|
||||
// all later stores (PercpuBlock, NUMA node) are ordered after.
|
||||
let cpu_id = LogicalCpuId::new(crate::CPU_COUNT.fetch_add(1, Ordering::SeqCst));
|
||||
if cpu_id.get() >= crate::cpu_set::MAX_CPU_COUNT {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} exceeds logical CPU limit, skipping",
|
||||
apic_id
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
let pcr_ptr = crate::arch::gdt::allocate_and_init_pcr(cpu_id, stack_end);
|
||||
let idt_ptr = crate::arch::idt::allocate_and_init_idt(cpu_id);
|
||||
|
||||
let args = KernelArgsAp {
|
||||
stack_end: stack_end as *mut u8,
|
||||
cpu_id,
|
||||
pcr_ptr,
|
||||
idt_ptr,
|
||||
};
|
||||
|
||||
let ap_ready = (TRAMPOLINE + 8) as *mut u64;
|
||||
let ap_args_ptr = unsafe { ap_ready.add(1) };
|
||||
let ap_page_table = unsafe { ap_ready.add(2) };
|
||||
let ap_code = unsafe { ap_ready.add(3) };
|
||||
|
||||
unsafe {
|
||||
ap_ready.write(0);
|
||||
ap_args_ptr.write(&args as *const _ as u64);
|
||||
ap_page_table.write(page_table_physaddr as u64);
|
||||
#[expect(clippy::fn_to_numeric_cast)]
|
||||
ap_code.write(kstart_ap as u64);
|
||||
// Ensure all trampoline writes are visible to the AP.
|
||||
core::sync::atomic::fence(Ordering::SeqCst);
|
||||
}
|
||||
AP_READY.store(false, Ordering::SeqCst);
|
||||
|
||||
// Clear APIC Error Status Register before starting AP.
|
||||
unsafe { local_apic.esr(); }
|
||||
|
||||
// Send INIT IPI (Assert)
|
||||
{
|
||||
let mut icr = 0x4500u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Intel SDM Vol 3A §8.4.4: wait 10ms after INIT
|
||||
early_udelay(10_000);
|
||||
|
||||
// Send START IPI #1
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
let mut icr = 0x0600u64 | ap_segment as u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Intel SDM: wait 200µs between SIPIs
|
||||
early_udelay(200);
|
||||
|
||||
// Send START IPI #2 (recommended for compatibility)
|
||||
{
|
||||
let ap_segment = (TRAMPOLINE >> 12) & 0xFF;
|
||||
let mut icr = 0x0600u64 | ap_segment as u64;
|
||||
icr |= u64::from(apic_id) << 32;
|
||||
local_apic.set_icr(icr);
|
||||
}
|
||||
|
||||
// Wait briefly for SIPI acceptance
|
||||
early_udelay(200);
|
||||
|
||||
// Check ESR for delivery errors.
|
||||
let esr_val = unsafe { local_apic.esr() };
|
||||
if esr_val != 0 {
|
||||
println!(
|
||||
"KERNEL AP: CPU {} SIPI delivery error (ESR={:#x}), continuing",
|
||||
apic_id, esr_val
|
||||
);
|
||||
}
|
||||
|
||||
let mut trampoline_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if unsafe { (*ap_ready.cast::<AtomicU8>()).load(Ordering::SeqCst) } != 0 {
|
||||
trampoline_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !trampoline_ready {
|
||||
println!("KERNEL AP: CPU {} trampoline timeout, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut kernel_ready = false;
|
||||
for _ in 0..AP_SPIN_LIMIT {
|
||||
if AP_READY.load(Ordering::SeqCst) {
|
||||
kernel_ready = true;
|
||||
break;
|
||||
}
|
||||
hint::spin_loop();
|
||||
}
|
||||
if !kernel_ready {
|
||||
println!("KERNEL AP: CPU {} AP_READY timeout, skipping", apic_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Record APIC→CPU mapping for NUMA topology.
|
||||
unsafe {
|
||||
record_apic_mapping(apic_id, cpu_id);
|
||||
}
|
||||
// Set NUMA node from SRAT data.
|
||||
if let Some(percpu) = crate::percpu::get_for_cpu(cpu_id) {
|
||||
if let Some(node) = crate::acpi::srat::numa_node_for_apic(apic_id) {
|
||||
percpu.numa_node.set(node);
|
||||
}
|
||||
}
|
||||
|
||||
RmmA::invalidate_all();
|
||||
}
|
||||
} else if let MadtEntry::LocalApicNmi(nmi) = madt_entry {
|
||||
let target_apic = nmi.processor;
|
||||
if target_apic == 0xFF || target_apic == local_apic.id().get() as u8 {
|
||||
unsafe { local_apic.set_lvt_nmi(nmi.nmi_pin, nmi.flags) };
|
||||
}
|
||||
} else if let MadtEntry::LocalX2ApicNmi(nmi) = madt_entry {
|
||||
let current_uid = current_x2apic_processor_uid(&madt, me.get());
|
||||
if nmi.processor_uid == u32::MAX || current_uid == Some(nmi.processor_uid) {
|
||||
unsafe { local_apic.set_lvt_nmi(nmi.nmi_pin, nmi.flags) };
|
||||
}
|
||||
} else if let MadtEntry::LapicAddressOverride(override_entry) = madt_entry {
|
||||
apply_lapic_address_override(local_apic, override_entry.local_apic_address);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize NUMA topology from APIC→CPU mappings and SRAT.
|
||||
{
|
||||
let mappings = unsafe { &APIC_MAPPINGS[..APIC_MAPPING_COUNT] };
|
||||
let mappings_ref: Vec<(u32, LogicalCpuId)> = mappings
|
||||
.iter()
|
||||
.map(|m| (m.apic_id, m.cpu_id))
|
||||
.collect();
|
||||
crate::numa::init_from_srat(&mappings_ref);
|
||||
}
|
||||
// Set BSP's NUMA node from SRAT.
|
||||
if let Some(node) = crate::acpi::srat::numa_node_for_apic(me.get()) {
|
||||
crate::percpu::PercpuBlock::current().numa_node.set(node);
|
||||
}
|
||||
|
||||
// Log final CPU count vs maximum
|
||||
let cpu_count = crate::CPU_COUNT.load(Ordering::SeqCst);
|
||||
info!(
|
||||
"SMP: {} CPUs online (max {})",
|
||||
cpu_count, crate::cpu_set::MAX_CPU_COUNT
|
||||
);
|
||||
if cpu_count > crate::cpu_set::MAX_CPU_COUNT * 80 / 100 {
|
||||
warn!(
|
||||
"SMP: CPU count approaching MAX_CPU_COUNT limit ({}/{})",
|
||||
cpu_count, crate::cpu_set::MAX_CPU_COUNT
|
||||
);
|
||||
}
|
||||
|
||||
// Unmap trampoline
|
||||
if let Some((_frame, _, flush)) = unsafe {
|
||||
KernelMapper::lock_rw()
|
||||
.unmap_phys(trampoline_page.start_address())
|
||||
} {
|
||||
flush.flush();
|
||||
} else {
|
||||
println!("KERNEL AP: failed to unmap trampoline page (non-fatal)");
|
||||
}
|
||||
}
|
||||
@@ -1,340 +0,0 @@
|
||||
use core::cell::SyncUnsafeCell;
|
||||
|
||||
use super::sdt::Sdt;
|
||||
use crate::find_one_sdt;
|
||||
|
||||
/// The Multiple APIC Descriptor Table
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Madt {
|
||||
sdt: &'static Sdt,
|
||||
pub local_address: u32,
|
||||
pub flags: u32,
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
#[path = "arch/aarch64.rs"]
|
||||
mod arch;
|
||||
|
||||
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
||||
#[path = "arch/x86.rs"]
|
||||
mod arch;
|
||||
|
||||
#[cfg(not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")))]
|
||||
#[path = "arch/other.rs"]
|
||||
mod arch;
|
||||
|
||||
static MADT: SyncUnsafeCell<Option<Madt>> = SyncUnsafeCell::new(None);
|
||||
pub fn madt() -> Option<&'static Madt> {
|
||||
unsafe { &*MADT.get() }.as_ref()
|
||||
}
|
||||
pub const FLAG_PCAT: u32 = 1;
|
||||
|
||||
impl Madt {
|
||||
pub fn init() {
|
||||
let madt = Madt::new(find_one_sdt!("APIC"));
|
||||
|
||||
if let Some(madt) = madt {
|
||||
// Validate MADT checksum per ACPI 6.5 §5.2.2
|
||||
if !madt.sdt.validate_checksum() {
|
||||
error!("MADT checksum validation failed, skipping APIC initialization");
|
||||
return;
|
||||
}
|
||||
|
||||
// safe because no APs have been started yet.
|
||||
unsafe { MADT.get().write(Some(madt)) };
|
||||
|
||||
debug!(" APIC: {:>08X}: {}", madt.local_address, madt.flags);
|
||||
|
||||
arch::init(madt);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(sdt: &'static Sdt) -> Option<Madt> {
|
||||
if &sdt.signature == b"APIC" && sdt.data_len() >= 8 {
|
||||
//Not valid if no local address and flags
|
||||
let local_address = unsafe { (sdt.data_address() as *const u32).read_unaligned() };
|
||||
let flags = unsafe {
|
||||
(sdt.data_address() as *const u32)
|
||||
.offset(1)
|
||||
.read_unaligned()
|
||||
};
|
||||
|
||||
Some(Madt {
|
||||
sdt,
|
||||
local_address,
|
||||
flags,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> MadtIter {
|
||||
MadtIter {
|
||||
sdt: self.sdt,
|
||||
i: 8, // Skip local controller address and flags
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// MADT Local APIC
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtLocalApic {
|
||||
/// Processor ID
|
||||
pub processor: u8,
|
||||
/// Local APIC ID
|
||||
pub id: u8,
|
||||
/// Flags. 1 means that the processor is enabled
|
||||
pub flags: u32,
|
||||
}
|
||||
|
||||
/// MADT I/O APIC
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtIoApic {
|
||||
/// I/O APIC ID
|
||||
pub id: u8,
|
||||
/// reserved
|
||||
_reserved: u8,
|
||||
/// I/O APIC address
|
||||
pub address: u32,
|
||||
/// Global system interrupt base
|
||||
pub gsi_base: u32,
|
||||
}
|
||||
|
||||
/// MADT Interrupt Source Override
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtIntSrcOverride {
|
||||
/// Bus Source
|
||||
pub bus_source: u8,
|
||||
/// IRQ Source
|
||||
pub irq_source: u8,
|
||||
/// Global system interrupt base
|
||||
pub gsi_base: u32,
|
||||
/// Flags
|
||||
pub flags: u16,
|
||||
}
|
||||
|
||||
/// MADT GICC
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtGicc {
|
||||
_reserved: u16,
|
||||
pub cpu_interface_number: u32,
|
||||
pub acpi_processor_uid: u32,
|
||||
pub flags: u32,
|
||||
pub parking_protocol_version: u32,
|
||||
pub performance_interrupt_gsiv: u32,
|
||||
pub parked_address: u64,
|
||||
pub physical_base_address: u64,
|
||||
pub gicv: u64,
|
||||
pub gich: u64,
|
||||
pub vgic_maintenance_interrupt: u32,
|
||||
pub gicr_base_address: u64,
|
||||
pub mpidr: u64,
|
||||
pub processor_power_efficiency_class: u8,
|
||||
_reserved2: u8,
|
||||
pub spe_overflow_interrupt: u16,
|
||||
//TODO: optional field introduced in ACPI 6.5: pub trbe_interrupt: u16,
|
||||
}
|
||||
|
||||
/// MADT GICD
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtGicd {
|
||||
_reserved: u16,
|
||||
pub gic_id: u32,
|
||||
pub physical_base_address: u64,
|
||||
pub system_vector_base: u32,
|
||||
pub gic_version: u8,
|
||||
_reserved2: [u8; 3],
|
||||
}
|
||||
|
||||
/// MADT Local x2APIC (entry type 0x9)
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtLocalX2Apic {
|
||||
_reserved: u16,
|
||||
pub x2apic_id: u32,
|
||||
pub flags: u32,
|
||||
pub processor_uid: u32,
|
||||
}
|
||||
|
||||
/// MADT Local APIC NMI (entry type 0x4)
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtLocalApicNmi {
|
||||
pub processor: u8,
|
||||
pub flags: u16,
|
||||
pub nmi_pin: u8,
|
||||
}
|
||||
|
||||
/// MADT Local APIC address override (entry type 0x5)
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtLapicAddressOverride {
|
||||
_reserved: u16,
|
||||
pub local_apic_address: u64,
|
||||
}
|
||||
|
||||
/// MADT Local x2APIC NMI (entry type 0xA)
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct MadtLocalX2ApicNmi {
|
||||
_reserved: u16,
|
||||
pub processor_uid: u32,
|
||||
pub flags: u16,
|
||||
pub nmi_pin: u8,
|
||||
_reserved2: u8,
|
||||
}
|
||||
|
||||
const _: () = assert!(size_of::<MadtLocalApicNmi>() == 4);
|
||||
const _: () = assert!(size_of::<MadtLapicAddressOverride>() == 10);
|
||||
const _: () = assert!(size_of::<MadtLocalX2ApicNmi>() == 10);
|
||||
|
||||
/// MADT Entries
|
||||
#[derive(Debug)]
|
||||
#[allow(dead_code)]
|
||||
pub enum MadtEntry {
|
||||
LocalApic(&'static MadtLocalApic),
|
||||
InvalidLocalApic(usize),
|
||||
IoApic(&'static MadtIoApic),
|
||||
InvalidIoApic(usize),
|
||||
IntSrcOverride(&'static MadtIntSrcOverride),
|
||||
InvalidIntSrcOverride(usize),
|
||||
LocalApicNmi(&'static MadtLocalApicNmi),
|
||||
InvalidLocalApicNmi(usize),
|
||||
LapicAddressOverride(&'static MadtLapicAddressOverride),
|
||||
InvalidLapicAddressOverride(usize),
|
||||
Gicc(&'static MadtGicc),
|
||||
InvalidGicc(usize),
|
||||
Gicd(&'static MadtGicd),
|
||||
InvalidGicd(usize),
|
||||
LocalX2Apic(&'static MadtLocalX2Apic),
|
||||
InvalidLocalX2Apic(usize),
|
||||
LocalX2ApicNmi(&'static MadtLocalX2ApicNmi),
|
||||
InvalidLocalX2ApicNmi(usize),
|
||||
Unknown(u8),
|
||||
}
|
||||
|
||||
pub struct MadtIter {
|
||||
sdt: &'static Sdt,
|
||||
i: usize,
|
||||
}
|
||||
|
||||
impl Iterator for MadtIter {
|
||||
type Item = MadtEntry;
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.i + 1 < self.sdt.data_len() {
|
||||
let entry_type = unsafe { *(self.sdt.data_address() as *const u8).add(self.i) };
|
||||
let entry_len =
|
||||
unsafe { *(self.sdt.data_address() as *const u8).add(self.i + 1) } as usize;
|
||||
|
||||
if entry_len < 2 {
|
||||
return None;
|
||||
}
|
||||
|
||||
if self.i + entry_len <= self.sdt.data_len() {
|
||||
let item = match entry_type {
|
||||
0x0 => {
|
||||
if entry_len == size_of::<MadtLocalApic>() + 2 {
|
||||
MadtEntry::LocalApic(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2) as *const MadtLocalApic)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidLocalApic(entry_len)
|
||||
}
|
||||
}
|
||||
0x1 => {
|
||||
if entry_len == size_of::<MadtIoApic>() + 2 {
|
||||
MadtEntry::IoApic(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2) as *const MadtIoApic)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidIoApic(entry_len)
|
||||
}
|
||||
}
|
||||
0x2 => {
|
||||
if entry_len == size_of::<MadtIntSrcOverride>() + 2 {
|
||||
MadtEntry::IntSrcOverride(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2)
|
||||
as *const MadtIntSrcOverride)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidIntSrcOverride(entry_len)
|
||||
}
|
||||
}
|
||||
0x4 => {
|
||||
if entry_len == size_of::<MadtLocalApicNmi>() + 2 {
|
||||
MadtEntry::LocalApicNmi(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2)
|
||||
as *const MadtLocalApicNmi)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidLocalApicNmi(entry_len)
|
||||
}
|
||||
}
|
||||
0x5 => {
|
||||
if entry_len == size_of::<MadtLapicAddressOverride>() + 2 {
|
||||
MadtEntry::LapicAddressOverride(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2)
|
||||
as *const MadtLapicAddressOverride)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidLapicAddressOverride(entry_len)
|
||||
}
|
||||
}
|
||||
0x9 => {
|
||||
if entry_len == size_of::<MadtLocalX2Apic>() + 2 {
|
||||
MadtEntry::LocalX2Apic(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2)
|
||||
as *const MadtLocalX2Apic)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidLocalX2Apic(entry_len)
|
||||
}
|
||||
}
|
||||
0xA => {
|
||||
if entry_len == size_of::<MadtLocalX2ApicNmi>() + 2 {
|
||||
MadtEntry::LocalX2ApicNmi(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2)
|
||||
as *const MadtLocalX2ApicNmi)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidLocalX2ApicNmi(entry_len)
|
||||
}
|
||||
}
|
||||
0xB => {
|
||||
if entry_len >= size_of::<MadtGicc>() + 2 {
|
||||
MadtEntry::Gicc(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2) as *const MadtGicc)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidGicc(entry_len)
|
||||
}
|
||||
}
|
||||
0xC => {
|
||||
if entry_len >= size_of::<MadtGicd>() + 2 {
|
||||
MadtEntry::Gicd(unsafe {
|
||||
&*((self.sdt.data_address() + self.i + 2) as *const MadtGicd)
|
||||
})
|
||||
} else {
|
||||
MadtEntry::InvalidGicd(entry_len)
|
||||
}
|
||||
}
|
||||
_ => MadtEntry::Unknown(entry_type),
|
||||
};
|
||||
|
||||
self.i += entry_len;
|
||||
|
||||
Some(item)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
//! # ACPI
|
||||
//! Code to parse the ACPI tables
|
||||
|
||||
use alloc::{boxed::Box, string::String, vec::Vec};
|
||||
|
||||
use hashbrown::HashMap;
|
||||
use spin::{Once, RwLock};
|
||||
|
||||
use crate::memory::{KernelMapper, PageFlags, PhysicalAddress, RmmA, RmmArch};
|
||||
|
||||
use self::{hpet::Hpet, madt::Madt, rsdp::Rsdp, rsdt::Rsdt, rxsdt::Rxsdt, sdt::Sdt, xsdt::Xsdt};
|
||||
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
mod gtdt;
|
||||
pub mod hpet;
|
||||
pub mod madt;
|
||||
mod rsdp;
|
||||
mod rsdt;
|
||||
mod rxsdt;
|
||||
pub mod sdt;
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
mod spcr;
|
||||
pub mod slit;
|
||||
pub mod srat;
|
||||
mod xsdt;
|
||||
|
||||
unsafe fn map_linearly(addr: PhysicalAddress, len: usize, mapper: &mut crate::memory::PageMapper) {
|
||||
unsafe {
|
||||
let base = PhysicalAddress::new(crate::memory::round_down_pages(addr.data()));
|
||||
let aligned_len = crate::memory::round_up_pages(len + (addr.data() - base.data()));
|
||||
|
||||
for page_idx in 0..aligned_len / crate::memory::PAGE_SIZE {
|
||||
let (_, flush) = mapper
|
||||
.map_linearly(
|
||||
base.add(page_idx * crate::memory::PAGE_SIZE),
|
||||
PageFlags::new(),
|
||||
)
|
||||
.expect("failed to linearly map SDT");
|
||||
flush.flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_sdt(sdt_address: PhysicalAddress, mapper: &mut KernelMapper<true>) -> &'static Sdt {
|
||||
let sdt;
|
||||
|
||||
unsafe {
|
||||
const SDT_SIZE: usize = size_of::<Sdt>();
|
||||
map_linearly(sdt_address, SDT_SIZE, mapper);
|
||||
|
||||
sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
|
||||
|
||||
map_linearly(
|
||||
sdt_address.add(SDT_SIZE),
|
||||
sdt.length as usize - SDT_SIZE,
|
||||
mapper,
|
||||
);
|
||||
}
|
||||
sdt
|
||||
}
|
||||
|
||||
#[repr(C, packed)]
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct GenericAddressStructure {
|
||||
pub address_space: u8,
|
||||
pub bit_width: u8,
|
||||
pub bit_offset: u8,
|
||||
pub access_size: u8,
|
||||
pub address: u64,
|
||||
}
|
||||
|
||||
pub enum RxsdtEnum {
|
||||
Rsdt(Rsdt),
|
||||
Xsdt(Xsdt),
|
||||
}
|
||||
impl Rxsdt for RxsdtEnum {
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
|
||||
match self {
|
||||
Self::Rsdt(rsdt) => <Rsdt as Rxsdt>::iter(rsdt),
|
||||
Self::Xsdt(xsdt) => <Xsdt as Rxsdt>::iter(xsdt),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static RXSDT_ENUM: Once<RxsdtEnum> = Once::new();
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct AcpiRootInfo {
|
||||
pub revision: u8,
|
||||
pub root_sdt_address: PhysicalAddress,
|
||||
}
|
||||
|
||||
pub static ACPI_ROOT_INFO: Once<AcpiRootInfo> = Once::new();
|
||||
|
||||
/// Parse the ACPI tables to gather CPU, interrupt, and timer information
|
||||
pub unsafe fn init(already_supplied_rsdp: Option<*const u8>) {
|
||||
unsafe {
|
||||
{
|
||||
let mut sdt_ptrs = SDT_POINTERS.write();
|
||||
*sdt_ptrs = Some(HashMap::new());
|
||||
}
|
||||
|
||||
// Search for RSDP
|
||||
let rsdp_opt = Rsdp::get_rsdp(already_supplied_rsdp);
|
||||
|
||||
if let Some(rsdp) = rsdp_opt {
|
||||
let root_info = ACPI_ROOT_INFO.call_once(|| AcpiRootInfo {
|
||||
revision: rsdp.revision(),
|
||||
root_sdt_address: rsdp.sdt_address(),
|
||||
});
|
||||
|
||||
if root_info.root_sdt_address != rsdp.sdt_address() || root_info.revision != rsdp.revision() {
|
||||
error!("ACPI_ROOT_INFO already initialized with a different RSDP root");
|
||||
}
|
||||
|
||||
debug!("SDT address: {:#x}", rsdp.sdt_address().data());
|
||||
let rxsdt = get_sdt(rsdp.sdt_address(), &mut KernelMapper::lock_rw());
|
||||
|
||||
let rxsdt = if let Some(rsdt) = Rsdt::new(rxsdt) {
|
||||
let mut initialized = false;
|
||||
|
||||
let rsdt = RXSDT_ENUM.call_once(|| {
|
||||
initialized = true;
|
||||
|
||||
RxsdtEnum::Rsdt(rsdt)
|
||||
});
|
||||
|
||||
if !initialized {
|
||||
error!("RXSDT_ENUM already initialized");
|
||||
}
|
||||
|
||||
rsdt
|
||||
} else if let Some(xsdt) = Xsdt::new(rxsdt) {
|
||||
let mut initialized = false;
|
||||
|
||||
let xsdt = RXSDT_ENUM.call_once(|| {
|
||||
initialized = true;
|
||||
|
||||
RxsdtEnum::Xsdt(xsdt)
|
||||
});
|
||||
if !initialized {
|
||||
error!("RXSDT_ENUM already initialized");
|
||||
}
|
||||
|
||||
xsdt
|
||||
} else {
|
||||
warn!("UNKNOWN RSDT OR XSDT SIGNATURE");
|
||||
return;
|
||||
};
|
||||
|
||||
// TODO: Don't touch ACPI tables in kernel?
|
||||
|
||||
for sdt in rxsdt.iter() {
|
||||
get_sdt(sdt, &mut KernelMapper::lock_rw());
|
||||
}
|
||||
|
||||
for sdt_address in rxsdt.iter() {
|
||||
let sdt = &*(RmmA::phys_to_virt(sdt_address).data() as *const Sdt);
|
||||
|
||||
let signature = get_sdt_signature(sdt);
|
||||
if let Some(ref mut ptrs) = *(SDT_POINTERS.write()) {
|
||||
ptrs.insert(signature, sdt);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Enumerate processors in userspace, and then provide an ACPI-independent interface
|
||||
// to initialize enumerated processors to userspace?
|
||||
// Parse SRAT BEFORE MADT so NUMA node mapping is available
|
||||
// when APs are started and PercpuBlocks are created.
|
||||
srat::init();
|
||||
|
||||
Madt::init();
|
||||
|
||||
// Parse SLIT after MADT for the NUMA distance matrix.
|
||||
slit::init();
|
||||
//TODO: support this on any arch
|
||||
// SPCR must be initialized after MADT for interrupt controllers
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
spcr::Spcr::init();
|
||||
// TODO: Let userspace setup HPET, and then provide an interface to specify which timer to
|
||||
// use?
|
||||
Hpet::init();
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
gtdt::Gtdt::init();
|
||||
} else {
|
||||
error!("NO RSDP FOUND");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub type SdtSignature = (String, [u8; 6], [u8; 8]);
|
||||
pub static SDT_POINTERS: RwLock<Option<HashMap<SdtSignature, &'static Sdt>>> = RwLock::new(None);
|
||||
|
||||
pub fn find_sdt(name: &str) -> Vec<&'static Sdt> {
|
||||
let mut sdts: Vec<&'static Sdt> = vec![];
|
||||
|
||||
if let Some(ref ptrs) = *(SDT_POINTERS.read()) {
|
||||
for (signature, sdt) in ptrs {
|
||||
if signature.0 == name {
|
||||
sdts.push(sdt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sdts
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! find_one_sdt {
|
||||
($name:expr) => {{
|
||||
use $crate::acpi::find_sdt;
|
||||
match find_sdt($name).as_slice() {
|
||||
[] => {
|
||||
println!("Unable to find {}", $name);
|
||||
return;
|
||||
}
|
||||
[x] => *x,
|
||||
x => {
|
||||
println!("{} {} found, expected 1", x.len(), $name);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}};
|
||||
}
|
||||
|
||||
pub fn get_sdt_signature(sdt: &'static Sdt) -> SdtSignature {
|
||||
let signature =
|
||||
String::from_utf8(sdt.signature.to_vec()).expect("Error converting signature to string");
|
||||
(signature, sdt.oem_id, sdt.oem_table_id)
|
||||
}
|
||||
|
||||
pub struct Acpi {
|
||||
pub hpet: RwLock<Option<Hpet>>,
|
||||
}
|
||||
|
||||
pub static ACPI_TABLE: Acpi = Acpi {
|
||||
hpet: RwLock::new(None),
|
||||
};
|
||||
@@ -1,62 +0,0 @@
|
||||
use rmm::PhysicalAddress;
|
||||
|
||||
/// RSDP
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct Rsdp {
|
||||
signature: [u8; 8],
|
||||
_checksum: u8,
|
||||
_oemid: [u8; 6],
|
||||
revision: u8,
|
||||
rsdt_address: u32,
|
||||
_length: u32,
|
||||
xsdt_address: u64,
|
||||
_extended_checksum: u8,
|
||||
_reserved: [u8; 3],
|
||||
}
|
||||
|
||||
impl Rsdp {
|
||||
pub unsafe fn get_rsdp(already_supplied_rsdp: Option<*const u8>) -> Option<Rsdp> {
|
||||
already_supplied_rsdp.and_then(|rsdp_ptr| {
|
||||
let rsdp = unsafe { *(rsdp_ptr as *const Rsdp) };
|
||||
|
||||
// Validate signature "RSD PTR "
|
||||
if &rsdp.signature != b"RSD PTR " {
|
||||
return None;
|
||||
}
|
||||
|
||||
// ACPI 1.0 checksum: sum of first 20 bytes must be zero
|
||||
let bytes_v1 = unsafe { core::slice::from_raw_parts(rsdp_ptr, 20) };
|
||||
if bytes_v1.iter().fold(0u8, |sum, &b| sum.wrapping_add(b)) != 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
// ACPI 2.0+ extended checksum: sum of entire table (length bytes) must be zero
|
||||
if rsdp.revision >= 2 {
|
||||
let full_len = rsdp._length as usize;
|
||||
if full_len < 36 || full_len > 256 {
|
||||
return None;
|
||||
}
|
||||
let bytes_full = unsafe { core::slice::from_raw_parts(rsdp_ptr, full_len) };
|
||||
if bytes_full.iter().fold(0u8, |sum, &b| sum.wrapping_add(b)) != 0 {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
Some(rsdp)
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the RSDT or XSDT address
|
||||
pub fn sdt_address(&self) -> PhysicalAddress {
|
||||
PhysicalAddress::new(if self.revision >= 2 {
|
||||
self.xsdt_address as usize
|
||||
} else {
|
||||
self.rsdt_address as usize
|
||||
})
|
||||
}
|
||||
|
||||
pub fn revision(&self) -> u8 {
|
||||
self.revision
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
use core::convert::TryFrom;
|
||||
use rmm::PhysicalAddress;
|
||||
|
||||
use super::{rxsdt::Rxsdt, sdt::Sdt};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Rsdt(&'static Sdt);
|
||||
|
||||
impl Rsdt {
|
||||
pub fn new(sdt: &'static Sdt) -> Option<Rsdt> {
|
||||
if &sdt.signature == b"RSDT" {
|
||||
Some(Rsdt(sdt))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
let length =
|
||||
usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize");
|
||||
|
||||
unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Rxsdt for Rsdt {
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
|
||||
Box::new(RsdtIter { sdt: self.0, i: 0 })
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RsdtIter {
|
||||
sdt: &'static Sdt,
|
||||
i: usize,
|
||||
}
|
||||
|
||||
impl Iterator for RsdtIter {
|
||||
type Item = PhysicalAddress;
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.i < self.sdt.data_len() / size_of::<u32>() {
|
||||
let item = unsafe {
|
||||
(self.sdt.data_address() as *const u32)
|
||||
.add(self.i)
|
||||
.read_unaligned()
|
||||
};
|
||||
self.i += 1;
|
||||
Some(PhysicalAddress::new(item as usize))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
use rmm::PhysicalAddress;
|
||||
|
||||
pub trait Rxsdt {
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>>;
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
#[derive(Copy, Clone, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct Sdt {
|
||||
pub signature: [u8; 4],
|
||||
pub length: u32,
|
||||
pub revision: u8,
|
||||
pub checksum: u8,
|
||||
pub oem_id: [u8; 6],
|
||||
pub oem_table_id: [u8; 8],
|
||||
pub oem_revision: u32,
|
||||
pub creator_id: u32,
|
||||
pub creator_revision: u32,
|
||||
}
|
||||
|
||||
impl Sdt {
|
||||
/// Get the address of this tables data
|
||||
pub fn data_address(&self) -> usize {
|
||||
self as *const _ as usize + size_of::<Sdt>()
|
||||
}
|
||||
|
||||
/// Get the length of this tables data
|
||||
pub fn data_len(&self) -> usize {
|
||||
let total_size = self.length as usize;
|
||||
let header_size = size_of::<Sdt>();
|
||||
total_size.saturating_sub(header_size)
|
||||
}
|
||||
|
||||
/// Validate the SDT checksum.
|
||||
///
|
||||
/// Per ACPI 6.5 §5.2.2: the entire table (including the checksum field)
|
||||
/// must sum to 0 when all bytes are added together as unsigned 8-bit values.
|
||||
pub fn validate_checksum(&self) -> bool {
|
||||
let ptr = self as *const _ as *const u8;
|
||||
let len = self.length as usize;
|
||||
if len < size_of::<Sdt>() {
|
||||
return false;
|
||||
}
|
||||
let sum = unsafe { core::slice::from_raw_parts(ptr, len) }
|
||||
.iter()
|
||||
.fold(0u8, |acc, &b| acc.wrapping_add(b));
|
||||
sum == 0
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
//! SLIT (System Locality Information Table) parser.
|
||||
//!
|
||||
//! Parses the NUMA distance matrix for scheduler NUMA-aware work stealing.
|
||||
|
||||
use super::sdt::Sdt;
|
||||
use crate::acpi::find_sdt;
|
||||
|
||||
const MAX_NODES: usize = 8;
|
||||
|
||||
static mut SLIT_MATRIX: [[u8; MAX_NODES]; MAX_NODES] = [[10u8; MAX_NODES]; MAX_NODES];
|
||||
static mut SLIT_NUM_NODES: usize = 0;
|
||||
static mut SLIT_AVAILABLE: bool = false;
|
||||
|
||||
pub fn is_available() -> bool { unsafe { SLIT_AVAILABLE } }
|
||||
pub fn num_nodes() -> usize { unsafe { SLIT_NUM_NODES } }
|
||||
|
||||
pub fn distance(from: u8, to: u8) -> u8 {
|
||||
if !unsafe { SLIT_AVAILABLE } { return 10; }
|
||||
let (from, to) = (from as usize, to as usize);
|
||||
if from >= MAX_NODES || to >= MAX_NODES { return 10; }
|
||||
unsafe { SLIT_MATRIX[from][to] }
|
||||
}
|
||||
|
||||
pub fn same_socket(node1: u8, node2: u8) -> bool { distance(node1, node2) <= 20 }
|
||||
|
||||
pub fn init() {
|
||||
let sdt = match find_sdt("SLIT").as_slice() {
|
||||
[] => return,
|
||||
[x] => *x,
|
||||
xs => { println!("SLIT: {} tables found, expected 1", xs.len()); return; }
|
||||
};
|
||||
if &sdt.signature != b"SLIT" { return; }
|
||||
let data_addr = sdt.data_address();
|
||||
let data_len = sdt.data_len();
|
||||
if data_len < 8 { return; }
|
||||
let num_nodes = unsafe { *(data_addr as *const u64) } as usize;
|
||||
if num_nodes == 0 || num_nodes > MAX_NODES { println!("SLIT: {num_nodes} nodes (max {MAX_NODES}), ignoring"); return; }
|
||||
let matrix_start = 8;
|
||||
let matrix_size = num_nodes * num_nodes;
|
||||
if data_len < matrix_start + matrix_size { println!("SLIT: matrix truncated ({data_len} < {})", matrix_start + matrix_size); return; }
|
||||
let matrix = unsafe { &mut SLIT_MATRIX };
|
||||
for i in 0..num_nodes { for j in 0..num_nodes { matrix[i][j] = unsafe { *((data_addr + matrix_start + i * num_nodes + j) as *const u8) }; } }
|
||||
unsafe { SLIT_NUM_NODES = num_nodes; SLIT_AVAILABLE = true; }
|
||||
debug!("SLIT: {} nodes, distance matrix loaded", num_nodes);
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
use super::{find_sdt, sdt::Sdt, GenericAddressStructure};
|
||||
use crate::{
|
||||
arch::device::serial::COM1,
|
||||
devices::{serial::SerialKind, uart_pl011},
|
||||
log::LOG,
|
||||
memory::{map_device_memory, PhysicalAddress, PAGE_SIZE},
|
||||
};
|
||||
|
||||
const INTERRUPT_TYPE_8259: u8 = 1 << 0;
|
||||
const INTERRUPT_TYPE_APIC: u8 = 1 << 1;
|
||||
const INTERRUPT_TYPE_SAPIC: u8 = 1 << 2;
|
||||
const INTERRUPT_TYPE_GIC: u8 = 1 << 3;
|
||||
const INTERRUPT_TYPE_PLIC: u8 = 1 << 4;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(C, packed)]
|
||||
pub struct Spcr {
|
||||
pub header: Sdt,
|
||||
pub interface_type: u8,
|
||||
_reserved: [u8; 3],
|
||||
pub base_address: GenericAddressStructure,
|
||||
pub interrupt_type: u8,
|
||||
pub irq: u8,
|
||||
pub gsiv: u32,
|
||||
pub configured_baud_rate: u8,
|
||||
pub parity: u8,
|
||||
pub stop_bits: u8,
|
||||
pub flow_control: u8,
|
||||
pub terminal_type: u8,
|
||||
pub language: u8,
|
||||
pub pci_device_id: u16,
|
||||
pub pci_vendor_id: u16,
|
||||
pub pci_bus: u8,
|
||||
pub pci_device: u8,
|
||||
pub pci_function: u8,
|
||||
pub pci_flags: u32,
|
||||
pub pci_segment: u8,
|
||||
/*TODO: these fields are optional based on the table revision
|
||||
pub uart_clock_frequency: u32,
|
||||
pub precise_baud_rate: u32,
|
||||
pub namespace_string_length: u16,
|
||||
pub namespace_string_offset: u16,
|
||||
*/
|
||||
// namespace_string
|
||||
}
|
||||
|
||||
impl Spcr {
|
||||
pub fn init() {
|
||||
let spcr_sdt = find_sdt("SPCR");
|
||||
let spcr = if spcr_sdt.len() == 1 {
|
||||
match Spcr::new(spcr_sdt[0]) {
|
||||
Some(spcr) => spcr,
|
||||
None => {
|
||||
warn!("Failed to parse SPCR");
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("Unable to find SPCR");
|
||||
return;
|
||||
};
|
||||
|
||||
if spcr.base_address.address == 0 {
|
||||
// Serial disabled
|
||||
return;
|
||||
}
|
||||
|
||||
let serial_was_empty = !matches!(*COM1.lock(), SerialKind::NotPresent);
|
||||
if spcr.header.revision >= 2 {
|
||||
match spcr.interface_type {
|
||||
3 => {
|
||||
// PL011
|
||||
if spcr.base_address.address_space == 0
|
||||
&& spcr.base_address.bit_width == 32
|
||||
&& spcr.base_address.bit_offset == 0
|
||||
&& spcr.base_address.access_size == 3
|
||||
{
|
||||
let virt = unsafe {
|
||||
map_device_memory(
|
||||
PhysicalAddress::new(spcr.base_address.address as usize),
|
||||
PAGE_SIZE,
|
||||
)
|
||||
};
|
||||
let serial_port = uart_pl011::SerialPort::new(virt.data(), false);
|
||||
*COM1.lock() = SerialKind::Pl011(serial_port);
|
||||
//TODO: enable IRQ on more platforms and interrupt types
|
||||
if (spcr.interrupt_type & INTERRUPT_TYPE_GIC) == INTERRUPT_TYPE_GIC {
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
unsafe {
|
||||
crate::arch::device::serial::init_acpi(spcr.gsiv);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!(
|
||||
"SPCR unsuppoted address for PL011 {:#x?}",
|
||||
spcr.base_address
|
||||
);
|
||||
}
|
||||
}
|
||||
//TODO: support more types!
|
||||
unsupported => {
|
||||
warn!(
|
||||
"SPCR revision {} unsupported interface type {}",
|
||||
spcr.header.revision, unsupported
|
||||
);
|
||||
}
|
||||
}
|
||||
} else if spcr.header.revision == 1 {
|
||||
match spcr.interface_type {
|
||||
//TODO: support more types!
|
||||
unsupported => {
|
||||
warn!("SPCR revision 1 unsupported interface type {}", unsupported);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("SPCR unsupported revision {}", spcr.header.revision);
|
||||
}
|
||||
let mut serial_port = COM1.lock();
|
||||
if serial_was_empty && !matches!(*serial_port, SerialKind::NotPresent) {
|
||||
// backfill logs since the heap is loaded
|
||||
if let Some(ref mut early_log) = *LOG.lock() {
|
||||
let (s1, s2) = early_log.read();
|
||||
if !s1.is_empty() {
|
||||
serial_port.write(s1);
|
||||
}
|
||||
if !s2.is_empty() {
|
||||
serial_port.write(s2);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(sdt: &'static Sdt) -> Option<&'static Spcr> {
|
||||
if &sdt.signature == b"SPCR" && sdt.length as usize >= size_of::<Spcr>() {
|
||||
Some(unsafe { &*((sdt as *const Sdt) as *const Spcr) })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
//! SRAT (System Resource Affinity Table) parser.
|
||||
//!
|
||||
//! Parses CPU-to-NUMA-node and memory-to-NUMA-node affinity information.
|
||||
//! Called before MADT init so that NUMA data is available during AP startup.
|
||||
|
||||
use super::sdt::Sdt;
|
||||
use crate::acpi::find_sdt;
|
||||
|
||||
const MAX_CPU_ENTRIES: usize = 256;
|
||||
const MAX_MEM_ENTRIES: usize = 64;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SratCpuEntry { apic_id: u32, node: u8, enabled: bool }
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SratMemEntry { node: u8, base: u64, length: u64, enabled: bool }
|
||||
|
||||
const CPU_NONE: SratCpuEntry = SratCpuEntry { apic_id: u32::MAX, node: 0, enabled: false };
|
||||
const MEM_NONE: SratMemEntry = SratMemEntry { node: 0, base: 0, length: 0, enabled: false };
|
||||
|
||||
static mut SRAT_CPU_ENTRIES: [SratCpuEntry; MAX_CPU_ENTRIES] = [CPU_NONE; MAX_CPU_ENTRIES];
|
||||
static mut SRAT_MEM_ENTRIES: [SratMemEntry; MAX_MEM_ENTRIES] = [MEM_NONE; MAX_MEM_ENTRIES];
|
||||
static mut SRAT_CPU_COUNT: usize = 0;
|
||||
static mut SRAT_MEM_COUNT: usize = 0;
|
||||
static mut SRAT_AVAILABLE: bool = false;
|
||||
|
||||
pub fn is_available() -> bool { unsafe { SRAT_AVAILABLE } }
|
||||
|
||||
pub fn numa_node_for_apic(apic_id: u32) -> Option<u8> {
|
||||
if !unsafe { SRAT_AVAILABLE } { return None; }
|
||||
let count = unsafe { SRAT_CPU_COUNT };
|
||||
let entries = unsafe { &SRAT_CPU_ENTRIES };
|
||||
for i in 0..count {
|
||||
if entries[i].apic_id == apic_id && entries[i].enabled { return Some(entries[i].node); }
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub fn numa_node_count() -> usize {
|
||||
if !unsafe { SRAT_AVAILABLE } { return 1; }
|
||||
let mut max_node: u8 = 0;
|
||||
let count = unsafe { SRAT_CPU_COUNT };
|
||||
let entries = unsafe { &SRAT_CPU_ENTRIES };
|
||||
for i in 0..count { if entries[i].enabled && entries[i].node > max_node { max_node = entries[i].node; } }
|
||||
(max_node as usize) + 1
|
||||
}
|
||||
|
||||
#[repr(C, packed)]
|
||||
struct SratLocalApic { _proximity_lo: u8, apic_id: u8, flags: u32, _local_sapic_eid: u8, _proximity_hi: [u8; 3], _clock_domain: u32 }
|
||||
|
||||
#[repr(C, packed)]
|
||||
struct SratMemoryAffinity { proximity_domain: u32, _reserved1: u16, base_address_lo: u32, base_address_hi: u32, length_lo: u32, length_hi: u32, _reserved2: u32, flags: u32, _reserved3: u64 }
|
||||
|
||||
#[repr(C, packed)]
|
||||
struct SratLocalX2Apic { _reserved: u16, proximity_domain: u32, x2apic_id: u32, flags: u32, _clock_domain: u32, _reserved2: u32 }
|
||||
|
||||
pub fn init() {
|
||||
let sdt = match find_sdt("SRAT").as_slice() {
|
||||
[] => return,
|
||||
[x] => *x,
|
||||
xs => { println!("SRAT: {} tables found, expected 1", xs.len()); return; }
|
||||
};
|
||||
if &sdt.signature != b"SRAT" { return; }
|
||||
let data_addr = sdt.data_address();
|
||||
let data_len = sdt.data_len();
|
||||
if data_len < 12 { println!("SRAT: table too short ({data_len} bytes)"); return; }
|
||||
let mut offset: usize = 12;
|
||||
let cpu_entries = unsafe { &mut SRAT_CPU_ENTRIES };
|
||||
let mem_entries = unsafe { &mut SRAT_MEM_ENTRIES };
|
||||
let mut cpu_count: usize = 0;
|
||||
let mut mem_count: usize = 0;
|
||||
while offset + 2 <= data_len {
|
||||
let entry_type = unsafe { *((data_addr + offset) as *const u8) };
|
||||
let entry_len = unsafe { *((data_addr + offset + 1) as *const u8) } as usize;
|
||||
if entry_len < 2 || offset + entry_len > data_len { break; }
|
||||
let entry_data = data_addr + offset + 2;
|
||||
match entry_type {
|
||||
0x0 if entry_len >= size_of::<SratLocalApic>() + 2 => {
|
||||
let e = unsafe { &*(entry_data as *const SratLocalApic) };
|
||||
let enabled = (e.flags & 1) == 1;
|
||||
let node = (e._proximity_lo as u32) | ((e._proximity_hi[0] as u32) << 8) | ((e._proximity_hi[1] as u32) << 16) | ((e._proximity_hi[2] as u32) << 24);
|
||||
if cpu_count < MAX_CPU_ENTRIES { cpu_entries[cpu_count] = SratCpuEntry { apic_id: e.apic_id as u32, node: node as u8, enabled }; cpu_count += 1; }
|
||||
}
|
||||
0x1 if entry_len >= size_of::<SratMemoryAffinity>() + 2 => {
|
||||
let e = unsafe { &*(entry_data as *const SratMemoryAffinity) };
|
||||
let enabled = (e.flags & 1) == 1;
|
||||
let base = (e.base_address_hi as u64) << 32 | e.base_address_lo as u64;
|
||||
let length = (e.length_hi as u64) << 32 | e.length_lo as u64;
|
||||
if mem_count < MAX_MEM_ENTRIES { mem_entries[mem_count] = SratMemEntry { node: e.proximity_domain as u8, base, length, enabled }; mem_count += 1; }
|
||||
}
|
||||
0x2 if entry_len >= size_of::<SratLocalX2Apic>() + 2 => {
|
||||
let e = unsafe { &*(entry_data as *const SratLocalX2Apic) };
|
||||
let enabled = (e.flags & 1) == 1;
|
||||
if cpu_count < MAX_CPU_ENTRIES { cpu_entries[cpu_count] = SratCpuEntry { apic_id: e.x2apic_id, node: e.proximity_domain as u8, enabled }; cpu_count += 1; }
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
offset += entry_len;
|
||||
}
|
||||
unsafe { SRAT_CPU_COUNT = cpu_count; SRAT_MEM_COUNT = mem_count; SRAT_AVAILABLE = true; }
|
||||
debug!("SRAT: {} CPU entries, {} memory entries", cpu_count, mem_count);
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
use core::convert::TryFrom;
|
||||
use rmm::PhysicalAddress;
|
||||
|
||||
use super::{rxsdt::Rxsdt, sdt::Sdt};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Xsdt(&'static Sdt);
|
||||
|
||||
impl Xsdt {
|
||||
pub fn new(sdt: &'static Sdt) -> Option<Xsdt> {
|
||||
if &sdt.signature == b"XSDT" {
|
||||
Some(Xsdt(sdt))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
pub fn as_slice(&self) -> &[u8] {
|
||||
let length =
|
||||
usize::try_from(self.0.length).expect("expected 32-bit length to fit within usize");
|
||||
|
||||
unsafe { core::slice::from_raw_parts(self.0 as *const _ as *const u8, length) }
|
||||
}
|
||||
}
|
||||
|
||||
impl Rxsdt for Xsdt {
|
||||
fn iter(&self) -> Box<dyn Iterator<Item = PhysicalAddress>> {
|
||||
Box::new(XsdtIter { sdt: self.0, i: 0 })
|
||||
}
|
||||
}
|
||||
|
||||
pub struct XsdtIter {
|
||||
sdt: &'static Sdt,
|
||||
i: usize,
|
||||
}
|
||||
|
||||
impl Iterator for XsdtIter {
|
||||
type Item = PhysicalAddress;
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.i < self.sdt.data_len() / size_of::<u64>() {
|
||||
let item = unsafe {
|
||||
core::ptr::read_unaligned((self.sdt.data_address() as *const u64).add(self.i))
|
||||
};
|
||||
self.i += 1;
|
||||
Some(PhysicalAddress::new(item as usize))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
use crate::memory::KernelMapper;
|
||||
use core::{
|
||||
alloc::{GlobalAlloc, Layout},
|
||||
ptr::NonNull,
|
||||
};
|
||||
use linked_list_allocator::Heap;
|
||||
use spin::Mutex;
|
||||
|
||||
static HEAP: Mutex<Option<Heap>> = Mutex::new(None);
|
||||
|
||||
pub struct Allocator;
|
||||
|
||||
impl Allocator {
|
||||
pub unsafe fn init(offset: usize, size: usize) {
|
||||
unsafe {
|
||||
*HEAP.lock() = Some(Heap::new(offset, size));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl GlobalAlloc for Allocator {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
unsafe {
|
||||
while let Some(ref mut heap) = *HEAP.lock() {
|
||||
match heap.allocate_first_fit(layout) {
|
||||
Ok(ptr) => return ptr.as_ptr(),
|
||||
Err(()) => {
|
||||
let size = heap.size();
|
||||
super::map_heap(
|
||||
&mut KernelMapper::lock_rw(),
|
||||
crate::kernel_heap_offset() + size,
|
||||
super::KERNEL_HEAP_SIZE,
|
||||
);
|
||||
heap.extend(super::KERNEL_HEAP_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
panic!("__rust_allocate: heap not initialized");
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
unsafe {
|
||||
HEAP.lock()
|
||||
.as_mut()
|
||||
.expect("heap not initialized")
|
||||
.deallocate(NonNull::new_unchecked(ptr), layout)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
use crate::memory::{KernelMapper, Page, PageFlags, VirtualAddress};
|
||||
use rmm::{Flusher, FrameAllocator, PageFlushAll};
|
||||
|
||||
pub use self::linked_list::Allocator;
|
||||
mod linked_list;
|
||||
|
||||
/// Size of kernel heap
|
||||
const KERNEL_HEAP_SIZE: usize = ::rmm::MEGABYTE;
|
||||
|
||||
#[cold]
|
||||
fn halt_kernel_heap_init(message: &str) -> ! {
|
||||
print!("{message}");
|
||||
println!("Kernel heap initialization cannot continue. Halting.");
|
||||
loop {
|
||||
core::hint::spin_loop();
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn map_heap(mapper: &mut KernelMapper<true>, offset: usize, size: usize) {
|
||||
let mut flush_all = PageFlushAll::new();
|
||||
|
||||
let heap_start_page = Page::containing_address(VirtualAddress::new(offset));
|
||||
let heap_end_page = Page::containing_address(VirtualAddress::new(offset + size - 1));
|
||||
for page in Page::range_inclusive(heap_start_page, heap_end_page) {
|
||||
let phys = match mapper.allocator_mut().allocate_one() {
|
||||
Some(phys) => phys,
|
||||
None => halt_kernel_heap_init(
|
||||
"FATAL: failed to allocate physical frame for kernel heap\n",
|
||||
),
|
||||
};
|
||||
let flush = unsafe {
|
||||
match mapper.map_phys(
|
||||
page.start_address(),
|
||||
phys,
|
||||
PageFlags::new()
|
||||
.write(true)
|
||||
.global(cfg!(not(feature = "pti"))),
|
||||
) {
|
||||
Some(flush) => flush,
|
||||
None => halt_kernel_heap_init(
|
||||
"FATAL: failed to map kernel heap virtual page\n",
|
||||
),
|
||||
}
|
||||
};
|
||||
flush_all.consume(flush);
|
||||
}
|
||||
|
||||
flush_all.flush();
|
||||
}
|
||||
|
||||
pub unsafe fn init() {
|
||||
unsafe {
|
||||
let offset = crate::kernel_heap_offset();
|
||||
let size = KERNEL_HEAP_SIZE;
|
||||
|
||||
// Map heap pages
|
||||
map_heap(&mut KernelMapper::lock_rw(), offset, size);
|
||||
|
||||
// Initialize global heap
|
||||
Allocator::init(offset, size);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
// Because the memory map is so important to not be aliased, it is defined here, in one place
|
||||
// The lower 256 PML4 entries are reserved for userspace
|
||||
// Each PML4 entry references up to 512 GB of memory
|
||||
// The second from the top (510) PML4 is reserved for the kernel
|
||||
/// The size of a single PML4
|
||||
pub const PML4_SIZE: usize = 0x0000_0080_0000_0000;
|
||||
|
||||
/// Offset to kernel heap
|
||||
#[inline(always)]
|
||||
pub fn kernel_heap_offset() -> usize {
|
||||
crate::kernel_executable_offsets::KERNEL_OFFSET() - PML4_SIZE
|
||||
}
|
||||
|
||||
/// End offset of the user image, i.e. kernel start
|
||||
pub const USER_END_OFFSET: usize = 256 * PML4_SIZE;
|
||||
@@ -1,19 +0,0 @@
|
||||
use spin::MutexGuard;
|
||||
|
||||
use crate::{arch::device::serial::COM1, devices::serial::SerialKind};
|
||||
|
||||
pub struct Writer<'a> {
|
||||
serial: MutexGuard<'a, SerialKind>,
|
||||
}
|
||||
|
||||
impl<'a> Writer<'a> {
|
||||
pub fn new() -> Writer<'a> {
|
||||
Writer {
|
||||
serial: COM1.lock(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn write(&mut self, buf: &[u8]) {
|
||||
self.serial.write(buf);
|
||||
}
|
||||
}
|
||||
@@ -1,277 +0,0 @@
|
||||
use core::fmt::{Result, Write};
|
||||
|
||||
use crate::arch::device::cpu::registers::{control_regs, id_regs};
|
||||
|
||||
pub mod registers;
|
||||
|
||||
bitfield::bitfield! {
|
||||
pub struct MachineId(u32);
|
||||
get_implementer, _: 31, 24;
|
||||
get_variant, _: 23, 20;
|
||||
get_architecture, _: 19, 16;
|
||||
get_part_number, _: 15, 4;
|
||||
get_revision, _: 3, 0;
|
||||
}
|
||||
|
||||
enum ImplementerID {
|
||||
Unknown,
|
||||
Arm,
|
||||
Broadcom,
|
||||
Cavium,
|
||||
Digital,
|
||||
Fujitsu,
|
||||
Infineon,
|
||||
Motorola,
|
||||
Nvidia,
|
||||
AMCC,
|
||||
Qualcomm,
|
||||
Marvell,
|
||||
Intel,
|
||||
Ampere,
|
||||
}
|
||||
|
||||
const IMPLEMENTERS: [&'static str; 14] = [
|
||||
"Unknown", "Arm", "Broadcom", "Cavium", "Digital", "Fujitsu", "Infineon", "Motorola", "Nvidia",
|
||||
"AMCC", "Qualcomm", "Marvell", "Intel", "Ampere",
|
||||
];
|
||||
|
||||
enum VariantID {
|
||||
Unknown,
|
||||
}
|
||||
|
||||
const VARIANTS: [&'static str; 1] = ["Unknown"];
|
||||
|
||||
enum ArchitectureID {
|
||||
Unknown,
|
||||
V4,
|
||||
V4T,
|
||||
V5,
|
||||
V5T,
|
||||
V5TE,
|
||||
V5TEJ,
|
||||
V6,
|
||||
}
|
||||
|
||||
const ARCHITECTURES: [&'static str; 8] =
|
||||
["Unknown", "v4", "v4T", "v5", "v5T", "v5TE", "v5TEJ", "v6"];
|
||||
|
||||
enum PartNumberID {
|
||||
Unknown,
|
||||
Thunder,
|
||||
Foundation,
|
||||
CortexA35,
|
||||
CortexA53,
|
||||
CortexA55,
|
||||
CortexA57,
|
||||
CortexA72,
|
||||
CortexA73,
|
||||
CortexA75,
|
||||
}
|
||||
|
||||
const PART_NUMBERS: [&'static str; 10] = [
|
||||
"Unknown",
|
||||
"Thunder",
|
||||
"Foundation",
|
||||
"Cortex-A35",
|
||||
"Cortex-A53",
|
||||
"Cortex-A55",
|
||||
"Cortex-A57",
|
||||
"Cortex-A72",
|
||||
"Cortex-A73",
|
||||
"Cortex-A75",
|
||||
];
|
||||
|
||||
enum RevisionID {
|
||||
Unknown,
|
||||
Thunder1_0,
|
||||
Thunder1_1,
|
||||
}
|
||||
|
||||
const REVISIONS: [&'static str; 3] = ["Unknown", "Thunder-1.0", "Thunder-1.1"];
|
||||
|
||||
struct CpuInfo {
|
||||
implementer: &'static str,
|
||||
variant: &'static str,
|
||||
architecture: &'static str,
|
||||
part_number: &'static str,
|
||||
revision: &'static str,
|
||||
aa64isar0: id_regs::AA64Isar0,
|
||||
aa64isar1: id_regs::AA64Isar1,
|
||||
}
|
||||
|
||||
impl CpuInfo {
|
||||
fn new() -> CpuInfo {
|
||||
let midr = unsafe { control_regs::midr() };
|
||||
let midr = MachineId(midr);
|
||||
|
||||
let implementer = match midr.get_implementer() {
|
||||
0x41 => IMPLEMENTERS[ImplementerID::Arm as usize],
|
||||
0x42 => IMPLEMENTERS[ImplementerID::Broadcom as usize],
|
||||
0x43 => IMPLEMENTERS[ImplementerID::Cavium as usize],
|
||||
0x44 => IMPLEMENTERS[ImplementerID::Digital as usize],
|
||||
0x46 => IMPLEMENTERS[ImplementerID::Fujitsu as usize],
|
||||
0x49 => IMPLEMENTERS[ImplementerID::Infineon as usize],
|
||||
0x4d => IMPLEMENTERS[ImplementerID::Motorola as usize],
|
||||
0x4e => IMPLEMENTERS[ImplementerID::Nvidia as usize],
|
||||
0x50 => IMPLEMENTERS[ImplementerID::AMCC as usize],
|
||||
0x51 => IMPLEMENTERS[ImplementerID::Qualcomm as usize],
|
||||
0x56 => IMPLEMENTERS[ImplementerID::Marvell as usize],
|
||||
0x69 => IMPLEMENTERS[ImplementerID::Intel as usize],
|
||||
0xc0 => IMPLEMENTERS[ImplementerID::Ampere as usize],
|
||||
_ => IMPLEMENTERS[ImplementerID::Unknown as usize],
|
||||
};
|
||||
|
||||
let variant = match midr.get_variant() {
|
||||
_ => VARIANTS[VariantID::Unknown as usize],
|
||||
};
|
||||
|
||||
let architecture = match midr.get_architecture() {
|
||||
0b0001 => ARCHITECTURES[ArchitectureID::V4 as usize],
|
||||
0b0010 => ARCHITECTURES[ArchitectureID::V4T as usize],
|
||||
0b0011 => ARCHITECTURES[ArchitectureID::V5 as usize],
|
||||
0b0100 => ARCHITECTURES[ArchitectureID::V5T as usize],
|
||||
0b0101 => ARCHITECTURES[ArchitectureID::V5TE as usize],
|
||||
0b0110 => ARCHITECTURES[ArchitectureID::V5TEJ as usize],
|
||||
0b0111 => ARCHITECTURES[ArchitectureID::V6 as usize],
|
||||
_ => ARCHITECTURES[ArchitectureID::Unknown as usize],
|
||||
};
|
||||
|
||||
let part_number = match midr.get_part_number() {
|
||||
0x0a1 => PART_NUMBERS[PartNumberID::Thunder as usize],
|
||||
0xd00 => PART_NUMBERS[PartNumberID::Foundation as usize],
|
||||
0xd04 => PART_NUMBERS[PartNumberID::CortexA35 as usize],
|
||||
0xd03 => PART_NUMBERS[PartNumberID::CortexA53 as usize],
|
||||
0xd05 => PART_NUMBERS[PartNumberID::CortexA55 as usize],
|
||||
0xd07 => PART_NUMBERS[PartNumberID::CortexA57 as usize],
|
||||
0xd08 => PART_NUMBERS[PartNumberID::CortexA72 as usize],
|
||||
0xd09 => PART_NUMBERS[PartNumberID::CortexA73 as usize],
|
||||
0xd0a => PART_NUMBERS[PartNumberID::CortexA75 as usize],
|
||||
_ => PART_NUMBERS[PartNumberID::Unknown as usize],
|
||||
};
|
||||
|
||||
let revision = match part_number {
|
||||
"Thunder" => {
|
||||
let val = match midr.get_revision() {
|
||||
0x00 => REVISIONS[RevisionID::Thunder1_0 as usize],
|
||||
0x01 => REVISIONS[RevisionID::Thunder1_1 as usize],
|
||||
_ => REVISIONS[RevisionID::Unknown as usize],
|
||||
};
|
||||
val
|
||||
}
|
||||
_ => REVISIONS[RevisionID::Unknown as usize],
|
||||
};
|
||||
|
||||
let aa64isar0 = id_regs::aa64isar0();
|
||||
let aa64isar1 = id_regs::aa64isar1();
|
||||
|
||||
CpuInfo {
|
||||
implementer,
|
||||
variant,
|
||||
architecture,
|
||||
part_number,
|
||||
revision,
|
||||
aa64isar0,
|
||||
aa64isar1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cpu_info<W: Write>(w: &mut W) -> Result {
|
||||
let cpuinfo = CpuInfo::new();
|
||||
|
||||
writeln!(w, "Implementer: {}", cpuinfo.implementer)?;
|
||||
writeln!(w, "Variant: {}", cpuinfo.variant)?;
|
||||
writeln!(w, "Architecture version: {}", cpuinfo.architecture)?;
|
||||
writeln!(w, "Part Number: {}", cpuinfo.part_number)?;
|
||||
writeln!(w, "Revision: {}", cpuinfo.revision)?;
|
||||
|
||||
// Print detected CPU features.
|
||||
// Follow the naming convention estabilished by `std::arch::is_aarch64_feature_detected`.
|
||||
write!(w, "Features:")?;
|
||||
|
||||
// ID_AA64ISAR0_EL1
|
||||
if cpuinfo.aa64isar0.has_feat_rng() {
|
||||
write!(w, " rand")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_flagm() {
|
||||
write!(w, " flagm")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_flagm2() {
|
||||
write!(w, " flagm2")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_fhm() {
|
||||
write!(w, " fhm")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_dotprod() {
|
||||
write!(w, " dotprod")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_sm3() && cpuinfo.aa64isar0.has_feat_sm4() {
|
||||
write!(w, " sm4")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_sha512() && cpuinfo.aa64isar0.has_feat_sha3() {
|
||||
write!(w, " sha3")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_rdm() {
|
||||
write!(w, " rdm")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_lse() {
|
||||
write!(w, " lse")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_lse128() {
|
||||
write!(w, " lse128")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_crc() {
|
||||
write!(w, " crc")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_sha1() && cpuinfo.aa64isar0.has_feat_sha256() {
|
||||
write!(w, " sha2")?;
|
||||
}
|
||||
if cpuinfo.aa64isar0.has_feat_aes() && cpuinfo.aa64isar0.has_feat_pmull() {
|
||||
write!(w, " aes")?;
|
||||
}
|
||||
|
||||
// ID_AA64ISAR1_EL1
|
||||
if cpuinfo.aa64isar1.has_feat_i8mm() {
|
||||
write!(w, " i8mm")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_bf16() {
|
||||
write!(w, " bf16")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_sb() {
|
||||
write!(w, " sb")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_frintts() {
|
||||
write!(w, " frintts")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.gpi() != 0 || cpuinfo.aa64isar1.gpa() != 0 {
|
||||
write!(w, " pacg")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_lrcpc() {
|
||||
write!(w, " rcpc")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_lrcpc2() {
|
||||
write!(w, " rcpc2")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_lrcpc3() {
|
||||
write!(w, " rcpc3")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_fcma() {
|
||||
write!(w, " fcma")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_jscvt() {
|
||||
write!(w, " jsconv")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.api() != 0 || cpuinfo.aa64isar1.apa() != 0 {
|
||||
write!(w, " paca")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_dpb() {
|
||||
write!(w, " dpb")?;
|
||||
}
|
||||
if cpuinfo.aa64isar1.has_feat_dpb2() {
|
||||
write!(w, " dpb2")?;
|
||||
}
|
||||
|
||||
writeln!(w)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,167 +0,0 @@
|
||||
#![allow(unused)]
|
||||
|
||||
//! Functions to read and write control registers.
|
||||
|
||||
use core::arch::asm;
|
||||
|
||||
pub unsafe fn ttbr0_el1() -> u64 {
|
||||
unsafe {
|
||||
let ret: u64;
|
||||
asm!("mrs {}, ttbr0_el1", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ttbr0_el1_write(val: u64) {
|
||||
unsafe {
|
||||
asm!("msr ttbr0_el1, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ttbr1_el1() -> u64 {
|
||||
unsafe {
|
||||
let ret: u64;
|
||||
asm!("mrs {}, ttbr1_el1", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ttbr1_el1_write(val: u64) {
|
||||
unsafe {
|
||||
asm!("msr ttbr1_el1, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidr_el0() -> u64 {
|
||||
unsafe {
|
||||
let ret: u64;
|
||||
asm!("mrs {}, tpidr_el0", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidr_el0_write(val: u64) {
|
||||
unsafe {
|
||||
asm!("msr tpidr_el0, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidr_el1() -> u64 {
|
||||
unsafe {
|
||||
let ret: u64;
|
||||
asm!("mrs {}, tpidr_el1", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidr_el1_write(val: u64) {
|
||||
unsafe {
|
||||
asm!("msr tpidr_el1, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidrro_el0() -> u64 {
|
||||
unsafe {
|
||||
let ret: u64;
|
||||
asm!("mrs {}, tpidrro_el0", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn tpidrro_el0_write(val: u64) {
|
||||
unsafe {
|
||||
asm!("msr tpidrro_el0, {}", in(reg) val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn esr_el1() -> u32 {
|
||||
unsafe {
|
||||
let ret: u32;
|
||||
asm!("mrs {0:w}, esr_el1", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn vhe_present() -> bool {
|
||||
unsafe {
|
||||
let mut mmfr1: u64;
|
||||
asm!("mrs {}, id_aa64mmfr1_el1", out(reg) mmfr1);
|
||||
|
||||
// The VHE (Virtualization Host Extensions) field is in bits [7:4].
|
||||
let vhe_field = (mmfr1 >> 4) & 0b1111;
|
||||
|
||||
vhe_field != 0
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn cntfrq_el0() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {}, cntfrq_el0", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ptmr_ctrl() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {}, cntp_ctl_el0", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ptmr_ctrl_write(val: u32) {
|
||||
unsafe {
|
||||
asm!("msr cntp_ctl_el0, {}", in(reg) val as usize);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ptmr_tval() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {0}, cntp_tval_el0", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn ptmr_tval_write(val: u32) {
|
||||
unsafe {
|
||||
asm!("msr cntp_tval_el0, {}", in(reg) val as usize);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn vtmr_ctrl() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {}, cntv_ctl_el0", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn vtmr_ctrl_write(val: u32) {
|
||||
unsafe {
|
||||
asm!("msr cntv_ctl_el0, {}", in(reg) val as usize);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn vtmr_tval() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {0}, cntv_tval_el0", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn vtmr_tval_write(val: u32) {
|
||||
unsafe {
|
||||
asm!("msr cntv_tval_el0, {}", in(reg) val as usize);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn midr() -> u32 {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
asm!("mrs {}, midr_el1", out(reg) ret);
|
||||
ret as u32
|
||||
}
|
||||
}
|
||||
@@ -1,151 +0,0 @@
|
||||
//! Functions and bitfield definitions for `ID_AA64*` system registers. (e.g. `ID_AA64ISAR0_EL1`)
|
||||
|
||||
use core::arch::asm;
|
||||
|
||||
bitfield::bitfield! {
|
||||
pub struct AA64Isar0(u64);
|
||||
impl Debug;
|
||||
pub rndr, _: 63, 60;
|
||||
pub tlb, _: 59, 56;
|
||||
pub ts, _: 55, 52;
|
||||
pub fhm, _: 51, 48;
|
||||
pub dp, _: 47, 44;
|
||||
pub sm4, _: 43, 40;
|
||||
pub sm3, _: 39, 36;
|
||||
pub sha3, _: 35, 32;
|
||||
pub rdm, _: 31, 28;
|
||||
pub atomic, _: 23, 20;
|
||||
pub crc32, _: 19, 16;
|
||||
pub sha2, _: 15, 12;
|
||||
pub sha1, _: 11, 8;
|
||||
pub aes, _: 7, 4;
|
||||
}
|
||||
|
||||
bitfield::bitfield! {
|
||||
pub struct AA64Isar1(u64);
|
||||
impl Debug;
|
||||
pub ls64, _: 63, 60;
|
||||
pub xs, _: 59, 56;
|
||||
pub i8mm, _: 55, 52;
|
||||
pub dgh, _: 51, 48;
|
||||
pub bf16, _: 47, 44;
|
||||
pub specres, _: 43, 40;
|
||||
pub sb, _: 39, 36;
|
||||
pub frintts, _: 35, 32;
|
||||
pub gpi, _: 31, 28;
|
||||
pub gpa, _: 27, 24;
|
||||
pub lrcpc, _: 23, 20;
|
||||
pub fcma, _: 19, 16;
|
||||
pub jscvt, _: 15, 12;
|
||||
pub api, _: 11, 8;
|
||||
pub apa, _: 7, 4;
|
||||
pub dpb, _: 3, 0;
|
||||
}
|
||||
|
||||
impl AA64Isar0 {
|
||||
pub fn has_feat_rng(&self) -> bool {
|
||||
self.rndr() == 0b0001
|
||||
}
|
||||
pub fn has_feat_flagm(&self) -> bool {
|
||||
self.ts() == 0b0001
|
||||
}
|
||||
pub fn has_feat_flagm2(&self) -> bool {
|
||||
self.ts() == 0b0010
|
||||
}
|
||||
pub fn has_feat_fhm(&self) -> bool {
|
||||
self.fhm() == 0b0001
|
||||
}
|
||||
pub fn has_feat_dotprod(&self) -> bool {
|
||||
self.dp() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sm4(&self) -> bool {
|
||||
self.sm4() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sm3(&self) -> bool {
|
||||
self.sm3() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sha3(&self) -> bool {
|
||||
self.sha3() == 0b0001
|
||||
}
|
||||
pub fn has_feat_rdm(&self) -> bool {
|
||||
self.rdm() == 0b0001
|
||||
}
|
||||
pub fn has_feat_lse(&self) -> bool {
|
||||
self.atomic() == 0b0010
|
||||
}
|
||||
pub fn has_feat_lse128(&self) -> bool {
|
||||
self.atomic() == 0b0011
|
||||
}
|
||||
/// The current Arm Architecture Registers Manual calls it FEAT_CRC32,
|
||||
/// but everyone else seems to call it FEAT_CRC.
|
||||
pub fn has_feat_crc(&self) -> bool {
|
||||
self.crc32() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sha256(&self) -> bool {
|
||||
self.sha2() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sha512(&self) -> bool {
|
||||
self.sha2() == 0b0010
|
||||
}
|
||||
pub fn has_feat_sha1(&self) -> bool {
|
||||
self.sha1() == 0b0001
|
||||
}
|
||||
pub fn has_feat_aes(&self) -> bool {
|
||||
self.aes() == 0b0001
|
||||
}
|
||||
pub fn has_feat_pmull(&self) -> bool {
|
||||
self.aes() == 0b0010
|
||||
}
|
||||
}
|
||||
|
||||
impl AA64Isar1 {
|
||||
pub fn has_feat_i8mm(&self) -> bool {
|
||||
self.i8mm() == 0b0001
|
||||
}
|
||||
pub fn has_feat_bf16(&self) -> bool {
|
||||
self.bf16() == 0b0001
|
||||
}
|
||||
pub fn has_feat_sb(&self) -> bool {
|
||||
self.sb() == 0b0001
|
||||
}
|
||||
pub fn has_feat_frintts(&self) -> bool {
|
||||
self.frintts() == 0b0001
|
||||
}
|
||||
pub fn has_feat_lrcpc(&self) -> bool {
|
||||
self.lrcpc() == 0b0001
|
||||
}
|
||||
pub fn has_feat_lrcpc2(&self) -> bool {
|
||||
self.lrcpc() == 0b0010
|
||||
}
|
||||
pub fn has_feat_lrcpc3(&self) -> bool {
|
||||
self.lrcpc() == 0b0011
|
||||
}
|
||||
pub fn has_feat_fcma(&self) -> bool {
|
||||
self.fcma() == 0b0001
|
||||
}
|
||||
pub fn has_feat_jscvt(&self) -> bool {
|
||||
self.jscvt() == 0b0011
|
||||
}
|
||||
pub fn has_feat_dpb(&self) -> bool {
|
||||
self.dpb() == 0b0001
|
||||
}
|
||||
pub fn has_feat_dpb2(&self) -> bool {
|
||||
self.dpb() == 0b0010
|
||||
}
|
||||
}
|
||||
|
||||
pub fn aa64isar0() -> AA64Isar0 {
|
||||
let ret: u64;
|
||||
unsafe {
|
||||
asm!("mrs {}, ID_AA64ISAR0_EL1", out(reg) ret);
|
||||
}
|
||||
AA64Isar0(ret)
|
||||
}
|
||||
|
||||
pub fn aa64isar1() -> AA64Isar1 {
|
||||
let ret: u64;
|
||||
unsafe {
|
||||
asm!("mrs {}, ID_AA64ISAR1_EL1", out(reg) ret);
|
||||
}
|
||||
AA64Isar1(ret)
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
pub mod control_regs;
|
||||
pub mod id_regs;
|
||||
@@ -1,145 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
|
||||
use super::ic_for_chip;
|
||||
use crate::{
|
||||
arch::device::cpu::registers::control_regs,
|
||||
context::{self, timeout},
|
||||
dtb::{
|
||||
get_interrupt,
|
||||
irqchip::{register_irq, InterruptHandler, IRQ_CHIP},
|
||||
},
|
||||
scheme::irq::irq_trigger,
|
||||
sync::CleanLockToken,
|
||||
time,
|
||||
};
|
||||
use fdt::Fdt;
|
||||
|
||||
bitflags! {
|
||||
struct TimerCtrlFlags: u32 {
|
||||
const ENABLE = 1 << 0;
|
||||
const IMASK = 1 << 1;
|
||||
const ISTATUS = 1 << 2;
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn init(fdt: &Fdt) {
|
||||
unsafe {
|
||||
let mut timer = GenericTimer::new();
|
||||
timer.init();
|
||||
if let Some(node) = fdt.find_compatible(&["arm,armv7-timer"]) {
|
||||
let irq = get_interrupt(fdt, &node, 1).unwrap();
|
||||
debug!("irq = {:?}", irq);
|
||||
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
|
||||
//PHYS_NONSECURE_PPI only
|
||||
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
|
||||
.ic
|
||||
.irq_xlate(irq)
|
||||
.unwrap();
|
||||
info!("generic_timer virq = {}", virq);
|
||||
register_irq(virq as u32, Box::new(timer));
|
||||
IRQ_CHIP.irq_enable(virq as u32);
|
||||
} else {
|
||||
error!("Failed to find irq parent for generic timer");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct GenericTimer {
|
||||
pub use_virtual_timer: bool,
|
||||
pub clk_freq: u32,
|
||||
pub reload_count: u32,
|
||||
}
|
||||
|
||||
impl GenericTimer {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
use_virtual_timer: false,
|
||||
clk_freq: 0,
|
||||
reload_count: 0,
|
||||
}
|
||||
}
|
||||
pub fn init(&mut self) {
|
||||
self.use_virtual_timer = unsafe { !control_regs::vhe_present() };
|
||||
debug!(
|
||||
"generic_timer use_virtual_timer = {:?}",
|
||||
self.use_virtual_timer
|
||||
);
|
||||
let clk_freq = unsafe { control_regs::cntfrq_el0() };
|
||||
self.clk_freq = clk_freq;
|
||||
self.reload_count = clk_freq / 100;
|
||||
self.reload_count();
|
||||
}
|
||||
|
||||
fn read_tmr_ctrl(&self) -> TimerCtrlFlags {
|
||||
TimerCtrlFlags::from_bits_truncate(if self.use_virtual_timer {
|
||||
unsafe { control_regs::vtmr_ctrl() }
|
||||
} else {
|
||||
unsafe { control_regs::ptmr_ctrl() }
|
||||
})
|
||||
}
|
||||
|
||||
fn write_tmr_ctrl(&self, ctrl: TimerCtrlFlags) {
|
||||
if self.use_virtual_timer {
|
||||
unsafe { control_regs::vtmr_ctrl_write(ctrl.bits()) };
|
||||
} else {
|
||||
unsafe { control_regs::ptmr_ctrl_write(ctrl.bits()) };
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
fn disable(&self) {
|
||||
let mut ctrl = self.read_tmr_ctrl();
|
||||
ctrl.remove(TimerCtrlFlags::ENABLE);
|
||||
self.write_tmr_ctrl(ctrl);
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn set_irq(&mut self) {
|
||||
let mut ctrl = self.read_tmr_ctrl();
|
||||
ctrl.remove(TimerCtrlFlags::IMASK);
|
||||
self.write_tmr_ctrl(ctrl);
|
||||
}
|
||||
|
||||
pub fn clear_irq(&mut self) {
|
||||
let mut ctrl = self.read_tmr_ctrl();
|
||||
|
||||
if ctrl.contains(TimerCtrlFlags::ISTATUS) {
|
||||
ctrl.insert(TimerCtrlFlags::IMASK);
|
||||
self.write_tmr_ctrl(ctrl);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn reload_count(&mut self) {
|
||||
if self.use_virtual_timer {
|
||||
unsafe { control_regs::vtmr_tval_write(self.reload_count) };
|
||||
} else {
|
||||
unsafe { control_regs::ptmr_tval_write(self.reload_count) };
|
||||
}
|
||||
let mut ctrl = self.read_tmr_ctrl();
|
||||
ctrl.insert(TimerCtrlFlags::ENABLE);
|
||||
ctrl.remove(TimerCtrlFlags::IMASK);
|
||||
self.write_tmr_ctrl(ctrl);
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptHandler for GenericTimer {
|
||||
fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
|
||||
self.clear_irq();
|
||||
{
|
||||
*time::OFFSET.write(token.token()) += self.clk_freq as u128;
|
||||
}
|
||||
|
||||
timeout::trigger(token);
|
||||
context::switch::tick(token);
|
||||
|
||||
unsafe {
|
||||
// FIXME add_irq accepts a u8 as irq number
|
||||
// PercpuBlock::current().stats.add_irq(irq);
|
||||
|
||||
irq_trigger(irq.try_into().unwrap(), token);
|
||||
IRQ_CHIP.irq_eoi(irq);
|
||||
}
|
||||
self.reload_count();
|
||||
}
|
||||
}
|
||||
@@ -1,288 +0,0 @@
|
||||
use super::InterruptController;
|
||||
use crate::{
|
||||
dtb::{
|
||||
get_mmio_address,
|
||||
irqchip::{InterruptHandler, IrqCell, IrqDesc},
|
||||
},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
use core::ptr::{read_volatile, write_volatile};
|
||||
use fdt::{node::FdtNode, Fdt};
|
||||
use syscall::{
|
||||
error::{Error, EINVAL},
|
||||
Result,
|
||||
};
|
||||
|
||||
static GICD_CTLR: u32 = 0x000;
|
||||
static GICD_TYPER: u32 = 0x004;
|
||||
static GICD_ISENABLER: u32 = 0x100;
|
||||
static GICD_ICENABLER: u32 = 0x180;
|
||||
static GICD_IPRIORITY: u32 = 0x400;
|
||||
static GICD_ITARGETSR: u32 = 0x800;
|
||||
static GICD_ICFGR: u32 = 0xc00;
|
||||
|
||||
static GICC_EOIR: u32 = 0x0010;
|
||||
static GICC_IAR: u32 = 0x000c;
|
||||
static GICC_CTLR: u32 = 0x0000;
|
||||
static GICC_PMR: u32 = 0x0004;
|
||||
|
||||
pub struct GenericInterruptController {
|
||||
pub gic_dist_if: GicDistIf,
|
||||
pub gic_cpu_if: GicCpuIf,
|
||||
pub irq_range: (usize, usize),
|
||||
}
|
||||
|
||||
impl GenericInterruptController {
|
||||
pub fn new() -> Self {
|
||||
let gic_dist_if = GicDistIf::default();
|
||||
let gic_cpu_if = GicCpuIf::default();
|
||||
|
||||
GenericInterruptController {
|
||||
gic_dist_if,
|
||||
gic_cpu_if,
|
||||
irq_range: (0, 0),
|
||||
}
|
||||
}
|
||||
pub fn parse(fdt: &Fdt) -> Result<(usize, usize, usize, usize)> {
|
||||
if let Some(node) = fdt.find_compatible(&["arm,cortex-a15-gic", "arm,gic-400"]) {
|
||||
return GenericInterruptController::parse_inner(fdt, &node);
|
||||
} else {
|
||||
return Err(Error::new(EINVAL));
|
||||
}
|
||||
}
|
||||
fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, usize, usize)> {
|
||||
//assert address_cells == 0x2, size_cells == 0x2
|
||||
let reg = node.reg().unwrap();
|
||||
let mut regs = (0, 0, 0, 0);
|
||||
let mut idx = 0;
|
||||
|
||||
for chunk in reg {
|
||||
if chunk.size.is_none() {
|
||||
break;
|
||||
}
|
||||
let addr = get_mmio_address(fdt, node, &chunk).unwrap();
|
||||
match idx {
|
||||
0 => (regs.0, regs.1) = (addr, chunk.size.unwrap()),
|
||||
2 => (regs.2, regs.3) = (addr, chunk.size.unwrap()),
|
||||
_ => break,
|
||||
}
|
||||
idx += 2;
|
||||
}
|
||||
|
||||
if idx == 4 {
|
||||
Ok(regs)
|
||||
} else {
|
||||
Err(Error::new(EINVAL))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptHandler for GenericInterruptController {
|
||||
fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
|
||||
}
|
||||
|
||||
impl InterruptController for GenericInterruptController {
|
||||
fn irq_init(
|
||||
&mut self,
|
||||
fdt_opt: Option<&Fdt>,
|
||||
irq_desc: &mut [IrqDesc; 1024],
|
||||
ic_idx: usize,
|
||||
irq_idx: &mut usize,
|
||||
) -> Result<()> {
|
||||
if let Some(fdt) = fdt_opt {
|
||||
let (dist_addr, _dist_size, cpu_addr, _cpu_size) =
|
||||
match GenericInterruptController::parse(fdt) {
|
||||
Ok(regs) => regs,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
unsafe {
|
||||
self.gic_dist_if.init(crate::PHYS_OFFSET + dist_addr);
|
||||
self.gic_cpu_if.init(crate::PHYS_OFFSET + cpu_addr);
|
||||
}
|
||||
}
|
||||
let idx = *irq_idx;
|
||||
let cnt = if self.gic_dist_if.nirqs > 1024 {
|
||||
1024
|
||||
} else {
|
||||
self.gic_dist_if.nirqs as usize
|
||||
};
|
||||
let mut i: usize = 0;
|
||||
//only support linear irq map now.
|
||||
while i < cnt && (idx + i < 1024) {
|
||||
irq_desc[idx + i].basic.ic_idx = ic_idx;
|
||||
irq_desc[idx + i].basic.ic_irq = i as u32;
|
||||
irq_desc[idx + i].basic.used = true;
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
info!("gic irq_range = ({}, {})", idx, idx + cnt);
|
||||
self.irq_range = (idx, idx + cnt);
|
||||
*irq_idx = idx + cnt;
|
||||
Ok(())
|
||||
}
|
||||
fn irq_ack(&mut self) -> u32 {
|
||||
unsafe { self.gic_cpu_if.irq_ack() }
|
||||
}
|
||||
fn irq_eoi(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_cpu_if.irq_eoi(irq_num) }
|
||||
}
|
||||
fn irq_enable(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_dist_if.irq_enable(irq_num) }
|
||||
}
|
||||
fn irq_disable(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_dist_if.irq_disable(irq_num) }
|
||||
}
|
||||
fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
|
||||
let off = match irq_data {
|
||||
IrqCell::L3(0, irq, _flags) => irq as usize + 32, // SPI
|
||||
IrqCell::L3(1, irq, _flags) => irq as usize + 16, // PPI
|
||||
_ => return Err(Error::new(EINVAL)),
|
||||
};
|
||||
return Ok(off + self.irq_range.0);
|
||||
}
|
||||
fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
|
||||
if hwirq >= self.gic_dist_if.nirqs {
|
||||
None
|
||||
} else {
|
||||
Some(self.irq_range.0 + hwirq as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct GicDistIf {
|
||||
pub address: usize,
|
||||
pub ncpus: u32,
|
||||
pub nirqs: u32,
|
||||
}
|
||||
|
||||
impl GicDistIf {
|
||||
pub unsafe fn init(&mut self, addr: usize) {
|
||||
unsafe {
|
||||
self.address = addr;
|
||||
|
||||
// Disable IRQ Distribution
|
||||
self.write(GICD_CTLR, 0);
|
||||
|
||||
let typer = self.read(GICD_TYPER);
|
||||
self.ncpus = ((typer & (0x7 << 5)) >> 5) + 1;
|
||||
self.nirqs = ((typer & 0x1f) + 1) * 32;
|
||||
info!(
|
||||
"gic: Distributor supports {:?} CPUs and {:?} IRQs",
|
||||
self.ncpus, self.nirqs
|
||||
);
|
||||
|
||||
// Set all SPIs to level triggered
|
||||
for irq in (32..self.nirqs).step_by(16) {
|
||||
self.write(GICD_ICFGR + ((irq / 16) * 4), 0);
|
||||
}
|
||||
|
||||
// Disable all SPIs
|
||||
for irq in (32..self.nirqs).step_by(32) {
|
||||
self.write(GICD_ICENABLER + ((irq / 32) * 4), 0xffff_ffff);
|
||||
}
|
||||
|
||||
// Affine all SPIs to CPU0 and set priorities for all IRQs
|
||||
for irq in 0..self.nirqs {
|
||||
if irq > 31 {
|
||||
let ext_offset = GICD_ITARGETSR + (4 * (irq / 4));
|
||||
let int_offset = irq % 4;
|
||||
let mut val = self.read(ext_offset);
|
||||
val |= 0b0000_0001 << (8 * int_offset);
|
||||
self.write(ext_offset, val);
|
||||
}
|
||||
|
||||
let ext_offset = GICD_IPRIORITY + (4 * (irq / 4));
|
||||
let int_offset = irq % 4;
|
||||
let mut val = self.read(ext_offset);
|
||||
val |= 0b0000_0000 << (8 * int_offset);
|
||||
self.write(ext_offset, val);
|
||||
}
|
||||
|
||||
// Enable IRQ group 0 and group 1 non-secure distribution
|
||||
self.write(GICD_CTLR, 0x3);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn irq_enable(&mut self, irq: u32) {
|
||||
unsafe {
|
||||
let offset = GICD_ISENABLER + (4 * (irq / 32));
|
||||
let shift = 1 << (irq % 32);
|
||||
let mut val = self.read(offset);
|
||||
val |= shift;
|
||||
self.write(offset, val);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn irq_disable(&mut self, irq: u32) {
|
||||
unsafe {
|
||||
let offset = GICD_ICENABLER + (4 * (irq / 32));
|
||||
let shift = 1 << (irq % 32);
|
||||
let mut val = self.read(offset);
|
||||
val |= shift;
|
||||
self.write(offset, val);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn read(&self, reg: u32) -> u32 {
|
||||
unsafe {
|
||||
let val = read_volatile((self.address + reg as usize) as *const u32);
|
||||
val
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn write(&mut self, reg: u32, value: u32) {
|
||||
unsafe {
|
||||
write_volatile((self.address + reg as usize) as *mut u32, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct GicCpuIf {
|
||||
pub address: usize,
|
||||
}
|
||||
|
||||
impl GicCpuIf {
|
||||
pub unsafe fn init(&mut self, addr: usize) {
|
||||
unsafe {
|
||||
self.address = addr;
|
||||
|
||||
// Enable CPU0's GIC interface
|
||||
self.write(GICC_CTLR, 1);
|
||||
// Set CPU0's Interrupt Priority Mask
|
||||
self.write(GICC_PMR, 0xff);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn irq_ack(&mut self) -> u32 {
|
||||
unsafe {
|
||||
let irq = self.read(GICC_IAR) & 0x1ff;
|
||||
if irq == 1023 {
|
||||
panic!("irq_ack: got ID 1023!!!");
|
||||
}
|
||||
irq
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn irq_eoi(&mut self, irq: u32) {
|
||||
unsafe {
|
||||
self.write(GICC_EOIR, irq);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn read(&self, reg: u32) -> u32 {
|
||||
unsafe {
|
||||
let val = read_volatile((self.address + reg as usize) as *const u32);
|
||||
val
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn write(&mut self, reg: u32, value: u32) {
|
||||
unsafe {
|
||||
write_volatile((self.address + reg as usize) as *mut u32, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
use alloc::vec::Vec;
|
||||
use core::arch::asm;
|
||||
use fdt::{node::NodeProperty, Fdt};
|
||||
|
||||
use super::{gic::GicDistIf, InterruptController};
|
||||
use crate::{
|
||||
dtb::{
|
||||
get_mmio_address,
|
||||
irqchip::{InterruptHandler, IrqCell, IrqDesc},
|
||||
},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
use syscall::{
|
||||
error::{Error, EINVAL},
|
||||
Result,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GicV3 {
|
||||
pub gic_dist_if: GicDistIf,
|
||||
pub gic_cpu_if: GicV3CpuIf,
|
||||
pub gicrs: Vec<(usize, usize)>,
|
||||
//TODO: GICC, GICH, GICV?
|
||||
pub irq_range: (usize, usize),
|
||||
}
|
||||
|
||||
impl GicV3 {
|
||||
pub fn new() -> Self {
|
||||
GicV3 {
|
||||
gic_dist_if: GicDistIf::default(),
|
||||
gic_cpu_if: GicV3CpuIf,
|
||||
gicrs: Vec::new(),
|
||||
irq_range: (0, 0),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(&mut self, fdt: &Fdt) -> Result<()> {
|
||||
let Some(node) = fdt.find_compatible(&["arm,gic-v3"]) else {
|
||||
return Err(Error::new(EINVAL));
|
||||
};
|
||||
|
||||
// Clear current registers
|
||||
//TODO: deinit?
|
||||
self.gic_dist_if.address = 0;
|
||||
self.gicrs.clear();
|
||||
|
||||
// Get number of GICRs
|
||||
let gicrs = node
|
||||
.property("#redistributor-regions")
|
||||
.and_then(NodeProperty::as_usize)
|
||||
.unwrap_or(1);
|
||||
|
||||
// Read registers
|
||||
let mut chunks = node.reg().unwrap();
|
||||
if let Some(gicd) = chunks.next()
|
||||
&& let Some(addr) = get_mmio_address(fdt, &node, &gicd)
|
||||
{
|
||||
unsafe {
|
||||
self.gic_dist_if.init(crate::PHYS_OFFSET + addr);
|
||||
}
|
||||
}
|
||||
for _ in 0..gicrs {
|
||||
if let Some(gicr) = chunks.next() {
|
||||
self.gicrs.push((
|
||||
get_mmio_address(fdt, &node, &gicr).unwrap(),
|
||||
gicr.size.unwrap(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if self.gic_dist_if.address == 0 || self.gicrs.is_empty() {
|
||||
Err(Error::new(EINVAL))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptHandler for GicV3 {
|
||||
fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
|
||||
}
|
||||
|
||||
impl InterruptController for GicV3 {
|
||||
fn irq_init(
|
||||
&mut self,
|
||||
fdt_opt: Option<&Fdt>,
|
||||
irq_desc: &mut [IrqDesc; 1024],
|
||||
ic_idx: usize,
|
||||
irq_idx: &mut usize,
|
||||
) -> Result<()> {
|
||||
if let Some(fdt) = fdt_opt {
|
||||
self.parse(fdt)?;
|
||||
}
|
||||
info!("{:X?}", self);
|
||||
|
||||
unsafe {
|
||||
self.gic_cpu_if.init();
|
||||
}
|
||||
let idx = *irq_idx;
|
||||
let cnt = if self.gic_dist_if.nirqs > 1024 {
|
||||
1024
|
||||
} else {
|
||||
self.gic_dist_if.nirqs as usize
|
||||
};
|
||||
let mut i: usize = 0;
|
||||
//only support linear irq map now.
|
||||
while i < cnt && (idx + i < 1024) {
|
||||
irq_desc[idx + i].basic.ic_idx = ic_idx;
|
||||
irq_desc[idx + i].basic.ic_irq = i as u32;
|
||||
irq_desc[idx + i].basic.used = true;
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
info!("gic irq_range = ({}, {})", idx, idx + cnt);
|
||||
self.irq_range = (idx, idx + cnt);
|
||||
*irq_idx = idx + cnt;
|
||||
Ok(())
|
||||
}
|
||||
fn irq_ack(&mut self) -> u32 {
|
||||
let irq_num = unsafe { self.gic_cpu_if.irq_ack() };
|
||||
irq_num
|
||||
}
|
||||
fn irq_eoi(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_cpu_if.irq_eoi(irq_num) }
|
||||
}
|
||||
fn irq_enable(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_dist_if.irq_enable(irq_num) }
|
||||
}
|
||||
fn irq_disable(&mut self, irq_num: u32) {
|
||||
unsafe { self.gic_dist_if.irq_disable(irq_num) }
|
||||
}
|
||||
fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
|
||||
let off = match irq_data {
|
||||
IrqCell::L3(0, irq, _flags) => irq as usize + 32, // SPI
|
||||
IrqCell::L3(1, irq, _flags) => irq as usize + 16, // PPI
|
||||
_ => return Err(Error::new(EINVAL)),
|
||||
};
|
||||
return Ok(off + self.irq_range.0);
|
||||
}
|
||||
fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
|
||||
if hwirq >= self.gic_dist_if.nirqs {
|
||||
None
|
||||
} else {
|
||||
Some(self.irq_range.0 + hwirq as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct GicV3CpuIf;
|
||||
|
||||
impl GicV3CpuIf {
|
||||
pub unsafe fn init(&mut self) {
|
||||
unsafe {
|
||||
// Enable system register access
|
||||
{
|
||||
let value = 1_usize;
|
||||
asm!("msr icc_sre_el1, {}", in(reg) value);
|
||||
}
|
||||
// Set control register
|
||||
{
|
||||
let value = 0_usize;
|
||||
asm!("msr icc_ctlr_el1, {}", in(reg) value);
|
||||
}
|
||||
// Enable non-secure group 1
|
||||
{
|
||||
let value = 1_usize;
|
||||
asm!("msr icc_igrpen1_el1, {}", in(reg) value);
|
||||
}
|
||||
// Set CPU0's Interrupt Priority Mask
|
||||
{
|
||||
let value = 0xFF_usize;
|
||||
asm!("msr icc_pmr_el1, {}", in(reg) value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn irq_ack(&mut self) -> u32 {
|
||||
unsafe {
|
||||
let mut irq: usize;
|
||||
asm!("mrs {}, icc_iar1_el1", out(reg) irq);
|
||||
irq &= 0x1ff;
|
||||
if irq == 1023 {
|
||||
panic!("irq_ack: got ID 1023!!!");
|
||||
}
|
||||
irq as u32
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn irq_eoi(&mut self, irq: u32) {
|
||||
unsafe {
|
||||
asm!("msr icc_eoir1_el1, {}", in(reg) irq as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,299 +0,0 @@
|
||||
use core::ptr::{read_volatile, write_volatile};
|
||||
use fdt::{node::FdtNode, Fdt};
|
||||
|
||||
use super::InterruptController;
|
||||
use crate::{
|
||||
dtb::{
|
||||
get_interrupt, get_mmio_address,
|
||||
irqchip::{InterruptHandler, IrqCell, IrqDesc, IRQ_CHIP},
|
||||
},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
use syscall::{
|
||||
error::{Error, EINVAL},
|
||||
Result,
|
||||
};
|
||||
|
||||
#[inline(always)]
|
||||
fn ffs(num: u32) -> u32 {
|
||||
let mut x = num;
|
||||
if x == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut r = 1;
|
||||
if (x & 0xffff) == 0 {
|
||||
x >>= 16;
|
||||
r += 16;
|
||||
}
|
||||
if (x & 0xff) == 0 {
|
||||
x >>= 8;
|
||||
r += 8;
|
||||
}
|
||||
if (x & 0xf) == 0 {
|
||||
x >>= 4;
|
||||
r += 4;
|
||||
}
|
||||
if (x & 0x3) == 0 {
|
||||
x >>= 2;
|
||||
r += 2;
|
||||
}
|
||||
if (x & 0x1) == 0 {
|
||||
r += 1;
|
||||
}
|
||||
|
||||
r
|
||||
}
|
||||
|
||||
const PENDING_0: u32 = 0x0;
|
||||
const PENDING_1: u32 = 0x4;
|
||||
const PENDING_2: u32 = 0x8;
|
||||
const ENABLE_0: u32 = 0x18;
|
||||
const ENABLE_1: u32 = 0x10;
|
||||
const ENABLE_2: u32 = 0x14;
|
||||
const DISABLE_0: u32 = 0x24;
|
||||
const DISABLE_1: u32 = 0x1c;
|
||||
const DISABLE_2: u32 = 0x20;
|
||||
|
||||
pub struct Bcm2835ArmInterruptController {
|
||||
pub address: usize,
|
||||
pub irq_range: (usize, usize),
|
||||
}
|
||||
|
||||
impl Bcm2835ArmInterruptController {
|
||||
pub fn new() -> Self {
|
||||
Bcm2835ArmInterruptController {
|
||||
address: 0,
|
||||
irq_range: (0, 0),
|
||||
}
|
||||
}
|
||||
pub fn parse(fdt: &Fdt) -> Result<(usize, usize, Option<usize>)> {
|
||||
if let Some(node) = fdt.find_compatible(&["brcm,bcm2836-armctrl-ic"]) {
|
||||
return unsafe { Bcm2835ArmInterruptController::parse_inner(fdt, &node) };
|
||||
} else {
|
||||
return Err(Error::new(EINVAL));
|
||||
}
|
||||
}
|
||||
unsafe fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize, Option<usize>)> {
|
||||
unsafe {
|
||||
//assert address_cells == 0x1, size_cells == 0x1
|
||||
let mem = node.reg().unwrap().nth(0).unwrap();
|
||||
let base = get_mmio_address(fdt, node, &mem).unwrap();
|
||||
let size = mem.size.unwrap() as u32;
|
||||
let mut ret_virq = None;
|
||||
|
||||
if let Some(interrupt_parent) = node.property("interrupt-parent") {
|
||||
let phandle = interrupt_parent.as_usize().unwrap() as u32;
|
||||
let irq = get_interrupt(fdt, node, 0).unwrap();
|
||||
let ic_idx = IRQ_CHIP.phandle_to_ic_idx(phandle).unwrap();
|
||||
//PHYS_NONSECURE_PPI only
|
||||
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
|
||||
.ic
|
||||
.irq_xlate(irq)
|
||||
.unwrap();
|
||||
info!(
|
||||
"register bcm2835arm_ctrl as ic_idx {}'s child virq = {}",
|
||||
ic_idx, virq
|
||||
);
|
||||
ret_virq = Some(virq);
|
||||
}
|
||||
Ok((base as usize, size as usize, ret_virq))
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn init(&mut self) {
|
||||
unsafe {
|
||||
debug!("IRQ BCM2835 INIT");
|
||||
//disable all interrupt
|
||||
self.write(DISABLE_0, 0xffff_ffff);
|
||||
self.write(DISABLE_1, 0xffff_ffff);
|
||||
self.write(DISABLE_2, 0xffff_ffff);
|
||||
|
||||
debug!("IRQ BCM2835 END");
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn read(&self, reg: u32) -> u32 {
|
||||
unsafe {
|
||||
let val = read_volatile((self.address + reg as usize) as *const u32);
|
||||
val
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn write(&mut self, reg: u32, value: u32) {
|
||||
unsafe {
|
||||
write_volatile((self.address + reg as usize) as *mut u32, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptController for Bcm2835ArmInterruptController {
|
||||
fn irq_init(
|
||||
&mut self,
|
||||
fdt_opt: Option<&Fdt>,
|
||||
irq_desc: &mut [IrqDesc; 1024],
|
||||
ic_idx: usize,
|
||||
irq_idx: &mut usize,
|
||||
) -> Result<()> {
|
||||
let (base, _size, _virq) = match Bcm2835ArmInterruptController::parse(fdt_opt.unwrap()) {
|
||||
Ok((a, b, c)) => (a, b, c),
|
||||
Err(_) => return Err(Error::new(EINVAL)),
|
||||
};
|
||||
unsafe {
|
||||
self.address = base + crate::PHYS_OFFSET;
|
||||
|
||||
self.init();
|
||||
let idx = *irq_idx;
|
||||
let cnt = 3 << 5; //3 * 32 irqs, basic == 8, reg1 = 32, reg2 = 32
|
||||
let mut i: usize = 0;
|
||||
//only support linear irq map now.
|
||||
while i < cnt && (idx + i < 1024) {
|
||||
irq_desc[idx + i].basic.ic_idx = ic_idx;
|
||||
irq_desc[idx + i].basic.ic_irq = i as u32;
|
||||
irq_desc[idx + i].basic.used = true;
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
info!("bcm2835 irq_range = ({}, {})", idx, idx + cnt);
|
||||
self.irq_range = (idx, idx + cnt);
|
||||
*irq_idx = idx + cnt;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn irq_ack(&mut self) -> u32 {
|
||||
//TODO: support smp self.read(LOCAL_IRQ_PENDING + 4 * cpu)
|
||||
let sources = unsafe { self.read(PENDING_0) };
|
||||
let pending_num = ffs(sources) - 1;
|
||||
let fast_irq = [
|
||||
7 + 32,
|
||||
9 + 32,
|
||||
10 + 32,
|
||||
18 + 32,
|
||||
19 + 32,
|
||||
21 + 64,
|
||||
22 + 64,
|
||||
23 + 64,
|
||||
24 + 64,
|
||||
25 + 64,
|
||||
30 + 64,
|
||||
];
|
||||
|
||||
//fast irq
|
||||
if pending_num >= 10 && pending_num <= 20 {
|
||||
return fast_irq[(pending_num - 10) as usize];
|
||||
}
|
||||
|
||||
let pending_num = ffs(sources & 0x3ff) - 1;
|
||||
match pending_num {
|
||||
num @ 0..=7 => return num,
|
||||
8 => {
|
||||
let sources1 = unsafe { self.read(PENDING_1) };
|
||||
let irq_0_31 = ffs(sources1) - 1;
|
||||
return irq_0_31 + 32;
|
||||
}
|
||||
9 => {
|
||||
let sources2 = unsafe { self.read(PENDING_2) };
|
||||
let irq_32_63 = ffs(sources2) - 1;
|
||||
return irq_32_63 + 64;
|
||||
}
|
||||
num => {
|
||||
error!(
|
||||
"unexpected irq pending in BASIC PENDING: 0x{}, sources = 0x{:08x}",
|
||||
num, sources
|
||||
);
|
||||
return num;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn irq_eoi(&mut self, _irq_num: u32) {}
|
||||
|
||||
fn irq_enable(&mut self, irq_num: u32) {
|
||||
debug!("bcm2835 enable {} {}", irq_num, irq_num & 0x1f);
|
||||
match irq_num {
|
||||
num @ 0..=31 => {
|
||||
let val = 1 << num;
|
||||
unsafe {
|
||||
self.write(ENABLE_0, val);
|
||||
}
|
||||
}
|
||||
num @ 32..=63 => {
|
||||
let val = 1 << (num & 0x1f);
|
||||
unsafe {
|
||||
self.write(ENABLE_1, val);
|
||||
}
|
||||
}
|
||||
num @ 64..=95 => {
|
||||
let val = 1 << (num & 0x1f);
|
||||
unsafe {
|
||||
self.write(ENABLE_2, val);
|
||||
}
|
||||
}
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
|
||||
fn irq_disable(&mut self, irq_num: u32) {
|
||||
match irq_num {
|
||||
num @ 0..=31 => {
|
||||
let val = 1 << num;
|
||||
unsafe {
|
||||
self.write(DISABLE_0, val);
|
||||
}
|
||||
}
|
||||
num @ 32..=63 => {
|
||||
let val = 1 << (num & 0x1f);
|
||||
unsafe {
|
||||
self.write(DISABLE_1, val);
|
||||
}
|
||||
}
|
||||
num @ 64..=95 => {
|
||||
let val = 1 << (num & 0x1f);
|
||||
unsafe {
|
||||
self.write(DISABLE_2, val);
|
||||
}
|
||||
}
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
|
||||
//assert interrupt-cells == 0x2
|
||||
match irq_data {
|
||||
IrqCell::L2(bank, irq) => {
|
||||
//TODO: check bank && irq
|
||||
let hwirq = (bank as usize) << 5 | (irq as usize);
|
||||
let off = hwirq + self.irq_range.0;
|
||||
Ok(off)
|
||||
}
|
||||
_ => Err(Error::new(EINVAL)),
|
||||
}
|
||||
}
|
||||
|
||||
fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
|
||||
if hwirq > 95 {
|
||||
None
|
||||
} else {
|
||||
Some(self.irq_range.0 + hwirq as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptHandler for Bcm2835ArmInterruptController {
|
||||
fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {
|
||||
unsafe {
|
||||
let irq = self.irq_ack();
|
||||
if let Some(virq) = self.irq_to_virq(irq)
|
||||
&& virq < 1024
|
||||
{
|
||||
if let Some(handler) = &mut IRQ_CHIP.irq_desc[virq].handler {
|
||||
handler.irq_handler(virq as u32, token);
|
||||
}
|
||||
} else {
|
||||
error!("unexpected irq num {}", irq);
|
||||
}
|
||||
self.irq_eoi(irq);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,231 +0,0 @@
|
||||
use super::InterruptController;
|
||||
use crate::{
|
||||
arch::device::{ROOT_IC_IDX, ROOT_IC_IDX_IS_SET},
|
||||
dtb::{
|
||||
get_mmio_address,
|
||||
irqchip::{InterruptHandler, IrqCell, IrqDesc},
|
||||
},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
use core::{
|
||||
arch::asm,
|
||||
ptr::{read_volatile, write_volatile},
|
||||
sync::atomic::Ordering,
|
||||
};
|
||||
use fdt::{node::FdtNode, Fdt};
|
||||
use syscall::{
|
||||
error::{Error, EINVAL},
|
||||
Result,
|
||||
};
|
||||
|
||||
const LOCAL_CONTROL: u32 = 0x000;
|
||||
const LOCAL_PRESCALER: u32 = 0x008;
|
||||
const LOCAL_GPU_ROUTING: u32 = 0x00C;
|
||||
const LOCAL_TIMER_INT_CONTROL0: u32 = 0x040;
|
||||
const LOCAL_IRQ_PENDING: u32 = 0x060;
|
||||
|
||||
const LOCAL_IRQ_CNTPNSIRQ: u32 = 0x1;
|
||||
const LOCAL_IRQ_GPU_FAST: u32 = 0x8;
|
||||
const LOCAL_IRQ_PMU_FAST: u32 = 0x9;
|
||||
const LOCAL_IRQ_LAST: u32 = LOCAL_IRQ_PMU_FAST;
|
||||
|
||||
#[inline(always)]
|
||||
fn ffs(num: u32) -> u32 {
|
||||
let mut x = num;
|
||||
if x == 0 {
|
||||
return 0;
|
||||
}
|
||||
let mut r = 1;
|
||||
if (x & 0xffff) == 0 {
|
||||
x >>= 16;
|
||||
r += 16;
|
||||
}
|
||||
if (x & 0xff) == 0 {
|
||||
x >>= 8;
|
||||
r += 8;
|
||||
}
|
||||
if (x & 0xf) == 0 {
|
||||
x >>= 4;
|
||||
r += 4;
|
||||
}
|
||||
if (x & 0x3) == 0 {
|
||||
x >>= 2;
|
||||
r += 2;
|
||||
}
|
||||
if (x & 0x1) == 0 {
|
||||
r += 1;
|
||||
}
|
||||
|
||||
r
|
||||
}
|
||||
|
||||
pub struct Bcm2836ArmInterruptController {
|
||||
pub address: usize,
|
||||
pub irq_range: (usize, usize),
|
||||
pub active_cpu: u32,
|
||||
}
|
||||
|
||||
impl Bcm2836ArmInterruptController {
|
||||
pub fn new() -> Self {
|
||||
Bcm2836ArmInterruptController {
|
||||
address: 0,
|
||||
irq_range: (0, 0),
|
||||
active_cpu: 0,
|
||||
}
|
||||
}
|
||||
pub fn parse(fdt: &Fdt) -> Result<(usize, usize)> {
|
||||
if let Some(node) = fdt.find_compatible(&["brcm,bcm2836-l1-intc"]) {
|
||||
return Bcm2836ArmInterruptController::parse_inner(fdt, &node);
|
||||
} else {
|
||||
return Err(Error::new(EINVAL));
|
||||
}
|
||||
}
|
||||
fn parse_inner(fdt: &Fdt, node: &FdtNode) -> Result<(usize, usize)> {
|
||||
//assert address_cells == 0x1, size_cells == 0x1
|
||||
let reg = node.reg().unwrap().nth(0).unwrap();
|
||||
let addr = get_mmio_address(fdt, node, ®).unwrap();
|
||||
|
||||
Ok((addr, reg.size.unwrap()))
|
||||
}
|
||||
|
||||
unsafe fn init(&mut self) {
|
||||
unsafe {
|
||||
debug!("IRQ BCM2836 INIT");
|
||||
//init local timer freq
|
||||
self.write(LOCAL_CONTROL, 0x0);
|
||||
self.write(LOCAL_PRESCALER, 0x8000_0000);
|
||||
|
||||
//routing all irq to core
|
||||
self.write(LOCAL_GPU_ROUTING, self.active_cpu);
|
||||
debug!("routing all irq to core {}", self.active_cpu);
|
||||
debug!("IRQ BCM2836 END");
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn read(&self, reg: u32) -> u32 {
|
||||
unsafe {
|
||||
let val = read_volatile((self.address + reg as usize) as *const u32);
|
||||
val
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn write(&mut self, reg: u32, value: u32) {
|
||||
unsafe {
|
||||
write_volatile((self.address + reg as usize) as *mut u32, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InterruptHandler for Bcm2836ArmInterruptController {
|
||||
fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
|
||||
}
|
||||
|
||||
impl InterruptController for Bcm2836ArmInterruptController {
|
||||
fn irq_init(
|
||||
&mut self,
|
||||
fdt_opt: Option<&Fdt>,
|
||||
irq_desc: &mut [IrqDesc; 1024],
|
||||
ic_idx: usize,
|
||||
irq_idx: &mut usize,
|
||||
) -> Result<()> {
|
||||
let (base, _size) = match Bcm2836ArmInterruptController::parse(fdt_opt.unwrap()) {
|
||||
Ok((a, b)) => (a, b),
|
||||
Err(_) => return Err(Error::new(EINVAL)),
|
||||
};
|
||||
unsafe {
|
||||
self.address = base + crate::PHYS_OFFSET;
|
||||
let cpuid: usize;
|
||||
asm!("mrs {}, mpidr_el1", out(reg) cpuid);
|
||||
self.active_cpu = cpuid as u32 & 0x3;
|
||||
|
||||
self.init();
|
||||
let idx = *irq_idx;
|
||||
let cnt = LOCAL_IRQ_LAST as usize;
|
||||
let mut i: usize = 0;
|
||||
//only support linear irq map now.
|
||||
while i < cnt && (idx + i < 1024) {
|
||||
irq_desc[idx + i].basic.ic_idx = ic_idx;
|
||||
irq_desc[idx + i].basic.ic_irq = i as u32;
|
||||
irq_desc[idx + i].basic.used = true;
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
info!("bcm2836 irq_range = ({}, {})", idx, idx + cnt);
|
||||
self.irq_range = (idx, idx + cnt);
|
||||
*irq_idx = idx + cnt;
|
||||
}
|
||||
|
||||
//raspi 3b+ dts doesn't follow the rule to set root parent interrupt controller
|
||||
//so we should set it manually.
|
||||
ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
|
||||
ROOT_IC_IDX_IS_SET.store(1, Ordering::Relaxed);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn irq_ack(&mut self) -> u32 {
|
||||
let cpuid: usize;
|
||||
unsafe {
|
||||
asm!("mrs {}, mpidr_el1", out(reg) cpuid);
|
||||
}
|
||||
let cpu = cpuid as u32 & 0x3;
|
||||
let sources: u32 = unsafe { self.read(LOCAL_IRQ_PENDING + 4 * cpu) };
|
||||
ffs(sources) - 1
|
||||
}
|
||||
|
||||
fn irq_eoi(&mut self, _irq_num: u32) {}
|
||||
|
||||
fn irq_enable(&mut self, irq_num: u32) {
|
||||
debug!("bcm2836 enable {}", irq_num);
|
||||
match irq_num {
|
||||
LOCAL_IRQ_CNTPNSIRQ => unsafe {
|
||||
let cpuid: usize;
|
||||
asm!("mrs {}, mpidr_el1", out(reg) cpuid);
|
||||
let cpu = cpuid as u32 & 0x3;
|
||||
let mut reg_val = self.read(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu);
|
||||
reg_val |= 0x2;
|
||||
self.write(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu, reg_val);
|
||||
},
|
||||
LOCAL_IRQ_GPU_FAST => {
|
||||
//GPU IRQ always enable
|
||||
}
|
||||
_ => {
|
||||
//ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn irq_disable(&mut self, irq_num: u32) {
|
||||
match irq_num {
|
||||
LOCAL_IRQ_CNTPNSIRQ => unsafe {
|
||||
let cpuid: usize;
|
||||
asm!("mrs {}, mpidr_el1", out(reg) cpuid);
|
||||
let cpu = cpuid as u32 & 0x3;
|
||||
let mut reg_val = self.read(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu);
|
||||
reg_val &= !0x2;
|
||||
self.write(LOCAL_TIMER_INT_CONTROL0 + 4 * cpu, reg_val);
|
||||
},
|
||||
LOCAL_IRQ_GPU_FAST => {
|
||||
//GPU IRQ always enable
|
||||
}
|
||||
_ => {
|
||||
//ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
fn irq_xlate(&self, irq_data: IrqCell) -> Result<usize> {
|
||||
//assert interrupt-cells == 0x2
|
||||
match irq_data {
|
||||
IrqCell::L2(irq, _) => Ok(irq as usize + self.irq_range.0),
|
||||
_ => Err(Error::new(EINVAL)),
|
||||
}
|
||||
}
|
||||
fn irq_to_virq(&self, hwirq: u32) -> Option<usize> {
|
||||
if hwirq > LOCAL_IRQ_LAST {
|
||||
None
|
||||
} else {
|
||||
Some(self.irq_range.0 + hwirq as usize)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
use crate::dtb::irqchip::{InterruptController, IRQ_CHIP};
|
||||
use alloc::boxed::Box;
|
||||
use fdt::{node::FdtNode, Fdt};
|
||||
|
||||
pub(crate) mod gic;
|
||||
pub(crate) mod gicv3;
|
||||
mod irq_bcm2835;
|
||||
mod irq_bcm2836;
|
||||
mod null;
|
||||
|
||||
pub(crate) fn new_irqchip(ic_str: &str) -> Option<Box<dyn InterruptController>> {
|
||||
if ic_str.contains("arm,gic-v3") {
|
||||
Some(Box::new(gicv3::GicV3::new()))
|
||||
} else if ic_str.contains("arm,cortex-a15-gic") || ic_str.contains("arm,gic-400") {
|
||||
Some(Box::new(gic::GenericInterruptController::new()))
|
||||
} else if ic_str.contains("brcm,bcm2836-l1-intc") {
|
||||
Some(Box::new(irq_bcm2836::Bcm2836ArmInterruptController::new()))
|
||||
} else if ic_str.contains("brcm,bcm2836-armctrl-ic") {
|
||||
Some(Box::new(irq_bcm2835::Bcm2835ArmInterruptController::new()))
|
||||
} else {
|
||||
warn!("no driver for interrupt controller {:?}", ic_str);
|
||||
//TODO: return None and handle it properly
|
||||
Some(Box::new(null::Null))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn ic_for_chip(fdt: &Fdt, node: &FdtNode) -> Option<usize> {
|
||||
if let Some(_) = node.property("interrupts-extended") {
|
||||
error!("multi-parented device not supported");
|
||||
None
|
||||
} else if let Some(irqc_phandle) = node
|
||||
.property("interrupt-parent")
|
||||
.or(fdt.root().property("interrupt-parent"))
|
||||
.and_then(|f| f.as_usize())
|
||||
{
|
||||
unsafe { IRQ_CHIP.phandle_to_ic_idx(irqc_phandle as u32) }
|
||||
} else {
|
||||
error!("no irq parent found");
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
use fdt::Fdt;
|
||||
use syscall::{
|
||||
error::{Error, EINVAL},
|
||||
Result,
|
||||
};
|
||||
|
||||
use super::InterruptController;
|
||||
use crate::{
|
||||
dtb::irqchip::{InterruptHandler, IrqCell, IrqDesc},
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
|
||||
pub struct Null;
|
||||
|
||||
impl InterruptHandler for Null {
|
||||
fn irq_handler(&mut self, _irq: u32, token: &mut CleanLockToken) {}
|
||||
}
|
||||
|
||||
impl InterruptController for Null {
|
||||
fn irq_init(
|
||||
&mut self,
|
||||
_fdt_opt: Option<&Fdt>,
|
||||
_irq_desc: &mut [IrqDesc; 1024],
|
||||
_ic_idx: usize,
|
||||
_irq_idx: &mut usize,
|
||||
) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn irq_ack(&mut self) -> u32 {
|
||||
unimplemented!()
|
||||
}
|
||||
fn irq_eoi(&mut self, _irq_num: u32) {}
|
||||
fn irq_enable(&mut self, _irq_num: u32) {}
|
||||
fn irq_disable(&mut self, _irq_num: u32) {}
|
||||
fn irq_xlate(&self, _irq_data: IrqCell) -> Result<usize> {
|
||||
Err(Error::new(EINVAL))
|
||||
}
|
||||
fn irq_to_virq(&self, _hwirq: u32) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -1,60 +0,0 @@
|
||||
use crate::info;
|
||||
use core::sync::atomic::{AtomicUsize, Ordering};
|
||||
use fdt::Fdt;
|
||||
|
||||
pub mod cpu;
|
||||
pub mod generic_timer;
|
||||
pub mod irqchip;
|
||||
pub mod rtc;
|
||||
pub mod serial;
|
||||
|
||||
use crate::dtb::irqchip::IRQ_CHIP;
|
||||
use irqchip::ic_for_chip;
|
||||
|
||||
pub static ROOT_IC_IDX: AtomicUsize = AtomicUsize::new(0);
|
||||
pub static ROOT_IC_IDX_IS_SET: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
unsafe fn init_root_ic(fdt: &Fdt) {
|
||||
unsafe {
|
||||
let is_set = ROOT_IC_IDX_IS_SET.load(Ordering::Relaxed);
|
||||
if is_set != 0 {
|
||||
let ic_idx = ROOT_IC_IDX.load(Ordering::Relaxed);
|
||||
info!("Already selected {} as root ic", ic_idx);
|
||||
return;
|
||||
}
|
||||
|
||||
let root_irqc_phandle = fdt
|
||||
.root()
|
||||
.property("interrupt-parent")
|
||||
.unwrap()
|
||||
.as_usize()
|
||||
.unwrap();
|
||||
let ic_idx = IRQ_CHIP
|
||||
.phandle_to_ic_idx(root_irqc_phandle as u32)
|
||||
.unwrap();
|
||||
info!("select {} as root ic", ic_idx);
|
||||
ROOT_IC_IDX.store(ic_idx, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn init_devicetree(fdt: &Fdt) {
|
||||
unsafe {
|
||||
info!("IRQCHIP INIT");
|
||||
crate::dtb::irqchip::init(&fdt);
|
||||
init_root_ic(&fdt);
|
||||
info!("GIT INIT");
|
||||
generic_timer::init(fdt);
|
||||
info!("SERIAL INIT");
|
||||
serial::init(fdt);
|
||||
info!("RTC INIT");
|
||||
rtc::init(fdt);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArchPercpuMisc;
|
||||
|
||||
impl ArchPercpuMisc {
|
||||
pub const fn default() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
use crate::{dtb::get_mmio_address, sync::CleanLockToken, time};
|
||||
use core::ptr::read_volatile;
|
||||
|
||||
static RTC_DR: usize = 0x000;
|
||||
|
||||
pub unsafe fn init(fdt: &fdt::Fdt) {
|
||||
if let Some(node) = fdt.find_compatible(&["arm,pl031"]) {
|
||||
match node
|
||||
.reg()
|
||||
.and_then(|mut iter| iter.next())
|
||||
.and_then(|region| get_mmio_address(fdt, &node, ®ion))
|
||||
{
|
||||
Some(phys) => {
|
||||
let mut rtc = Pl031rtc { phys };
|
||||
info!("PL031 RTC at {:#x}", rtc.phys);
|
||||
let mut token = unsafe { CleanLockToken::new() };
|
||||
*time::START.lock(token.token()) = (rtc.time() as u128) * time::NANOS_PER_SEC;
|
||||
}
|
||||
None => {
|
||||
warn!("No PL031 RTC registers");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("No PL031 RTC found");
|
||||
}
|
||||
}
|
||||
|
||||
struct Pl031rtc {
|
||||
pub phys: usize,
|
||||
}
|
||||
|
||||
impl Pl031rtc {
|
||||
unsafe fn read(&self, reg: usize) -> u32 {
|
||||
unsafe { read_volatile((crate::PHYS_OFFSET + self.phys + reg) as *const u32) }
|
||||
}
|
||||
|
||||
pub fn time(&mut self) -> u64 {
|
||||
let seconds = unsafe { self.read(RTC_DR) } as u64;
|
||||
seconds
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
use alloc::boxed::Box;
|
||||
use fdt::Fdt;
|
||||
|
||||
pub use crate::dtb::serial::COM1;
|
||||
use crate::{
|
||||
arch::device::irqchip::ic_for_chip,
|
||||
dtb::{
|
||||
get_interrupt,
|
||||
irqchip::{register_irq, InterruptHandler, IRQ_CHIP},
|
||||
},
|
||||
scheme::irq::irq_trigger,
|
||||
sync::CleanLockToken,
|
||||
};
|
||||
|
||||
pub struct Com1Irq {}
|
||||
|
||||
impl InterruptHandler for Com1Irq {
|
||||
fn irq_handler(&mut self, irq: u32, token: &mut CleanLockToken) {
|
||||
COM1.lock().receive(token);
|
||||
unsafe {
|
||||
// FIXME add_irq accepts a u8 as irq number
|
||||
// PercpuBlock::current().stats.add_irq(irq);
|
||||
irq_trigger(irq.try_into().unwrap(), token);
|
||||
IRQ_CHIP.irq_eoi(irq);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn init(fdt: &Fdt) {
|
||||
unsafe {
|
||||
//TODO: find actual serial device, not just any PL011
|
||||
if let Some(node) = fdt.find_compatible(&["arm,pl011"]) {
|
||||
let irq = get_interrupt(fdt, &node, 0).unwrap();
|
||||
if let Some(ic_idx) = ic_for_chip(&fdt, &node) {
|
||||
let virq = IRQ_CHIP.irq_chip_list.chips[ic_idx]
|
||||
.ic
|
||||
.irq_xlate(irq)
|
||||
.unwrap();
|
||||
info!("serial_port virq = {}", virq);
|
||||
register_irq(virq as u32, Box::new(Com1Irq {}));
|
||||
IRQ_CHIP.irq_enable(virq as u32);
|
||||
} else {
|
||||
error!("serial port irq parent not found");
|
||||
}
|
||||
}
|
||||
COM1.lock().enable_irq();
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn init_acpi(irq: u32) {
|
||||
unsafe {
|
||||
//TODO: what should chip index be?
|
||||
let virq = IRQ_CHIP.irq_chip_list.chips[0].ic.irq_to_virq(irq).unwrap();
|
||||
info!("serial_port virq = {}", virq);
|
||||
register_irq(virq as u32, Box::new(Com1Irq {}));
|
||||
IRQ_CHIP.irq_enable(virq as u32);
|
||||
COM1.lock().enable_irq();
|
||||
}
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
use ::syscall::Exception;
|
||||
use rmm::VirtualAddress;
|
||||
|
||||
use crate::{
|
||||
context::signal::excp_handler,
|
||||
exception_stack,
|
||||
memory::{ArchIntCtx, GenericPfFlags},
|
||||
sync::CleanLockToken,
|
||||
syscall,
|
||||
};
|
||||
|
||||
use super::InterruptStack;
|
||||
|
||||
exception_stack!(synchronous_exception_at_el1_with_sp0, |stack| {
|
||||
println!("Synchronous exception at EL1 with SP0");
|
||||
stack.trace();
|
||||
loop {}
|
||||
});
|
||||
|
||||
fn exception_code(esr: usize) -> u8 {
|
||||
((esr >> 26) & 0x3f) as u8
|
||||
}
|
||||
fn iss(esr: usize) -> u32 {
|
||||
(esr & 0x01ff_ffff) as u32
|
||||
}
|
||||
|
||||
unsafe fn far_el1() -> usize {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
core::arch::asm!("mrs {}, far_el1", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn instr_data_abort_inner(
|
||||
stack: &mut InterruptStack,
|
||||
from_user: bool,
|
||||
instr_not_data: bool,
|
||||
_from: &str,
|
||||
) -> bool {
|
||||
unsafe {
|
||||
let iss = iss(stack.iret.esr_el1);
|
||||
let fsc = iss & 0x3F;
|
||||
//dbg!(fsc);
|
||||
|
||||
let was_translation_fault = fsc >= 0b000100 && fsc <= 0b000111;
|
||||
//let was_permission_fault = fsc >= 0b001101 && fsc <= 0b001111;
|
||||
let write_not_read_if_data = iss & (1 << 6) != 0;
|
||||
|
||||
let mut flags = GenericPfFlags::empty();
|
||||
flags.set(GenericPfFlags::PRESENT, !was_translation_fault);
|
||||
|
||||
// TODO: RMW instructions may "involve" writing to (possibly invalid) memory, but AArch64
|
||||
// doesn't appear to require that flag to be set if the read alone would trigger a fault.
|
||||
flags.set(
|
||||
GenericPfFlags::INVOLVED_WRITE,
|
||||
write_not_read_if_data && !instr_not_data,
|
||||
);
|
||||
flags.set(GenericPfFlags::INSTR_NOT_DATA, instr_not_data);
|
||||
flags.set(GenericPfFlags::USER_NOT_SUPERVISOR, from_user);
|
||||
|
||||
let faulting_addr = VirtualAddress::new(far_el1());
|
||||
//dbg!(faulting_addr, flags, from);
|
||||
|
||||
crate::memory::page_fault_handler(stack, flags, faulting_addr).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn cntfrq_el0() -> usize {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
core::arch::asm!("mrs {}, cntfrq_el0", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn cntpct_el0() -> usize {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
core::arch::asm!("mrs {}, cntpct_el0", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn cntvct_el0() -> usize {
|
||||
unsafe {
|
||||
let ret: usize;
|
||||
core::arch::asm!("mrs {}, cntvct_el0", out(reg) ret);
|
||||
ret
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn instr_trapped_msr_mrs_inner(
|
||||
stack: &mut InterruptStack,
|
||||
_from_user: bool,
|
||||
_instr_not_data: bool,
|
||||
_from: &str,
|
||||
) -> bool {
|
||||
unsafe {
|
||||
let iss = iss(stack.iret.esr_el1);
|
||||
// let res0 = (iss & 0x1C0_0000) >> 22;
|
||||
let op0 = (iss & 0x030_0000) >> 20;
|
||||
let op2 = (iss & 0x00e_0000) >> 17;
|
||||
let op1 = (iss & 0x001_c000) >> 14;
|
||||
let crn = (iss & 0x000_3c00) >> 10;
|
||||
let rt = (iss & 0x000_03e0) >> 5;
|
||||
let crm = (iss & 0x000_001e) >> 1;
|
||||
let dir = iss & 0x000_0001;
|
||||
|
||||
/*
|
||||
print!("iss=0x{:x}, res0=0b{:03b}, op0=0b{:02b}\n
|
||||
op2=0b{:03b}, op1=0b{:03b}, crn=0b{:04b}\n
|
||||
rt=0b{:05b}, crm=0b{:04b}, dir=0b{:b}\n",
|
||||
iss, res0, op0, op2, op1, crn, rt, crm, dir);
|
||||
*/
|
||||
|
||||
match (op0, op1, crn, crm, op2, dir) {
|
||||
//MRS <Xt>, CNTFRQ_EL0
|
||||
(0b11, 0b011, 0b1110, 0b0000, 0b000, 0b1) => {
|
||||
let reg_val = cntfrq_el0();
|
||||
stack.store_reg(rt as usize, reg_val);
|
||||
//skip faulting instruction, A64 instructions are always 32-bits
|
||||
stack.iret.elr_el1 += 4;
|
||||
return true;
|
||||
}
|
||||
//MRS <Xt>, CNTPCT_EL0
|
||||
(0b11, 0b011, 0b1110, 0b0000, 0b001, 0b1) => {
|
||||
let reg_val = cntpct_el0();
|
||||
stack.store_reg(rt as usize, reg_val);
|
||||
//skip faulting instruction, A64 instructions are always 32-bits
|
||||
stack.iret.elr_el1 += 4;
|
||||
return true;
|
||||
}
|
||||
//MRS <Xt>, CNTVCT_EL0
|
||||
(0b11, 0b011, 0b1110, 0b0000, 0b010, 0b1) => {
|
||||
let reg_val = cntvct_el0();
|
||||
stack.store_reg(rt as usize, reg_val);
|
||||
//skip faulting instruction, A64 instructions are always 32-bits
|
||||
stack.iret.elr_el1 += 4;
|
||||
return true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
exception_stack!(synchronous_exception_at_el1_with_spx, |stack| {
|
||||
unsafe {
|
||||
if !pf_inner(
|
||||
stack,
|
||||
exception_code(stack.iret.esr_el1),
|
||||
"sync_exc_el1_spx",
|
||||
) {
|
||||
println!("Synchronous exception at EL1 with SPx");
|
||||
if exception_code(stack.iret.esr_el1) == 0b100101 {
|
||||
let far_el1 = far_el1();
|
||||
println!("FAR_EL1 = 0x{:08x}", far_el1);
|
||||
} else if exception_code(stack.iret.esr_el1) == 0b100100 {
|
||||
let far_el1 = far_el1();
|
||||
println!("USER FAR_EL1 = 0x{:08x}", far_el1);
|
||||
}
|
||||
stack.trace();
|
||||
loop {}
|
||||
}
|
||||
}
|
||||
});
|
||||
unsafe fn pf_inner(stack: &mut InterruptStack, ty: u8, from: &str) -> bool {
|
||||
unsafe {
|
||||
match ty {
|
||||
// "Data Abort taken from a lower Exception level"
|
||||
0b100100 => instr_data_abort_inner(stack, true, false, from),
|
||||
// "Data Abort taken without a change in Exception level"
|
||||
0b100101 => instr_data_abort_inner(stack, false, false, from),
|
||||
// "Instruction Abort taken from a lower Exception level"
|
||||
0b100000 => instr_data_abort_inner(stack, true, true, from),
|
||||
// "Instruction Abort taken without a change in Exception level"
|
||||
0b100001 => instr_data_abort_inner(stack, false, true, from),
|
||||
// "Trapped MSR, MRS or System instruction execution in AArch64 state"
|
||||
0b011000 => instr_trapped_msr_mrs_inner(stack, true, true, from),
|
||||
|
||||
_ => return false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exception_stack!(synchronous_exception_at_el0, |stack| {
|
||||
unsafe {
|
||||
match exception_code(stack.iret.esr_el1) {
|
||||
0b010101 => {
|
||||
let scratch = &stack.scratch;
|
||||
let mut token = CleanLockToken::new();
|
||||
let ret = syscall::syscall(
|
||||
scratch.x8, scratch.x0, scratch.x1, scratch.x2, scratch.x3, scratch.x4,
|
||||
scratch.x5, &mut token,
|
||||
);
|
||||
stack.scratch.x0 = ret;
|
||||
}
|
||||
|
||||
ty => {
|
||||
if !pf_inner(stack, ty as u8, "sync_exc_el0") {
|
||||
error!(
|
||||
"FATAL: Not an SVC induced synchronous exception (ty={:b})",
|
||||
ty
|
||||
);
|
||||
println!("FAR_EL1: {:#0x}", far_el1());
|
||||
//crate::debugger::debugger(None);
|
||||
stack.trace();
|
||||
excp_handler(Exception {
|
||||
kind: 0, // TODO
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
exception_stack!(unhandled_exception, |stack| {
|
||||
println!("Unhandled exception");
|
||||
stack.trace();
|
||||
loop {}
|
||||
});
|
||||
|
||||
impl ArchIntCtx for InterruptStack {
|
||||
fn ip(&self) -> usize {
|
||||
self.iret.elr_el1
|
||||
}
|
||||
fn recover_and_efault(&mut self) {
|
||||
// Set the return value to nonzero to indicate usercopy failure (EFAULT), and emulate the
|
||||
// return instruction by setting the return pointer to the saved LR value.
|
||||
|
||||
self.iret.elr_el1 = self.preserved.x30;
|
||||
self.scratch.x0 = 1;
|
||||
}
|
||||
}
|
||||
@@ -1,420 +0,0 @@
|
||||
use crate::{panic, syscall::IntRegisters};
|
||||
|
||||
#[derive(Default)]
|
||||
#[repr(C, packed)]
|
||||
pub struct ScratchRegisters {
|
||||
pub x0: usize,
|
||||
pub x1: usize,
|
||||
pub x2: usize,
|
||||
pub x3: usize,
|
||||
pub x4: usize,
|
||||
pub x5: usize,
|
||||
pub x6: usize,
|
||||
pub x7: usize,
|
||||
pub x8: usize,
|
||||
pub x9: usize,
|
||||
pub x10: usize,
|
||||
pub x11: usize,
|
||||
pub x12: usize,
|
||||
pub x13: usize,
|
||||
pub x14: usize,
|
||||
pub x15: usize,
|
||||
pub x16: usize,
|
||||
pub x17: usize,
|
||||
pub x18: usize,
|
||||
pub _padding: usize,
|
||||
}
|
||||
|
||||
impl ScratchRegisters {
|
||||
pub fn dump(&self) {
|
||||
println!("X0: {:>016X}", { self.x0 });
|
||||
println!("X1: {:>016X}", { self.x1 });
|
||||
println!("X2: {:>016X}", { self.x2 });
|
||||
println!("X3: {:>016X}", { self.x3 });
|
||||
println!("X4: {:>016X}", { self.x4 });
|
||||
println!("X5: {:>016X}", { self.x5 });
|
||||
println!("X6: {:>016X}", { self.x6 });
|
||||
println!("X7: {:>016X}", { self.x7 });
|
||||
println!("X8: {:>016X}", { self.x8 });
|
||||
println!("X9: {:>016X}", { self.x9 });
|
||||
println!("X10: {:>016X}", { self.x10 });
|
||||
println!("X11: {:>016X}", { self.x11 });
|
||||
println!("X12: {:>016X}", { self.x12 });
|
||||
println!("X13: {:>016X}", { self.x13 });
|
||||
println!("X14: {:>016X}", { self.x14 });
|
||||
println!("X15: {:>016X}", { self.x15 });
|
||||
println!("X16: {:>016X}", { self.x16 });
|
||||
println!("X17: {:>016X}", { self.x17 });
|
||||
println!("X18: {:>016X}", { self.x18 });
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[repr(C, packed)]
|
||||
pub struct PreservedRegisters {
|
||||
//TODO: is X30 a preserved register?
|
||||
pub x19: usize,
|
||||
pub x20: usize,
|
||||
pub x21: usize,
|
||||
pub x22: usize,
|
||||
pub x23: usize,
|
||||
pub x24: usize,
|
||||
pub x25: usize,
|
||||
pub x26: usize,
|
||||
pub x27: usize,
|
||||
pub x28: usize,
|
||||
pub x29: usize,
|
||||
pub x30: usize,
|
||||
}
|
||||
|
||||
impl PreservedRegisters {
|
||||
pub fn dump(&self) {
|
||||
println!("X19: {:>016X}", { self.x19 });
|
||||
println!("X20: {:>016X}", { self.x20 });
|
||||
println!("X21: {:>016X}", { self.x21 });
|
||||
println!("X22: {:>016X}", { self.x22 });
|
||||
println!("X23: {:>016X}", { self.x23 });
|
||||
println!("X24: {:>016X}", { self.x24 });
|
||||
println!("X25: {:>016X}", { self.x25 });
|
||||
println!("X26: {:>016X}", { self.x26 });
|
||||
println!("X27: {:>016X}", { self.x27 });
|
||||
println!("X28: {:>016X}", { self.x28 });
|
||||
println!("X29: {:>016X}", { self.x29 });
|
||||
println!("X30: {:>016X}", { self.x30 });
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[repr(C, packed)]
|
||||
pub struct IretRegisters {
|
||||
// occurred
|
||||
// The exception vector disambiguates at which EL the interrupt
|
||||
pub sp_el0: usize, // Shouldn't be used if interrupt occurred at EL1
|
||||
pub esr_el1: usize,
|
||||
pub spsr_el1: usize,
|
||||
pub elr_el1: usize,
|
||||
}
|
||||
|
||||
impl IretRegisters {
|
||||
pub fn dump(&self) {
|
||||
println!("ELR_EL1: {:>016X}", { self.elr_el1 });
|
||||
println!("SPSR_EL1: {:>016X}", { self.spsr_el1 });
|
||||
println!("ESR_EL1: {:>016X}", { self.esr_el1 });
|
||||
println!("SP_EL0: {:>016X}", { self.sp_el0 });
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
#[repr(C, packed)]
|
||||
pub struct InterruptStack {
|
||||
pub iret: IretRegisters,
|
||||
pub scratch: ScratchRegisters,
|
||||
pub preserved: PreservedRegisters,
|
||||
}
|
||||
|
||||
impl InterruptStack {
|
||||
pub fn init(&mut self) {}
|
||||
pub fn frame_pointer(&self) -> usize {
|
||||
self.preserved.x29
|
||||
}
|
||||
pub fn stack_pointer(&self) -> usize {
|
||||
self.iret.sp_el0
|
||||
}
|
||||
pub fn set_stack_pointer(&mut self, sp: usize) {
|
||||
self.iret.sp_el0 = sp;
|
||||
}
|
||||
pub fn sig_archdep_reg(&self) -> usize {
|
||||
self.scratch.x0
|
||||
}
|
||||
pub fn set_instr_pointer(&mut self, ip: usize) {
|
||||
self.iret.elr_el1 = ip;
|
||||
}
|
||||
pub fn instr_pointer(&self) -> usize {
|
||||
self.iret.elr_el1
|
||||
}
|
||||
pub fn set_arg1(&mut self, arg_opt: Option<usize>) {
|
||||
if let Some(arg) = arg_opt {
|
||||
self.scratch.x1 = arg;
|
||||
}
|
||||
}
|
||||
pub fn dump(&self) {
|
||||
self.iret.dump();
|
||||
self.scratch.dump();
|
||||
self.preserved.dump();
|
||||
}
|
||||
pub fn trace(&self) {
|
||||
self.dump();
|
||||
unsafe {
|
||||
panic::user_stack_trace(&self);
|
||||
panic::stack_trace();
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves all registers to a struct used by the proc:
|
||||
/// scheme to read/write registers.
|
||||
pub fn save(&self, all: &mut IntRegisters) {
|
||||
/*TODO: aarch64 registers
|
||||
all.elr_el1 = self.iret.elr_el1;
|
||||
all.spsr_el1 = self.iret.spsr_el1;
|
||||
all.esr_el1 = self.iret.esr_el1;
|
||||
all.sp_el0 = self.iret.sp_el0;
|
||||
all.padding = 0;
|
||||
*/
|
||||
all.x30 = self.preserved.x30;
|
||||
all.x29 = self.preserved.x29;
|
||||
all.x28 = self.preserved.x28;
|
||||
all.x27 = self.preserved.x27;
|
||||
all.x26 = self.preserved.x26;
|
||||
all.x25 = self.preserved.x25;
|
||||
all.x24 = self.preserved.x24;
|
||||
all.x23 = self.preserved.x23;
|
||||
all.x22 = self.preserved.x22;
|
||||
all.x21 = self.preserved.x21;
|
||||
all.x20 = self.preserved.x20;
|
||||
all.x19 = self.preserved.x19;
|
||||
all.x18 = self.scratch.x18;
|
||||
all.x17 = self.scratch.x17;
|
||||
all.x16 = self.scratch.x16;
|
||||
all.x15 = self.scratch.x15;
|
||||
all.x14 = self.scratch.x14;
|
||||
all.x13 = self.scratch.x13;
|
||||
all.x12 = self.scratch.x12;
|
||||
all.x11 = self.scratch.x11;
|
||||
all.x10 = self.scratch.x10;
|
||||
all.x9 = self.scratch.x9;
|
||||
all.x8 = self.scratch.x8;
|
||||
all.x7 = self.scratch.x7;
|
||||
all.x6 = self.scratch.x6;
|
||||
all.x5 = self.scratch.x5;
|
||||
all.x4 = self.scratch.x4;
|
||||
all.x3 = self.scratch.x3;
|
||||
all.x2 = self.scratch.x2;
|
||||
all.x1 = self.scratch.x1;
|
||||
all.x0 = self.scratch.x0;
|
||||
}
|
||||
|
||||
/// Loads all registers from a struct used by the proc:
|
||||
/// scheme to read/write registers.
|
||||
pub fn load(&mut self, all: &IntRegisters) {
|
||||
/*TODO: aarch64 registers
|
||||
self.iret.elr_el1 = all.elr_el1;
|
||||
self.iret.spsr_el1 = all.spsr_el1;
|
||||
self.iret.esr_el1 = all.esr_el1;
|
||||
self.iret.sp_el0 = all.sp_el0;
|
||||
*/
|
||||
self.preserved.x30 = all.x30;
|
||||
self.preserved.x29 = all.x29;
|
||||
self.preserved.x28 = all.x28;
|
||||
self.preserved.x27 = all.x27;
|
||||
self.preserved.x26 = all.x26;
|
||||
self.preserved.x25 = all.x25;
|
||||
self.preserved.x24 = all.x24;
|
||||
self.preserved.x23 = all.x23;
|
||||
self.preserved.x22 = all.x22;
|
||||
self.preserved.x21 = all.x21;
|
||||
self.preserved.x20 = all.x20;
|
||||
self.preserved.x19 = all.x19;
|
||||
self.scratch.x18 = all.x18;
|
||||
self.scratch.x17 = all.x17;
|
||||
self.scratch.x16 = all.x16;
|
||||
self.scratch.x15 = all.x15;
|
||||
self.scratch.x14 = all.x14;
|
||||
self.scratch.x13 = all.x13;
|
||||
self.scratch.x12 = all.x12;
|
||||
self.scratch.x11 = all.x11;
|
||||
self.scratch.x10 = all.x10;
|
||||
self.scratch.x9 = all.x9;
|
||||
self.scratch.x8 = all.x8;
|
||||
self.scratch.x7 = all.x7;
|
||||
self.scratch.x6 = all.x6;
|
||||
self.scratch.x5 = all.x5;
|
||||
self.scratch.x4 = all.x4;
|
||||
self.scratch.x3 = all.x3;
|
||||
self.scratch.x2 = all.x2;
|
||||
self.scratch.x1 = all.x1;
|
||||
self.scratch.x0 = all.x0;
|
||||
}
|
||||
|
||||
/// Store a specific generic registers
|
||||
pub fn store_reg(&mut self, idx: usize, val: usize) {
|
||||
match idx {
|
||||
0 => self.scratch.x0 = val,
|
||||
1 => self.scratch.x1 = val,
|
||||
2 => self.scratch.x2 = val,
|
||||
3 => self.scratch.x3 = val,
|
||||
4 => self.scratch.x4 = val,
|
||||
5 => self.scratch.x5 = val,
|
||||
6 => self.scratch.x6 = val,
|
||||
7 => self.scratch.x7 = val,
|
||||
8 => self.scratch.x8 = val,
|
||||
9 => self.scratch.x9 = val,
|
||||
10 => self.scratch.x10 = val,
|
||||
11 => self.scratch.x11 = val,
|
||||
12 => self.scratch.x12 = val,
|
||||
13 => self.scratch.x13 = val,
|
||||
14 => self.scratch.x14 = val,
|
||||
15 => self.scratch.x15 = val,
|
||||
16 => self.scratch.x16 = val,
|
||||
17 => self.scratch.x17 = val,
|
||||
18 => self.scratch.x18 = val,
|
||||
19 => self.preserved.x19 = val,
|
||||
20 => self.preserved.x20 = val,
|
||||
21 => self.preserved.x21 = val,
|
||||
22 => self.preserved.x22 = val,
|
||||
23 => self.preserved.x23 = val,
|
||||
24 => self.preserved.x24 = val,
|
||||
25 => self.preserved.x25 = val,
|
||||
26 => self.preserved.x26 = val,
|
||||
27 => self.preserved.x27 = val,
|
||||
28 => self.preserved.x28 = val,
|
||||
29 => self.preserved.x29 = val,
|
||||
30 => self.preserved.x30 = val,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
//TODO
|
||||
pub fn set_singlestep(&mut self, _singlestep: bool) {}
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! push_scratch {
|
||||
() => {
|
||||
"
|
||||
// Push scratch registers
|
||||
str x18, [sp, #-16]!
|
||||
stp x16, x17, [sp, #-16]!
|
||||
stp x14, x15, [sp, #-16]!
|
||||
stp x12, x13, [sp, #-16]!
|
||||
stp x10, x11, [sp, #-16]!
|
||||
stp x8, x9, [sp, #-16]!
|
||||
stp x6, x7, [sp, #-16]!
|
||||
stp x4, x5, [sp, #-16]!
|
||||
stp x2, x3, [sp, #-16]!
|
||||
stp x0, x1, [sp, #-16]!
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! pop_scratch {
|
||||
() => {
|
||||
"
|
||||
// Pop scratch registers
|
||||
ldp x0, x1, [sp], #16
|
||||
ldp x2, x3, [sp], #16
|
||||
ldp x4, x5, [sp], #16
|
||||
ldp x6, x7, [sp], #16
|
||||
ldp x8, x9, [sp], #16
|
||||
ldp x10, x11, [sp], #16
|
||||
ldp x12, x13, [sp], #16
|
||||
ldp x14, x15, [sp], #16
|
||||
ldp x16, x17, [sp], #16
|
||||
ldr x18, [sp], #16
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! push_preserved {
|
||||
() => {
|
||||
"
|
||||
// Push preserved registers
|
||||
stp x29, x30, [sp, #-16]!
|
||||
stp x27, x28, [sp, #-16]!
|
||||
stp x25, x26, [sp, #-16]!
|
||||
stp x23, x24, [sp, #-16]!
|
||||
stp x21, x22, [sp, #-16]!
|
||||
stp x19, x20, [sp, #-16]!
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! pop_preserved {
|
||||
() => {
|
||||
"
|
||||
// Pop preserved registers
|
||||
ldp x19, x20, [sp], #16
|
||||
ldp x21, x22, [sp], #16
|
||||
ldp x23, x24, [sp], #16
|
||||
ldp x25, x26, [sp], #16
|
||||
ldp x27, x28, [sp], #16
|
||||
ldp x29, x30, [sp], #16
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! push_special {
|
||||
() => {
|
||||
"
|
||||
mrs x14, spsr_el1
|
||||
mrs x15, elr_el1
|
||||
stp x14, x15, [sp, #-16]!
|
||||
|
||||
mrs x14, sp_el0
|
||||
mrs x15, esr_el1
|
||||
stp x14, x15, [sp, #-16]!
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! pop_special {
|
||||
() => {
|
||||
"
|
||||
ldp x14, x15, [sp], 16
|
||||
msr esr_el1, x15
|
||||
msr sp_el0, x14
|
||||
|
||||
ldp x14, x15, [sp], 16
|
||||
msr elr_el1, x15
|
||||
msr spsr_el1, x14
|
||||
"
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! exception_stack {
|
||||
($name:ident, |$stack:ident| $code:block) => {
|
||||
#[unsafe(naked)]
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn $name(stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
|
||||
unsafe extern "C" fn inner($stack: &mut $crate::arch::aarch64::interrupt::InterruptStack) {
|
||||
$code
|
||||
}
|
||||
core::arch::naked_asm!(
|
||||
// Backup all userspace registers to stack
|
||||
push_preserved!(),
|
||||
push_scratch!(),
|
||||
push_special!(),
|
||||
|
||||
// Call inner function with pointer to stack
|
||||
"mov x29, sp",
|
||||
"mov x0, sp",
|
||||
"bl {}",
|
||||
|
||||
// Restore all userspace registers
|
||||
pop_special!(),
|
||||
pop_scratch!(),
|
||||
pop_preserved!(),
|
||||
|
||||
"eret",
|
||||
|
||||
sym inner,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
#[unsafe(naked)]
|
||||
pub unsafe extern "C" fn enter_usermode() -> ! {
|
||||
core::arch::naked_asm!(
|
||||
"blr x28",
|
||||
// Restore all userspace registers
|
||||
pop_special!(),
|
||||
pop_scratch!(),
|
||||
pop_preserved!(),
|
||||
"eret",
|
||||
);
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
use crate::{arch::device::ROOT_IC_IDX, dtb::irqchip::IRQ_CHIP, sync::CleanLockToken};
|
||||
use core::sync::atomic::Ordering;
|
||||
|
||||
// use crate::percpu::PercpuBlock;
|
||||
|
||||
unsafe fn irq_ack() -> (u32, Option<usize>) {
|
||||
unsafe {
|
||||
let ic = &mut IRQ_CHIP.irq_chip_list.chips[ROOT_IC_IDX.load(Ordering::Relaxed)].ic;
|
||||
let irq = ic.irq_ack();
|
||||
(irq, ic.irq_to_virq(irq))
|
||||
}
|
||||
}
|
||||
|
||||
exception_stack!(irq_at_el0, |_stack| {
|
||||
unsafe {
|
||||
let mut token = CleanLockToken::new();
|
||||
let (irq, virq) = irq_ack();
|
||||
if let Some(virq) = virq
|
||||
&& virq < 1024
|
||||
{
|
||||
IRQ_CHIP.trigger_virq(virq as u32, &mut token);
|
||||
} else {
|
||||
println!("unexpected irq num {}", irq);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
exception_stack!(irq_at_el1, |_stack| {
|
||||
unsafe {
|
||||
let mut token = CleanLockToken::new();
|
||||
let (irq, virq) = irq_ack();
|
||||
if let Some(virq) = virq
|
||||
&& virq < 1024
|
||||
{
|
||||
IRQ_CHIP.trigger_virq(virq as u32, &mut token);
|
||||
} else {
|
||||
println!("unexpected irq num {}", irq);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/*
|
||||
pub unsafe fn irq_handler_gentimer(irq: u32) {
|
||||
GENTIMER.clear_irq();
|
||||
{
|
||||
*time::OFFSET.lock() += GENTIMER.clk_freq as u128;
|
||||
}
|
||||
|
||||
timeout::trigger();
|
||||
|
||||
context::switch::tick();
|
||||
|
||||
trigger(irq);
|
||||
GENTIMER.reload_count();
|
||||
}
|
||||
*/
|
||||
@@ -1,49 +0,0 @@
|
||||
//! Interrupt instructions
|
||||
|
||||
use core::arch::asm;
|
||||
|
||||
#[macro_use]
|
||||
pub mod handler;
|
||||
|
||||
pub mod exception;
|
||||
pub mod irq;
|
||||
pub mod syscall;
|
||||
pub mod trace;
|
||||
|
||||
pub use self::handler::InterruptStack;
|
||||
|
||||
/// Clear interrupts
|
||||
#[inline(always)]
|
||||
pub unsafe fn disable() {
|
||||
unsafe {
|
||||
asm!("msr daifset, #2");
|
||||
}
|
||||
}
|
||||
|
||||
/// Set interrupts and halt
|
||||
/// This will atomically wait for the next interrupt
|
||||
/// Performing enable followed by halt is not guaranteed to be atomic, use this instead!
|
||||
#[inline(always)]
|
||||
pub unsafe fn enable_and_halt() {
|
||||
unsafe {
|
||||
asm!("wfi", "msr daifclr, #2", "nop");
|
||||
}
|
||||
}
|
||||
|
||||
/// Set interrupts and nop
|
||||
/// This will enable interrupts and allow the IF flag to be processed
|
||||
/// Simply enabling interrupts does not gurantee that they will trigger, use this instead!
|
||||
#[inline(always)]
|
||||
pub unsafe fn enable_and_nop() {
|
||||
unsafe {
|
||||
asm!("msr daifclr, #2", "nop");
|
||||
}
|
||||
}
|
||||
|
||||
/// Halt instruction
|
||||
#[inline(always)]
|
||||
pub unsafe fn halt() {
|
||||
unsafe {
|
||||
asm!("wfi");
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn do_exception_unhandled() {}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn do_exception_synchronous() {}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[repr(C, packed)]
|
||||
pub struct SyscallStack {
|
||||
pub elr_el1: usize,
|
||||
pub padding: usize,
|
||||
pub tpidr: usize,
|
||||
pub tpidrro: usize,
|
||||
pub rflags: usize,
|
||||
pub esr: usize,
|
||||
pub sp: usize,
|
||||
pub lr: usize,
|
||||
pub fp: usize,
|
||||
pub x28: usize,
|
||||
pub x27: usize,
|
||||
pub x26: usize,
|
||||
pub x25: usize,
|
||||
pub x24: usize,
|
||||
pub x23: usize,
|
||||
pub x22: usize,
|
||||
pub x21: usize,
|
||||
pub x20: usize,
|
||||
pub x19: usize,
|
||||
pub x18: usize,
|
||||
pub x17: usize,
|
||||
pub x16: usize,
|
||||
pub x15: usize,
|
||||
pub x14: usize,
|
||||
pub x13: usize,
|
||||
pub x12: usize,
|
||||
pub x11: usize,
|
||||
pub x10: usize,
|
||||
pub x9: usize,
|
||||
pub x8: usize,
|
||||
pub x7: usize,
|
||||
pub x6: usize,
|
||||
pub x5: usize,
|
||||
pub x4: usize,
|
||||
pub x3: usize,
|
||||
pub x2: usize,
|
||||
pub x1: usize,
|
||||
pub x0: usize,
|
||||
}
|
||||
pub use super::handler::enter_usermode;
|
||||
@@ -1,32 +0,0 @@
|
||||
use core::arch::asm;
|
||||
|
||||
pub struct StackTrace {
|
||||
pub fp: usize,
|
||||
pub pc_ptr: *const usize,
|
||||
}
|
||||
|
||||
impl StackTrace {
|
||||
#[inline(always)]
|
||||
pub unsafe fn start() -> Option<Self> {
|
||||
unsafe {
|
||||
let fp: usize;
|
||||
asm!("mov {}, fp", out(reg) fp);
|
||||
let pc_ptr = fp.checked_add(size_of::<usize>())?;
|
||||
Some(StackTrace {
|
||||
fp,
|
||||
pc_ptr: pc_ptr as *const usize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn next(self) -> Option<Self> {
|
||||
unsafe {
|
||||
let fp = *(self.fp as *const usize);
|
||||
let pc_ptr = fp.checked_add(size_of::<usize>())?;
|
||||
Some(StackTrace {
|
||||
fp: fp,
|
||||
pc_ptr: pc_ptr as *const usize,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(u8)]
|
||||
pub enum IpiKind {
|
||||
Wakeup = 0x40,
|
||||
Tlb = 0x41,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
#[repr(u8)]
|
||||
pub enum IpiTarget {
|
||||
Other = 3,
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn ipi(_kind: IpiKind, _target: IpiTarget) {
|
||||
if cfg!(not(feature = "multi_core")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// FIXME implement
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn ipi_single(_kind: IpiKind, _target: &crate::percpu::PercpuBlock) {
|
||||
if cfg!(not(feature = "multi_core")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// FIXME implement
|
||||
}
|
||||
@@ -1,23 +0,0 @@
|
||||
use crate::{
|
||||
cpu_set::LogicalCpuId,
|
||||
memory::{RmmA, RmmArch},
|
||||
percpu::PercpuBlock,
|
||||
};
|
||||
|
||||
impl PercpuBlock {
|
||||
pub fn current() -> &'static Self {
|
||||
unsafe { &*(crate::arch::device::cpu::registers::control_regs::tpidr_el1() as *const Self) }
|
||||
}
|
||||
}
|
||||
|
||||
#[cold]
|
||||
pub unsafe fn init(cpu_id: LogicalCpuId) {
|
||||
unsafe {
|
||||
let frame = crate::memory::allocate_frame().expect("failed to allocate percpu memory");
|
||||
let virt = RmmA::phys_to_virt(frame.base()).data() as *mut PercpuBlock;
|
||||
|
||||
virt.write(PercpuBlock::init(cpu_id));
|
||||
|
||||
crate::arch::device::cpu::registers::control_regs::tpidr_el1_write(virt as u64);
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
/// Constants like memory locations
|
||||
pub mod consts;
|
||||
|
||||
/// Debugging support
|
||||
pub mod debug;
|
||||
|
||||
/// Devices
|
||||
pub mod device;
|
||||
|
||||
/// Interrupt instructions
|
||||
pub mod interrupt;
|
||||
|
||||
/// Inter-processor interrupts
|
||||
pub mod ipi;
|
||||
|
||||
/// Miscellaneous
|
||||
pub mod misc;
|
||||
|
||||
/// Paging
|
||||
pub mod paging;
|
||||
|
||||
/// Initialization and start function
|
||||
pub mod start;
|
||||
|
||||
/// Stop function
|
||||
pub mod stop;
|
||||
|
||||
// Interrupt vectors
|
||||
pub mod vectors;
|
||||
|
||||
pub mod time;
|
||||
|
||||
pub use ::rmm::aarch64::AArch64Arch as CurrentRmmArch;
|
||||
|
||||
pub use arch_copy_to_user as arch_copy_from_user;
|
||||
|
||||
#[unsafe(naked)]
|
||||
pub unsafe extern "C" fn arch_copy_to_user(dst: usize, src: usize, len: usize) -> u8 {
|
||||
// x0, x1, x2
|
||||
core::arch::naked_asm!(
|
||||
"
|
||||
.global __usercopy_start
|
||||
__usercopy_start:
|
||||
mov x4, x0
|
||||
mov x0, 0
|
||||
2:
|
||||
cmp x2, 0
|
||||
b.eq 3f
|
||||
|
||||
ldrb w3, [x1]
|
||||
strb w3, [x4]
|
||||
|
||||
add x4, x4, 1
|
||||
add x1, x1, 1
|
||||
sub x2, x2, 1
|
||||
|
||||
b 2b
|
||||
3:
|
||||
ret
|
||||
.global __usercopy_end
|
||||
__usercopy_end:
|
||||
"
|
||||
);
|
||||
}
|
||||
|
||||
pub const KFX_SIZE: usize = 1024;
|
||||
|
||||
// This function exists as the KFX size is dynamic on x86_64.
|
||||
pub fn kfx_size() -> usize {
|
||||
KFX_SIZE
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
/// Initialize MAIR
|
||||
#[cold]
|
||||
pub unsafe fn init() {
|
||||
unsafe {
|
||||
rmm::aarch64::init_mair();
|
||||
}
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
//! This function is where the kernel sets up IRQ handlers
|
||||
//! It is incredibly unsafe, and should be minimal in nature
|
||||
//! It must create the IDT with the correct entries, those entries are
|
||||
//! defined in other files inside of the `arch` module
|
||||
use core::{arch::naked_asm, cell::SyncUnsafeCell, slice};
|
||||
|
||||
use fdt::Fdt;
|
||||
|
||||
use crate::{
|
||||
allocator,
|
||||
arch::{device, paging},
|
||||
devices::graphical_debug,
|
||||
dtb,
|
||||
startup::KernelArgs,
|
||||
};
|
||||
|
||||
/// Test of zero values in BSS.
|
||||
static mut BSS_TEST_ZERO: usize = 0;
|
||||
/// Test of non-zero values in data.
|
||||
static mut DATA_TEST_NONZERO: usize = 0xFFFF_FFFF_FFFF_FFFF;
|
||||
|
||||
#[repr(C, align(16))]
|
||||
struct StackAlign<T>(T);
|
||||
|
||||
static STACK: SyncUnsafeCell<StackAlign<[u8; 128 * 1024]>> =
|
||||
SyncUnsafeCell::new(StackAlign([0; 128 * 1024]));
|
||||
|
||||
// FIXME use extern "custom"
|
||||
#[unsafe(naked)]
|
||||
#[unsafe(no_mangle)]
|
||||
extern "C" fn kstart() {
|
||||
naked_asm!("
|
||||
// BSS should already be zero
|
||||
adrp x9, {bss_test_zero}
|
||||
ldr x9, [x9, :lo12:{bss_test_zero}]
|
||||
cbnz x9, .Lkstart_crash
|
||||
adrp x9, {data_test_nonzero}
|
||||
ldr x9, [x9, :lo12:{data_test_nonzero}]
|
||||
cbz x9, .Lkstart_crash
|
||||
|
||||
adrp x1, {stack}
|
||||
add x1, x1, :lo12:{stack}
|
||||
mov x2, {stack_size}-16
|
||||
add sp, x1, x2
|
||||
|
||||
// Setup interrupt handlers
|
||||
ldr x9, =exception_vector_base
|
||||
msr vbar_el1, x9
|
||||
|
||||
mov lr, 0
|
||||
b {start}
|
||||
|
||||
.Lkstart_crash:
|
||||
mov x9, 0
|
||||
br x9
|
||||
",
|
||||
bss_test_zero = sym BSS_TEST_ZERO,
|
||||
data_test_nonzero = sym DATA_TEST_NONZERO,
|
||||
stack = sym STACK,
|
||||
stack_size = const size_of_val(&STACK),
|
||||
start = sym start,
|
||||
);
|
||||
}
|
||||
|
||||
/// The entry to Rust, all things must be initialized
|
||||
unsafe extern "C" fn start(args_ptr: *const KernelArgs) -> ! {
|
||||
unsafe {
|
||||
let bootstrap = {
|
||||
let args = args_ptr.read();
|
||||
|
||||
// Set up graphical debug
|
||||
graphical_debug::init(args.env());
|
||||
|
||||
// Get hardware descriptor data
|
||||
//TODO: use env {DTB,RSDT}_{BASE,SIZE}?
|
||||
let hwdesc_data = if args.hwdesc_base != 0 {
|
||||
Some(slice::from_raw_parts(
|
||||
(crate::PHYS_OFFSET + args.hwdesc_base as usize) as *const u8,
|
||||
args.hwdesc_size as usize,
|
||||
))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let dtb_res = hwdesc_data
|
||||
.ok_or(fdt::FdtError::BadPtr)
|
||||
.and_then(|data| Fdt::new(data));
|
||||
|
||||
// Try to find serial port prior to logging
|
||||
if let Ok(dtb) = &dtb_res {
|
||||
dtb::serial::init_early(dtb);
|
||||
}
|
||||
|
||||
info!("RedBear OS starting...");
|
||||
args.print();
|
||||
|
||||
// Initialize RMM
|
||||
crate::startup::memory::init(&args, None, None);
|
||||
|
||||
// Initialize paging
|
||||
paging::init();
|
||||
|
||||
crate::arch::misc::init(crate::cpu_set::LogicalCpuId::new(0));
|
||||
|
||||
// Setup kernel heap
|
||||
allocator::init();
|
||||
|
||||
// Activate memory logging
|
||||
crate::log::init();
|
||||
|
||||
// Initialize devices
|
||||
match dtb_res {
|
||||
Ok(dtb) => {
|
||||
dtb::init(hwdesc_data.map(|slice| (slice.as_ptr() as usize, slice.len())));
|
||||
device::init_devicetree(&dtb);
|
||||
}
|
||||
Err(err) => {
|
||||
dtb::init(None);
|
||||
warn!("failed to parse DTB: {}", err);
|
||||
|
||||
#[cfg(feature = "acpi")]
|
||||
{
|
||||
crate::acpi::init(args.acpi_rsdp());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
args.bootstrap()
|
||||
};
|
||||
|
||||
crate::startup::kmain(bootstrap);
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C, packed)]
|
||||
#[allow(unused)]
|
||||
pub struct KernelArgsAp {
|
||||
cpu_id: u64,
|
||||
page_table: u64,
|
||||
stack_start: u64,
|
||||
stack_end: u64,
|
||||
}
|
||||
|
||||
/// Entry to rust for an AP
|
||||
#[allow(unused)]
|
||||
pub unsafe extern "C" fn kstart_ap(_args_ptr: *const KernelArgsAp) -> ! {
|
||||
loop {}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
use crate::sync::CleanLockToken;
|
||||
use core::arch::asm;
|
||||
|
||||
pub unsafe fn kreset() -> ! {
|
||||
unsafe {
|
||||
println!("kreset");
|
||||
|
||||
asm!("hvc #0",
|
||||
in("x0") 0x8400_0009_usize,
|
||||
options(noreturn),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn emergency_reset() -> ! {
|
||||
unsafe {
|
||||
asm!("hvc #0",
|
||||
in("x0") 0x8400_0009_usize,
|
||||
options(noreturn),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub unsafe fn kstop(_token: &mut CleanLockToken) -> ! {
|
||||
unsafe {
|
||||
println!("kstop");
|
||||
|
||||
asm!("hvc #0",
|
||||
in("x0") 0x8400_0008_usize,
|
||||
options(noreturn),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
use crate::{sync::CleanLockToken, time::NANOS_PER_SEC};
|
||||
|
||||
pub fn monotonic_absolute(_token: &mut CleanLockToken) -> u128 {
|
||||
//TODO: aarch64 generic timer counter
|
||||
let ticks: usize;
|
||||
unsafe { core::arch::asm!("mrs {}, cntpct_el0", out(reg) ticks) };
|
||||
let freq: usize;
|
||||
unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) freq) };
|
||||
|
||||
ticks as u128 * NANOS_PER_SEC / freq as u128
|
||||
}
|
||||
|
||||
pub fn monotonic_resolution() -> u128 {
|
||||
let freq: usize;
|
||||
unsafe { core::arch::asm!("mrs {}, cntfrq_el0", out(reg) freq) };
|
||||
|
||||
NANOS_PER_SEC / freq as u128
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
core::arch::global_asm!(
|
||||
"
|
||||
// Exception vector stubs
|
||||
//
|
||||
// Unhandled exceptions spin in a wfi loop for the moment
|
||||
// This can be macro-ified
|
||||
|
||||
.globl exception_vector_base
|
||||
|
||||
.align 11
|
||||
exception_vector_base:
|
||||
|
||||
// Synchronous
|
||||
.align 7
|
||||
__vec_00:
|
||||
b synchronous_exception_at_el1_with_sp0
|
||||
b __vec_00
|
||||
|
||||
// IRQ
|
||||
.align 7
|
||||
__vec_01:
|
||||
b irq_at_el1
|
||||
b __vec_01
|
||||
|
||||
// FIQ
|
||||
.align 7
|
||||
__vec_02:
|
||||
b unhandled_exception
|
||||
b __vec_02
|
||||
|
||||
// SError
|
||||
.align 7
|
||||
__vec_03:
|
||||
b unhandled_exception
|
||||
b __vec_03
|
||||
|
||||
// Synchronous
|
||||
.align 7
|
||||
__vec_04:
|
||||
b synchronous_exception_at_el1_with_spx
|
||||
b __vec_04
|
||||
|
||||
// IRQ
|
||||
.align 7
|
||||
__vec_05:
|
||||
b irq_at_el1
|
||||
b __vec_05
|
||||
|
||||
// FIQ
|
||||
.align 7
|
||||
__vec_06:
|
||||
b unhandled_exception
|
||||
b __vec_06
|
||||
|
||||
// SError
|
||||
.align 7
|
||||
__vec_07:
|
||||
b unhandled_exception
|
||||
b __vec_07
|
||||
|
||||
// Synchronous
|
||||
.align 7
|
||||
__vec_08:
|
||||
b synchronous_exception_at_el0
|
||||
b __vec_08
|
||||
|
||||
// IRQ
|
||||
.align 7
|
||||
__vec_09:
|
||||
b irq_at_el0
|
||||
b __vec_09
|
||||
|
||||
// FIQ
|
||||
.align 7
|
||||
__vec_10:
|
||||
b unhandled_exception
|
||||
b __vec_10
|
||||
|
||||
// SError
|
||||
.align 7
|
||||
__vec_11:
|
||||
b unhandled_exception
|
||||
b __vec_11
|
||||
|
||||
// Synchronous
|
||||
.align 7
|
||||
__vec_12:
|
||||
b unhandled_exception
|
||||
b __vec_12
|
||||
|
||||
// IRQ
|
||||
.align 7
|
||||
__vec_13:
|
||||
b unhandled_exception
|
||||
b __vec_13
|
||||
|
||||
// FIQ
|
||||
.align 7
|
||||
__vec_14:
|
||||
b unhandled_exception
|
||||
b __vec_14
|
||||
|
||||
// SError
|
||||
.align 7
|
||||
__vec_15:
|
||||
b unhandled_exception
|
||||
b __vec_15
|
||||
|
||||
.align 7
|
||||
exception_vector_end:
|
||||
"
|
||||
);
|
||||
@@ -1,27 +0,0 @@
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
#[macro_use]
|
||||
pub mod aarch64;
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
pub use self::aarch64::*;
|
||||
|
||||
#[cfg(target_arch = "x86")]
|
||||
#[macro_use]
|
||||
pub mod x86;
|
||||
#[cfg(target_arch = "x86")]
|
||||
pub use self::x86::*;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
#[macro_use]
|
||||
pub mod x86_64;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub use self::x86_64::*;
|
||||
|
||||
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
|
||||
#[macro_use]
|
||||
mod x86_shared;
|
||||
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
#[macro_use]
|
||||
pub mod riscv64;
|
||||
#[cfg(target_arch = "riscv64")]
|
||||
pub use self::riscv64::*;
|
||||
@@ -1,16 +0,0 @@
|
||||
use super::CurrentRmmArch;
|
||||
use rmm::Arch;
|
||||
|
||||
const PML4_SHIFT: usize = (CurrentRmmArch::PAGE_LEVELS - 1) * CurrentRmmArch::PAGE_ENTRY_SHIFT
|
||||
+ CurrentRmmArch::PAGE_SHIFT;
|
||||
/// The size of a single PML4
|
||||
pub const PML4_SIZE: usize = 1_usize << PML4_SHIFT;
|
||||
|
||||
/// Offset to kernel heap
|
||||
#[inline(always)]
|
||||
pub fn kernel_heap_offset() -> usize {
|
||||
crate::kernel_executable_offsets::KERNEL_OFFSET() - PML4_SIZE
|
||||
}
|
||||
|
||||
/// End offset of the user image, i.e. kernel start
|
||||
pub const USER_END_OFFSET: usize = 1_usize << (CurrentRmmArch::PAGE_ADDRESS_SHIFT - 1);
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user