[RFC PATCH 01/18] aarch64: import the kernel-side port from bugaevc/wip-aarch64
Paulo Duarte <[email protected]>
| Newsgroups | gmane.os.hurd.bugs |
|---|---|
| Message-ID | <[email protected]> |
From: Sergey Bugaev <[email protected]> Upstream already carries the aarch64 stub: configfrag.ac sets the HOST_aarch64 conditional, Makefrag.am installs the public Mach headers under mach/aarch64/, and mach_aarch64.srv defines the per-arch MIG interface. This commit fills in the kernel-side implementation that the stub was waiting for, taken verbatim from Sergey Bugaev's wip-aarch64 branch at https://github.com/bugaevc/gnumach/tree/wip-aarch64 (last touched 2024-04-10). Every file added here is byte-identical to the corresponding file on that branch. New aarch64-only sources, gated by `if HOST_aarch64` in the existing aarch64/Makefrag.am: aarch64/aarch64/ locore.S, boot.S, cswitch.S, model_dep.c, pmap.c, pcb.c, trap.c, fpu.c, irq.c, conf.c, percpu.c, hwcaps.c, mach_aarch64.c, undef.c, strings.c, task.c, plus matching headers and an asm symbol file (aarch64asm.sym). aarch64/aarch64/bits/ MMU/CPU register-bit definitions: id_aa64, mair, pte, hcr, sctlr, spsr, tcr. aarch64/arm/ platform glue shared with future 32-bit ARM: gic-v2 (interrupt controller), pl011 (UART), psci (power state coordination), timer (ARMv8 generic timer). aarch64/ldscript linker script for the static-pie kernel image. aarch64/BOOTING brief notes on the bare-metal boot recipe. Two new device-layer files, also gated by `if HOST_aarch64`: device/dtb.{c,h} flattened-device-tree parser; consumed by aarch64/aarch64/model_dep.c to discover the timer, GIC, UART, and bootstrap modules. device/ramdisk.{c,h} RAM-backed block device; aarch64/aarch64/ conf.c's dev_name_list[] references it through the RAMDISK_DEV_OPS macro. Deliberately omitted from the import, in keeping with the minimum-touch principle documented in the cover letter: - Bugaev's ~150-file cross-arch refactor of kern/, ipc/, vm/, device/intr.{c,h}, and the i386 trees. Replaced by smaller per-arch shims in subsequent commits of this series. No shared files outside aarch64/ are modified by this commit; x86_64 and i686 builds remain identical to upstream master. NOTE: this import does not build standalone against current savannah master. wip-aarch64's Makefrag never wires device/ramdisk into libkernel_a_SOURCES even though conf.c references RAMDISK_DEV_OPS, so as-is the link fails on ramdisk_* symbols; on top of that, this series argues ramdisk is userland territory in Mach's design and removes it. Upstream's kern/clock.c also grew per-arch hpclock_* hooks after wip-aarch64 branched off, and two early-boot bugs in the imported code surface immediately under modern toolchains. Subsequent commits — kept individually atomic for review clarity — address each in turn: - aarch64: drop the ramdisk driver from the import - aarch64: add per-arch lock.h shim for kern/lock.h's machine include - aarch64: stub hpclock hooks for current master's clock subsystem - aarch64: move boot stack out of .bss so zero_out_bss doesn't clobber it - device/dtb: read 8-byte cells as two 4-byte loads to survive pre-MMU access --- aarch64/BOOTING | 62 + aarch64/Makefrag.am | 78 ++ aarch64/aarch64/aarch64asm.sym | 41 + aarch64/aarch64/bits/esr.h | 84 ++ aarch64/aarch64/bits/hcr.h | 17 + aarch64/aarch64/bits/id_aa64.h | 148 ++ aarch64/aarch64/bits/mair.h | 12 + aarch64/aarch64/bits/pte.h | 33 + aarch64/aarch64/bits/sctlr.h | 20 + aarch64/aarch64/bits/spsr.h | 42 + aarch64/aarch64/bits/tcr.h | 18 + aarch64/aarch64/boot.S | 100 ++ aarch64/aarch64/conf.c | 55 + aarch64/aarch64/cpu_number.h | 36 + aarch64/aarch64/cswitch.S | 97 ++ aarch64/aarch64/db_machdep.h | 25 + aarch64/aarch64/fpu.c | 157 +++ aarch64/aarch64/fpu.h | 27 + aarch64/aarch64/hwcaps.c | 164 +++ aarch64/aarch64/hwcaps.h | 31 + aarch64/aarch64/ipl.h | 24 + aarch64/aarch64/irq.c | 21 + aarch64/aarch64/irq.h | 74 + aarch64/aarch64/locore.S | 829 +++++++++++ aarch64/aarch64/locore.h | 50 + aarch64/aarch64/loose_ends.h | 24 + aarch64/aarch64/mach_aarch64.c | 45 + aarch64/aarch64/mach_param.h | 24 + aarch64/aarch64/machine_routines.h | 27 + aarch64/aarch64/model_dep.c | 417 ++++++ aarch64/aarch64/model_dep.h | 61 + aarch64/aarch64/mp_desc.h | 22 + aarch64/aarch64/pcb.c | 382 ++++++ aarch64/aarch64/pcb.h | 66 + aarch64/aarch64/percpu.c | 28 + aarch64/aarch64/percpu.h | 68 + aarch64/aarch64/pmap.c | 1207 +++++++++++++++++ aarch64/aarch64/pmap.h | 83 ++ aarch64/aarch64/setjmp.h | 19 + aarch64/aarch64/smp.h | 29 + aarch64/aarch64/spl.h | 100 ++ aarch64/aarch64/strings.c | 59 + aarch64/aarch64/task.c | 38 + aarch64/aarch64/task.h | 55 + aarch64/aarch64/thread.h | 67 + aarch64/aarch64/trap.c | 683 ++++++++++ aarch64/aarch64/trap.h | 47 + aarch64/aarch64/undef.c | 31 + aarch64/aarch64/vm_param.h | 56 + aarch64/aarch64/xpr.h | 25 + aarch64/arm/gic-v2.c | 194 +++ aarch64/arm/gic-v2.h | 38 + aarch64/arm/pl011.c | 207 +++ aarch64/arm/pl011.h | 24 + aarch64/arm/psci.c | 152 +++ aarch64/arm/psci.h | 29 + aarch64/arm/timer.c | 103 ++ aarch64/arm/timer.h | 29 + aarch64/configfrag.ac | 4 + aarch64/include/mach/aarch64/exception.h | 25 +- aarch64/include/mach/aarch64/exec/elf.h | 44 + .../include/mach/aarch64/mach_aarch64.defs | 2 +- .../include/mach/aarch64/mach_aarch64_types.h | 2 +- .../include/mach/aarch64/machine_types.defs | 21 +- aarch64/include/mach/aarch64/thread_status.h | 2 + aarch64/include/mach/aarch64/vm_param.h | 4 + aarch64/ldscript | 32 + device/dtb.c | 429 ++++++ device/dtb.h | 123 ++ device/ramdisk.c | 160 +++ device/ramdisk.h | 47 + 71 files changed, 7557 insertions(+), 22 deletions(-) create mode 100644 aarch64/BOOTING create mode 100644 aarch64/aarch64/aarch64asm.sym create mode 100644 aarch64/aarch64/bits/esr.h create mode 100644 aarch64/aarch64/bits/hcr.h create mode 100644 aarch64/aarch64/bits/id_aa64.h create mode 100644 aarch64/aarch64/bits/mair.h create mode 100644 aarch64/aarch64/bits/pte.h create mode 100644 aarch64/aarch64/bits/sctlr.h create mode 100644 aarch64/aarch64/bits/spsr.h create mode 100644 aarch64/aarch64/bits/tcr.h create mode 100644 aarch64/aarch64/boot.S create mode 100644 aarch64/aarch64/conf.c create mode 100644 aarch64/aarch64/cpu_number.h create mode 100644 aarch64/aarch64/cswitch.S create mode 100644 aarch64/aarch64/db_machdep.h create mode 100644 aarch64/aarch64/fpu.c create mode 100644 aarch64/aarch64/fpu.h create mode 100644 aarch64/aarch64/hwcaps.c create mode 100644 aarch64/aarch64/hwcaps.h create mode 100644 aarch64/aarch64/ipl.h create mode 100644 aarch64/aarch64/irq.c create mode 100644 aarch64/aarch64/irq.h create mode 100644 aarch64/aarch64/locore.S create mode 100644 aarch64/aarch64/locore.h create mode 100644 aarch64/aarch64/loose_ends.h create mode 100644 aarch64/aarch64/mach_aarch64.c create mode 100644 aarch64/aarch64/mach_param.h create mode 100644 aarch64/aarch64/machine_routines.h create mode 100644 aarch64/aarch64/model_dep.c create mode 100644 aarch64/aarch64/model_dep.h create mode 100644 aarch64/aarch64/mp_desc.h create mode 100644 aarch64/aarch64/pcb.c create mode 100644 aarch64/aarch64/pcb.h create mode 100644 aarch64/aarch64/percpu.c create mode 100644 aarch64/aarch64/percpu.h create mode 100644 aarch64/aarch64/pmap.c create mode 100644 aarch64/aarch64/pmap.h create mode 100644 aarch64/aarch64/setjmp.h create mode 100644 aarch64/aarch64/smp.h create mode 100644 aarch64/aarch64/spl.h create mode 100644 aarch64/aarch64/strings.c create mode 100644 aarch64/aarch64/task.c create mode 100644 aarch64/aarch64/task.h create mode 100644 aarch64/aarch64/thread.h create mode 100644 aarch64/aarch64/trap.c create mode 100644 aarch64/aarch64/trap.h create mode 100644 aarch64/aarch64/undef.c create mode 100644 aarch64/aarch64/vm_param.h create mode 100644 aarch64/aarch64/xpr.h create mode 100644 aarch64/arm/gic-v2.c create mode 100644 aarch64/arm/gic-v2.h create mode 100644 aarch64/arm/pl011.c create mode 100644 aarch64/arm/pl011.h create mode 100644 aarch64/arm/psci.c create mode 100644 aarch64/arm/psci.h create mode 100644 aarch64/arm/timer.c create mode 100644 aarch64/arm/timer.h create mode 100644 aarch64/include/mach/aarch64/exec/elf.h create mode 100644 aarch64/ldscript create mode 100644 device/dtb.c create mode 100644 device/dtb.h create mode 100644 device/ramdisk.c create mode 100644 device/ramdisk.h diff --git a/aarch64/BOOTING b/aarch64/BOOTING new file mode 100644 index 00000000..6248cc17 --- /dev/null +++ b/aarch64/BOOTING @@ -0,0 +1,62 @@ +GNU Mach on AArch64 follows the Linux AArch64 boot protocol, documented at +https://docs.kernel.org/arch/arm64/booting.html (or +Documentation/arch/arm64/booting.rst in the Linux source tree). A device tree +blob is required to boot. + +Initial development is happening against QEMU's "virt" machine. It should be +possible to boot GNU Mach on QEMU like this: + +$ qemu-system-aarch64 -machine virt -cpu cortex-a53 -m 1G -nographic \ + -kernel ./gnumach -append "gnumach cmdline goes here" + +To run any code in userland, you need to load some bootstrap modules along with +the kernel (for a GNU/Hurd system, this would be at least ext2fs and the exec +server). GNU Mach on x86 has traditionally used Multiboot modules for this, +but there's no Multiboot on AArch64. + +So on AArch64, GNU Mach instead expects the bootstrap modules to be described +in the Xen-compatible format in the device tree, as documented at +docs/misc/arm/device-tree/booting.txt in the Xen source tree. Specifically, +each bootstrap module should be described by a child node of the "/chosen" node +which has "compatible" set to "multiboot,module", and "bootargs" to the +corresponding line of the Mach boot script. For example: + +chosen { + bootargs = "gnumach cmdline goes here"; + stdout-path = "/pl011@9000000"; + #address-cells = <0x2>; + #size-cells = <0x2>; + + module@0x4000 { + compatible = "multiboot,kernel", "multiboot,module"; + reg = <0x0 0x4000 0x0 0x76670>; + bootargs = "example --host-priv-port=${host-port} --device-master-port=${device-port} $(task-create) $(task-resume)"; + }; + + /* More modules... */ +} + +This (or close enough to this) is also the format that QEMU's guest-loader +device implements, which makes it possible to boot GNU Mach with bootstrap +modules using QEMU invocations like this: + +$ qemu-system-aarch64 -machine virt -cpu cortex-a53 -m 1G -nographic \ + -kernel ./gnumach -append "gnumach cmdline goes here" \ + -device guest-loader,addr=0x4000,kernel=./example,bootargs='example --host-priv-port=${host-port} --device-master-port=${device-port} $(task-create) $(task-resume)' + +Note that you have to explicitly specify a physical memory address to load the +module at, and use the "kernel=" syntax instead of the more natural "initrd=", +since a module loaded using the "initrd=" syntax cannot have bootargs, which +are required for GNU Mach bootstrap modules. + +Something like the following should give you the required format with Das +U-Boot, but note this hasn't been tested at all yet: + +fdt set /chosen \#address-cells <2> +fdt set /chosen \#size-cells <2> +fdt mknod /chosen module@0x4000 +fdt set /chosen/module@0x4000 compatible "multiboot,kernel" "multiboot,module" +fdt set /chosen/module@0x4000 reg <0x0 ... 0x0 ...> +fdt set /chosen/module@0x4000 bootargs "..." + +booti ${gnumach_addr} - ${fdt_addr} diff --git a/aarch64/Makefrag.am b/aarch64/Makefrag.am index dd1837d4..cfcc1f6c 100644 --- a/aarch64/Makefrag.am +++ b/aarch64/Makefrag.am @@ -17,11 +17,81 @@ # Building a distribution. # EXTRA_DIST += \ + aarch64/aarch64/aarch64asm.sym \ aarch64/aarch64/mach_aarch64.srv \ aarch64/include/mach/aarch64 if HOST_aarch64 +# +# Source files for the aarch64 kernel. +# + +libkernel_a_SOURCES += \ + aarch64/aarch64/ast.h \ + aarch64/aarch64/boot.S \ + aarch64/aarch64/bits/id_aa64.h \ + aarch64/aarch64/bits/mair.h \ + aarch64/aarch64/bits/pte.h \ + aarch64/aarch64/bits/hcr.h \ + aarch64/aarch64/bits/sctlr.h \ + aarch64/aarch64/bits/spsr.h \ + aarch64/aarch64/bits/tcr.h \ + aarch64/aarch64/conf.c \ + aarch64/aarch64/cpu_number.h \ + aarch64/aarch64/cswitch.S \ + aarch64/aarch64/fpu.h \ + aarch64/aarch64/fpu.c \ + aarch64/aarch64/hwcaps.h \ + aarch64/aarch64/hwcaps.c \ + aarch64/aarch64/irq.c \ + aarch64/aarch64/locore.h \ + aarch64/aarch64/locore.S \ + aarch64/aarch64/mach_aarch64.c \ + aarch64/aarch64/pcb.c \ + aarch64/aarch64/percpu.c \ + aarch64/aarch64/percpu.h \ + aarch64/aarch64/pmap.h \ + aarch64/aarch64/pmap.c \ + aarch64/aarch64/setjmp.h \ + aarch64/aarch64/spl.h \ + aarch64/aarch64/strings.c \ + aarch64/aarch64/task.c \ + aarch64/aarch64/task.h \ + aarch64/aarch64/thread.h \ + aarch64/aarch64/trap.c \ + aarch64/aarch64/vm_param.h \ + aarch64/aarch64/model_dep.c \ + aarch64/aarch64/undef.c \ + aarch64/arm/gic-v2.h \ + aarch64/arm/gic-v2.c \ + aarch64/arm/pl011.h \ + aarch64/arm/pl011.c \ + aarch64/arm/psci.h \ + aarch64/arm/psci.c \ + aarch64/arm/timer.h \ + aarch64/arm/timer.c \ + device/dtb.h \ + device/dtb.c \ + device/cirbuf.c + + +# +# Automatically generated source files. +# +# See Makerules.mig.am. +# +nodist_lib_dep_tr_for_defs_a_SOURCES += \ + aarch64/aarch64/mach_aarch64.server.defs.c + +nodist_libkernel_a_SOURCES += \ + aarch64/aarch64/mach_aarch64.server.c \ + aarch64/aarch64/mach_aarch64.server.h \ + aarch64/aarch64/mach_aarch64.server.msgids + +nodist_libkernel_a_SOURCES += \ + aarch64/aarch64/aarch64asm.h + # # Installation. # @@ -40,4 +110,12 @@ include_mach_aarch64_HEADERS = \ aarch64/include/mach/aarch64/vm_param.h \ aarch64/include/mach/aarch64/vm_types.h +AM_CFLAGS += -mno-outline-atomics + +AM_CFLAGS += -static-pie -mcmodel=tiny -mgeneral-regs-only +gnumach_LINKFLAGS += -T '$(srcdir)'/aarch64/ldscript -static -pie --no-dynamic-linker + +# Hack to make gnumach.elf the ELF (useful for debugging), and bare gnumach the bootable image. +gnumach_LINKFLAGS += $(gnumach_LDADD) -o gnumach.elf && $(OBJCOPY) -O binary gnumach.elf gnumach && : + endif # HOST_aarch64 diff --git a/aarch64/aarch64/aarch64asm.sym b/aarch64/aarch64/aarch64asm.sym new file mode 100644 index 00000000..e59d0fd9 --- /dev/null +++ b/aarch64/aarch64/aarch64asm.sym @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/thread.h" +#include <kern/thread.h> + +offset thread th kernel_stack +offset thread th swap_func +expr &STACK_AKS(0)->k_regs KSS_REGS +size aarch64_kernel_state aks +size aarch64_exception_link ael +size aarch64_thread_state ats +size aarch64_kernel_exception_state akes + +offset aarch64_thread_state ats x[0] ATS_X +offset aarch64_thread_state ats sp +offset aarch64_thread_state ats pc +offset aarch64_thread_state ats tpidr_el0 +offset aarch64_thread_state ats cpsr + +offset aarch64_kernel_exception_state akes x[0] AKES_X +offset aarch64_kernel_exception_state akes cpsr +offset aarch64_kernel_exception_state akes pc + +offset percpu pc active_thread PERCPU_ACTIVE_THREAD +offset percpu pc active_stack PERCPU_ACTIVE_STACK diff --git a/aarch64/aarch64/bits/esr.h b/aarch64/aarch64/bits/esr.h new file mode 100644 index 00000000..bc06e658 --- /dev/null +++ b/aarch64/aarch64/bits/esr.h @@ -0,0 +1,84 @@ +#ifndef _AARCH64_BITS_ESR_ +#define _AARCH64_BITS_ESR_ + +/* Extract exception class from ESR value. */ +#define ESR_EC(esr) (((esr) >> 26) & 0x3f) +/* Exception classes. */ +#define ESR_EC_UNK 0x00 /* unknown reason */ +#define ESR_EC_WF 0x01 /* WFI/WFE */ +#define ESR_EC_FP_ACCESS 0x07 /* FP/AdvSIMD access when disabled */ +#define ESR_EC_BTI 0x0d /* BTI failure */ +#define ESR_EC_IL 0x0e /* illegal execution state */ +#define ESR_EC_SVC 0x15 /* SVC (syscall) */ +#define ESR_EC_HVC 0x16 /* HVC */ +#define ESR_EC_SMC 0x17 /* SMC */ +#define ESR_EC_MRS 0x18 /* MRS or MRS (or cache?) */ +#define ESR_EC_PAC 0x1c /* PAC failure */ +#define ESR_EC_IABT_LOWER_EL 0x20 /* instruction abort from lower EL */ +#define ESR_EC_IABT_SAME_EL 0x21 /* instruction abort from the same EL */ +#define ESR_EC_AL_PC 0x22 /* misaligned PC */ +#define ESR_EC_DABT_LOWER_EL 0x24 /* data abort from lower EL */ +#define ESR_EC_DABT_SAME_EL 0x25 /* data abort from the same EL */ +#define ESR_EC_AL_SP 0x26 /* misaligned SP */ +#define ESR_EC_FP_EXC 0x2c /* FP exception */ +#define ESR_EC_SERROR 0x2f /* SError */ +#define ESR_EC_BREAKPT_LOWER_EL 0x30 /* hardware breakpoint from lower EL */ +#define ESR_EC_BREAKPT_SAME_EL 0x31 /* hardware breakpoint from the same EL */ +#define ESR_EC_SS_LOWER_EL 0x32 /* software single step from lower EL */ +#define ESR_EC_SS_SAME_EL 0x33 /* software single step from the same EL */ +#define ESR_EC_WATCHPT_LOWER_EL 0x34 /* hardware watchpoint from lower EL */ +#define ESR_EC_WATCHPT_SAME_EL 0x35 /* hardware watchpoint from the same EL */ +#define ESR_EC_BRK 0x3c /* BRK */ + +#define ESR_WF_TI(esr) ((esr) & 0x3) /* WF* trapped instruction */ + +#define ESR_WF_TI_WFI 0x0 +#define ESR_WF_TI_WFE 0x1 +#define ESR_WF_TI_WFIT 0x2 +#define ESR_WF_TI_WFET 0x3 + +#define ESR_SVC_IMM(esr) ((esr) & 0xffff) +#define ESR_HVC_IMM(esr) ((esr) & 0xffff) +#define ESR_SMC_IMM(esr) ((esr) & 0xffff) +#define ESR_PAC_INFO(esr) ((esr) & 0x3) + +#define ESR_IABT_IFSC(esr) ((esr) & 0x3f) /* instruction fault status code */ + +#define ESR_IABT_IFSC_PERM_L0 0x0c /* permission fault, level 0 */ +#define ESR_IABT_IFSC_PERM_L1 0x0d /* permission fault, level 1 */ +#define ESR_IABT_IFSC_PERM_L2 0x0e /* permission fault, level 2 */ +#define ESR_IABT_IFSC_PERM_L3 0x0f /* permission fault, level 3 */ +#define ESR_IABT_IFSC_SYNC_EXT 0x10 /* synchronous external abort */ + +#define ESR_DABT_DFSC(esr) ((esr) & 0x3f) /* data fault status code */ + +#define ESR_DABT_DFSC_PERM_L0 0x0c /* permission fault, level 0 */ +#define ESR_DABT_DFSC_PERM_L1 0x0d /* permission fault, level 1 */ +#define ESR_DABT_DFSC_PERM_L2 0x0e /* permission fault, level 2 */ +#define ESR_DABT_DFSC_PERM_L3 0x0f /* permission fault, level 3 */ +#define ESR_DABT_DFSC_MTE 0x11 /* synchronous MTE tag check fault */ +#define ESR_DABT_DFSC_AL 0x21 /* alignment fault */ + +#define ESR_WNR 0x040 /* "write, not read" bit */ +#define ESR_CM 0x100 /* it was a cache maintenance operation */ + +#define ESR_ABT_FNV 0x400 /* "FAR not valid" bit (both IABT & DABT) */ + +#define ESR_FP_EXC_TFV(esr) ((esr) & 0x800000) /* other FP bits hols meaningful values */ +#define ESR_FP_EXC_IDF(esr) ((esr) & 0x80) /* input denormal */ +#define ESR_FP_EXC_IXF(esr) ((esr) & 0x10) /* inexact */ +#define ESR_FP_EXC_UFF(esr) ((esr) & 0x08) /* underflow */ +#define ESR_FP_EXC_OFF(esr) ((esr) & 0x04) /* overflow */ +#define ESR_FP_EXC_DZF(esr) ((esr) & 0x02) /* divide by zero */ +#define ESR_FP_EXC_IOF(esr) ((esr) & 0x01) /* invalid operation */ + +#define ESR_BTI_BTYPE(esr) ((esr) & 0x3) /* BTYPE that caused the BTI exception */ + +#define ESR_SS_ISV(esr) ((esr) & 0x1000000) /* EX holds a meaningful value */ +#define ESR_SS_EX(esr) ((esr) & 0x40) /* stepped instruction was load-exclusive */ + +#define ESR_WATCHPT_DFSC 0x22 + +#define ESR_BRK_IMM(esr) ((esr) & 0xffff) + +#endif /* _AARCH64_BITS_ESR_ */ diff --git a/aarch64/aarch64/bits/hcr.h b/aarch64/aarch64/bits/hcr.h new file mode 100644 index 00000000..1e6652ce --- /dev/null +++ b/aarch64/aarch64/bits/hcr.h @@ -0,0 +1,17 @@ +#ifndef _AARCH64_BITS_HCR_ +#define _AARCH64_BITS_HCR_ + +#define HCR_TWI 0x0000000000002000 /* trap WFI to EL2 */ +#define HCR_TWE 0x0000000000004000 /* trap WFE to EL2 */ +#define HCR_TGE 0x0000000008000000 /* trap general exceptions from EL0 to EL2 */ +#define HCR_RW 0x0000000080000000 /* register width, EL1 is AArch64 if set */ +#define HCR_E2H 0x0000000400000000 /* EL2 host */ +#define HCR_VSE 0x0000000000000100 /* virtual SError pending */ +#define HCR_VI 0x0000000000000080 /* virtual IRQ pending */ +#define HCR_VF 0x0000000000000040 /* virtual FIQ pending */ +#define HCR_AMO 0x0000000000000020 /* physical SError rounting (?) */ +#define HCR_IMO 0x0000000000000010 /* physical IRQ routing (?) */ +#define HCR_FMO 0x0000000000000008 /* physical FIQ routing (?) */ +#define HCR_VM 0x0000000000000001 /* enable virtualization (stage 2 translation) */ + +#endif /* _AARCH64_BITS_HCR_ */ diff --git a/aarch64/aarch64/bits/id_aa64.h b/aarch64/aarch64/bits/id_aa64.h new file mode 100644 index 00000000..2f9365e1 --- /dev/null +++ b/aarch64/aarch64/bits/id_aa64.h @@ -0,0 +1,148 @@ +#ifndef _AARCH64_BITS_ID_AA64_ +#define _AARCH64_BITS_ID_AA64_ + +#define ID_AA64PFR0_ASIMD(v) (((v) >> 20) & 0xf) +#define ID_AA64PFR0_FP(v) (((v) >> 16) & 0xf) +#define ID_AA64PFR0_SVE(v) (((v) >> 32) & 0xf) +#define ID_AA64PFR0_DIT(v) (((v) >> 48) & 0xf) + +#define ID_AA64PFR0_ASIMD_NONE 0xf /* AdvSIMD not supported */ +#define ID_AA64PFR0_ASIMD_ASIMD 0x0 /* AdvSIMD supported */ +#define ID_AA64PFR0_ASIMD_FP16 0x1 /* AdvSIMD + FP16/FPHP supported */ +#define ID_AA64PFR0_FP_NONE 0xf /* FP not supported */ +#define ID_AA64PFR0_FP_FP 0x0 /* FP supported */ +#define ID_AA64PFR0_FP_FP16 0x1 /* FP + FP16/FPHP supported */ +#define ID_AA64PFR0_SVE_NONE 0x0 /* SVE not supported */ +#define ID_AA64PFR0_SVE_SVE 0x1 /* SVE supported */ +#define ID_AA64PFR0_DIT_NONE 0x0 /* DIT not supported */ +#define ID_AA64PFR0_DIT_DIT 0x1 /* DIT supported */ + +#define ID_AA64PFR1_BT(v) ((v) & 0xf) +#define ID_AA64PFR1_SSBS(v) (((v) >> 4) & 0xf) +#define ID_AA64PFR1_SME(v) (((v) >> 24) 0xf) + +#define ID_AA64PFR1_BT_NONE 0x0 /* BTI not supported */ +#define ID_AA64PFR1_BT_BTI 0x1 /* BTI supported */ +#define ID_AA64PFR1_SSBS_NONE 0x0 /* SSBS not supported */ +#define ID_AA64PFR1_SSBS_SSBS 0x1 /* SSBS supported */ +#define ID_AA64PFR1_SSBS_SSBS2 0x2 /* SSBS + SSBS2 supported */ +#define ID_AA64PFR1_SME_NONE 0x0 /* SME not supported */ +#define ID_AA64PFR1_SME_SME 0x1 /* SME supported */ +#define ID_AA64PFR1_SME_SME2 0x2 /* SME + SME2 supported */ + + +#define ID_AA64ISAR0_AES(v) (((v) >> 4) & 0xf) +#define ID_AA64ISAR0_SHA1(v) (((v) >> 8) & 0xf) +#define ID_AA64ISAR0_SHA2(v) (((v) >> 12) & 0xf) +#define ID_AA64ISAR0_CRC32(v) (((v) >> 16) & 0xf) +#define ID_AA64ISAR0_ATOMIC(v) (((v) >> 20) & 0xf) +#define ID_AA64ISAR0_RDM(v) (((v) >> 28) & 0xf) +#define ID_AA64ISAR0_SHA3(v) (((v) >> 32) & 0xf) +#define ID_AA64ISAR0_SM3(v) (((v) >> 36) & 0xf) +#define ID_AA64ISAR0_SM4(v) (((v) >> 40) & 0xf) +#define ID_AA64ISAR0_DP(v) (((v) >> 44) & 0xf) +#define ID_AA64ISAR0_FHM(v) (((v) >> 48) & 0xf) +#define ID_AA64ISAR0_TS(v) (((v) >> 52) & 0xf) +#define ID_AA64ISAR0_RNDR(v) (((v) >> 60) & 0xf) + +#define ID_AA64ISAR0_AES_AES 0x1 /* AES supported */ +#define ID_AA64ISAR0_AES_PMULL 0x2 /* AES + PMULL supported */ +#define ID_AA64ISAR0_SHA1_NONE 0x0 /* SHA1 not supported */ +#define ID_AA64ISAR0_SHA2_NONE 0x0 /* SHA2 not supported */ +#define ID_AA64ISAR0_SHA2_SHA2 0x1 /* SHA2 supported */ +#define ID_AA64ISAR0_SHA2_SHA512 0x2 /* SHA2 + SHA512 supported */ +#define ID_AA64ISAR0_CRC32_NONE 0x0 /* CRC32 not supported */ +#define ID_AA64ISAR0_ATOMIC_NONE 0x0 /* atomics not supported */ +#define ID_AA64ISAR0_ATOMIC_LSE 0x2 /* LSE supported */ +#define ID_AA64ISAR0_ATOMIC_LSE128 0x3 /* LSE + LSE128 supported */ +#define ID_AA64ISAR0_RDM_NONE 0x0 /* RDM not supported */ +#define ID_AA64ISAR0_RDM_RDM 0x1 /* RDM supported */ +#define ID_AA64ISAR0_SHA3_NONE 0x0 /* SHA3 not supported */ +#define ID_AA64ISAR0_SHA3_SHA3 0x1 /* SHA3 supported */ +#define ID_AA64ISAR0_SM3_NONE 0x0 /* SM3 not supported */ +#define ID_AA64ISAR0_SM3_SM3 0x1 /* SM3 supported */ +#define ID_AA64ISAR0_SM4_NONE 0x0 /* SM4 not supported */ +#define ID_AA64ISAR0_SM4_SM4 0x1 /* SM4 supported */ +#define ID_AA64ISAR0_DP_NONE 0x0 /* DP not supported */ +#define ID_AA64ISAR0_DP_DP 0x1 /* DP supported */ +#define ID_AA64ISAR0_FHM_NONE 0x0 /* FHM not supported */ +#define ID_AA64ISAR0_FHM_FHM 0x1 /* FHM supported */ +#define ID_AA64ISAR0_TS_NONE 0x0 /* FLAGM not supported */ +#define ID_AA64ISAR0_TS_FLAGM 0x1 /* FLAGM supported */ +#define ID_AA64ISAR0_TS_FLAGM2 0x2 /* FLAGM + FLAGM2 supported */ +#define ID_AA64ISAR0_RNDR_NONE 0x0 /* RNDR not supported */ +#define ID_AA64ISAR0_RNDR_RNDR 0x1 /* RNDR supported */ + +#define ID_AA64ISAR1_DPB(v) ((v) & 0xf) +#define ID_AA64ISAR1_APA(v) (((v) >> 4) & 0xf) +#define ID_AA64ISAR1_API(v) (((v) >> 8) & 0xf) +#define ID_AA64ISAR1_JSCVT(v) (((v) >> 12) & 0xf) +#define ID_AA64ISAR1_FCMA(v) (((v) >> 16) & 0xf) +#define ID_AA64ISAR1_LRCPC(v) (((v) >> 20) & 0xf) +#define ID_AA64ISAR1_GPA(v) (((v) >> 24) & 0xf) +#define ID_AA64ISAR1_GPI(v) (((v) >> 28) & 0xf) +#define ID_AA64ISAR1_FRINTTS(v) (((v) >> 32) & 0xf) +#define ID_AA64ISAR1_SB(v) (((v) >> 36) & 0xf) +#define ID_AA64ISAR1_BF16(v) (((v) >> 44) & 0xf) +#define ID_AA64ISAR1_DGH(v) (((v) >> 48) & 0xf) +#define ID_AA64ISAR1_I8MM(v) (((v) >> 52) & 0xf) + +#define ID_AA64ISAR1_DPB_NONE 0x0 /* DPB not supported */ +#define ID_AA64ISAR1_DPB_DPB 0x1 /* DPB supported */ +#define ID_AA64ISAR1_DPB_DPB2 0x2 /* DPB + DPB2 supported */ +#define ID_AA64ISAR1_APA_NONE 0x0 /* PAC w/ QARMA5 not supported */ +#define ID_AA64ISAR1_APA_APA 0x1 /* PAC w/ QARMA5 supported */ +#define ID_AA64ISAR1_API_NONE 0x0 /* PAC w/ implementation-defined algo not supported */ +#define ID_AA64ISAR1_API_API 0x1 /* PAC w/ implementation-defined algo supported */ +#define ID_AA64ISAR1_JSCVT_NONE 0x0 /* JSCVT not supported */ +#define ID_AA64ISAR1_JSCVT_JSCVT 0x1 /* JSCVT supported */ +#define ID_AA64ISAR1_FCMA_NONE 0x0 /* FCMA not supported */ +#define ID_AA64ISAR1_FCMA_FCMA 0x1 /* FCMA supported */ +#define ID_AA64ISAR1_LRCPC_NONE 0x0 /* LRCPC not supported */ +#define ID_AA64ISAR1_LRCPC_LRCPC 0x1 /* LRCPC supported */ +#define ID_AA64ISAR1_LRCPC_LRCPC2 0x2 /* LRCPC + LRCPC2 supported */ +#define ID_AA64ISAR1_LRCPC_LRCPC3 0x3 /* LRCPC + LRCPC2 + LRCPC3 supported */ +#define ID_AA64ISAR1_GPA_NONE 0x0 /* PAC/GA w/ QARMA5 not supported */ +#define ID_AA64ISAR1_GPA_GPA 0x1 /* PAC/GA w/ QARMA5 supported */ +#define ID_AA64ISAR1_GPI_NONE 0x0 /* PAC/GA w/ implementation-defined algo not supported */ +#define ID_AA64ISAR1_GPI_GPI 0x1 /* PAC/GA w/ implementation-defined algo supported */ +#define ID_AA64ISAR1_FRINTTS_NONE 0x0 /* FRINT* not supported */ +#define ID_AA64ISAR1_FRINTTS_FRINTTS 0x1 /* FRINT* supported */ +#define ID_AA64ISAR1_SB_NONE 0x0 /* SB not supported */ +#define ID_AA64ISAR1_BF16_NONE 0x0 /* BFloat16 not supported */ +#define ID_AA64ISAR1_BF16_BF16 0x1 /* BFloat16 supported */ +#define ID_AA64ISAR1_BF16_EBF16 0x2 /* BFloat16 + EBF supported */ +#define ID_AA64ISAR1_DGH_NONE 0x0 /* DGH not supported */ +#define ID_AA64ISAR1_DGH_DGH 0x1 /* DGH supported */ +#define ID_AA64ISAR1_I8MM_NONE 0x0 /* Int8 matrix multiplication not supported */ +#define ID_AA64ISAR1_I8MM_I8MM 0x1 /* Int8 matrix multiplication supported */ + +/* FIXME: is this ASID, not VMID? */ +#define ID_AA64MMFR1_ASID(v) (((v) >> 4) & 0xf) +#define ID_AA64MMFR1_VH(v) ((v)) >> 8) & 0xf) +#define ID_AA64MMFR1_PAN(v) (((v) >> 20) & 0xf) + +#define ID_AA64MMFR1_VH_SHIFT 8 + +#define ID_AA64MMFR1_ASID_8 0x0 /* 8-bit ASID */ +#define ID_AA64MMFR1_ASID_16 0x2 /* 16-bit ASID */ +#define ID_AA64MMFR1_VH_NONE 0x0 /* VHE not supported */ +#define ID_AA64MMFR1_VH_VHE 0x1 /* VHE supported */ +#define ID_AA64MMFR1_PAN_NONE 0x0 /* PAN not supported */ +#define ID_AA64MMFR1_PAN_PAN 0x1 /* PAN supported */ +#define ID_AA64MMFR1_PAN_PAN2 0x2 /* PAN + PAN2 supported */ +#define ID_AA64MMFR1_PAN_PAN3 0x3 /* PAN + PAN2 + PAN3 supported */ + +#define ID_AA64MMFR2_UAO(v) (((v) >> 4) & 0xf) +#define ID_AA64MMFR2_NV(v) (((v) >> 24) & 0xf) +#define ID_AA64MMFR2_AT(v) (((v) >> 32) & 0xf) + +#define ID_AA64MMFR2_UAO_NONE 0x0 /* UAO not supported */ +#define ID_AA64MMFR2_UAO_UAO 0x1 /* UAO supported */ +#define ID_AA64MMFR2_NV_NONE 0x0 /* NV not supported */ +#define ID_AA64MMFR2_NV_NV 0x1 /* NV supported */ +#define ID_AA64MMFR2_NV_NV2 0x2 /* NV + NV2 supported */ +#define ID_AA64MMFR2_AT_NONE 0x0 /* AT not supported */ +#define ID_AA64MMFR2_AT_AT 0x1 /* AT supported */ + +#endif /* _AARCH64_BITS_ID_AA64_ */ diff --git a/aarch64/aarch64/bits/mair.h b/aarch64/aarch64/bits/mair.h new file mode 100644 index 00000000..8225ccac --- /dev/null +++ b/aarch64/aarch64/bits/mair.h @@ -0,0 +1,12 @@ +#ifndef _AARCH64_BITS_MAIR_ +#define _AARCH64_BITS_MAIR_ + +#define MAIR_NORMAL_INDEX 0 +#define MAIR_NORMAL_FLAGS 0xff +#define MAIR_DEVICE_INDEX 1 +#define MAIR_DEVICE_FLAGS 0x00 + +#define MAIR_VALUE_ENTRY(index, flags) ((flags) << ((index) * 8)) +#define MAIR_VALUE (MAIR_VALUE_ENTRY(MAIR_NORMAL_INDEX, MAIR_NORMAL_FLAGS) | MAIR_VALUE_ENTRY(MAIR_DEVICE_INDEX, MAIR_DEVICE_FLAGS)) + +#endif /* _AARCH64_BITS_MAIR_ */ diff --git a/aarch64/aarch64/bits/pte.h b/aarch64/aarch64/bits/pte.h new file mode 100644 index 00000000..8f0d9659 --- /dev/null +++ b/aarch64/aarch64/bits/pte.h @@ -0,0 +1,33 @@ +#ifndef _AARCH64_BITS_PTE_ +#define _AARCH64_BITS_PTE_ + +/* PTE bits */ +#define AARCH64_PTE_ADDR_MASK 0x0000fffffffff000UL +#define AARCH64_PTE_PROT_MASK 0x00600000000000c0UL + +/* Block or table */ +#define AARCH64_PTE_BLOCK 0x0000000000000000UL /* points to a block of phys memory */ +#define AARCH64_PTE_TABLE 0x0000000000000002UL /* points to a next level table */ +#define AARCH64_PTE_LEVEL3 0x0000000000000002UL /* this is a level 3 PTE (same value as table) */ + +#define AARCH64_PTE_VALID 0x0000000000000001UL /* this entry is valid */ +#define AARCH64_PTE_NS 0x0000000000000020UL /* security bit (only EL3 & secure EL1) */ +#define AARCH64_PTE_ACCESS 0x0000000000000400UL /* if unset, trap on access */ +#define AARCH64_PTE_NG 0x0000000000000800UL /* tag TLB entries with ASID */ +#define AARCH64_PTE_BTI 0x0004000000000000UL /* enable branch target identification */ +#define AARCH64_PTE_CONTIG 0x0010000000000000UL /* hint that this is a part of contigous set */ +#define AARCH64_PTE_PXN 0x0020000000000000UL /* privileged execute never */ +#define AARCH64_PTE_UXN 0x0040000000000000UL /* unprivileged execute never */ + +#define AARCH64_PTE_MAIR_INDEX(i) ((i) << 2) /* cache policies, as an index into MAIR table */ + +/* Access permissions */ +#define AARCH64_PTE_EL0_ACCESS 0x0000000000000040UL /* EL0 can access (read or write, subject to READ_ONLY) */ +#define AARCH64_PTE_READ_ONLY 0x0000000000000080UL /* can not be written */ + +/* Shareability */ +#define AARCH64_PTE_NON_SH 0x0000000000000000UL /* non-shareable */ +#define AARCH64_PTE_OUTER_SH 0x0000000000000200UL /* outer shareable */ +#define AARCH64_PTE_INNER_SH 0x0000000000000300UL /* inner shareable */ + +#endif /* _AARCH64_BITS_PTE_ */ diff --git a/aarch64/aarch64/bits/sctlr.h b/aarch64/aarch64/bits/sctlr.h new file mode 100644 index 00000000..ed7c066b --- /dev/null +++ b/aarch64/aarch64/bits/sctlr.h @@ -0,0 +1,20 @@ +#ifndef _AARCH64_BITS_SCTLR_ +#define _AARCH64_BITS_SCTLR_ + +#define SCTLR_M 0x0000000000000001UL /* enable MMU */ +#define SCTLR_A 0x0000000000000002UL /* enable alignment checking */ +#define SCTLR_SA 0x0000000000000008UL /* enable SP alignment checking in EL1 */ +#define SCTLR_SA0 0x0000000000000010UL /* enable SP alignment checking in EL0 */ +#define SCTLR_ENDB 0x0000000000002000UL /* PAC */ +#define SCTLR_UCT 0x0000000000008000UL /* allow EL0 to access CTR_EL0 */ +#define SCTLR_SPAN 0x0000000000800000UL /* don't set psate.PAN upon an exception to EL1 */ +#define SCTLR_UCI 0x0000000004000000UL /* allow EL0 to issue cache maintenance instructions */ +#define SCTLR_ENDA 0x0000000008000000UL /* PAC */ +#define SCTLR_ENIB 0x0000000040000000UL /* PAC */ +#define SCTLR_ENIA 0x0000000080000000UL /* PAC */ +#define SCTLR_BT0 0x0000000800000000UL /* PACIASP/PACIBSP does not act like BTI JC in EL0 */ +#define SCTLR_BT1 0x0000001000000000UL /* PACIASP/PACIBSP does not act like BTI JC in EL1 */ +#define SCTLR_SSBS 0x0000100000000000UL /* set SSBS to 1 on exception to EL1 (otherwise, to 0) */ +#define SCTLR_EPAN 0x0200000000000000UL /* enable EPAN */ + +#endif /* _AARCH64_BITS_SCTLR_ */ diff --git a/aarch64/aarch64/bits/spsr.h b/aarch64/aarch64/bits/spsr.h new file mode 100644 index 00000000..7b4c45df --- /dev/null +++ b/aarch64/aarch64/bits/spsr.h @@ -0,0 +1,42 @@ +#ifndef _AARCH64_BITS_SPSR_ +#define _AARCH64_BITS_SPSR_ + +#define SPSR_SPSEL(spsr) ((spsr) & 0x1) /* select sp: */ +#define SPSR_SPSEL_0 0x00000000 /* ...sp = SP_EL0 */ +#define SPSR_SPSEL_N 0x00000001 /* ...sp = SP_ELn */ + +#define SPSR_EL(spsr) (((spsr) & 0xc) >> 2) /* exception level, 0 to 3 */ +#define SPSR_MAKE_EL(el) ((el) << 2) + +#define SPSR_NRW(spsr) ((spsr) & 0x10) /* "not register width": */ +#define SPSR_NRW_64 0x00000000 /* ...AArch64 */ +#define SPSR_NRW_32 0x00000010 /* ...AArch32 */ + /* a reserved bit here */ +#define SPSR_F 0x00000040 /* FIQ masked */ +#define SPSR_I 0x00000080 /* IRQ masked */ +#define SPSR_A 0x00000100 /* SError masked */ +#define SPSR_D 0x00000200 /* debug exceptions masked */ +#define SPSR_DAIF (SPSR_D | SPSR_A | SPSR_I | SPSR_F) +#define SPSR_AIF (SPSR_A | SPSR_I | SPSR_F) + +#define SPSR_BTYPE_MASK 0x00000c00 /* branch type indicator (BTI) */ + +#define SPSR_SSBS 0x00001000 /* speculative store bypass safe */ +#define SPSR_ALLINT 0x00002000 /* IRQ and FIQ masked */ + /* reserved bits here */ +#define SPSR_IL 0x00100000 /* illegal execution state */ +#define SPSR_SS 0x00200000 /* software single step */ +#define SPSR_PAN 0x00400000 /* privileged access never */ +#define SPSR_UAO 0x00800000 /* user access override */ +#define SPSR_DIT 0x01000000 /* data independent timing */ +#define SPSR_TCO 0x02000000 /* tag check override (MTE) */ + /* reserved bits here */ +#define SPSR_V 0x10000000 /* overflow condition */ +#define SPSR_C 0x20000000 /* carry condition */ +#define SPSR_Z 0x40000000 /* zero condition */ +#define SPSR_N 0x80000000 /* negative condition */ +#define SPSR_NZCV (SPSR_N | SPSR_Z | SPSR_C | SPSR_V) + +#define SPSR_RES0 0xffffffff080fc020 + +#endif /* _AARCH64_BITS_SPSR_ */ diff --git a/aarch64/aarch64/bits/tcr.h b/aarch64/aarch64/bits/tcr.h new file mode 100644 index 00000000..52d9065d --- /dev/null +++ b/aarch64/aarch64/bits/tcr.h @@ -0,0 +1,18 @@ +#ifndef _AARCH64_BITS_TCR_ +#define _AARCH64_BITS_TCR_ + +#define TCR_A1 0x0000000000400000UL /* if set, TTBR1 defines ASID, otherwise TTBR0 */ + +#define TCR_T0SZ(size) (64 - size) +#define TCR_TG0_4K 0x0000000000000000UL +#define TCR_TG0_64K 0x0000000000004000UL +#define TCR_TG0_16K 0x0000000000008000UL + +#define TCR_T1SZ(size) ((64 - size) << 16) +#define TCR_TG1_16K 0x0000000040000000UL +#define TCR_TG1_4K 0x0000000080000000UL +#define TCR_TG1_64K 0x00000000c0000000UL + +#define TCR_VALUE (TCR_T0SZ(VM_AARCH64_T0SZ) | TCR_TG0_4K | TCR_T1SZ(VM_AARCH64_T1SZ) | TCR_TG1_4K) + +#endif /* _AARCH64_BITS_TCR_ */ diff --git a/aarch64/aarch64/boot.S b/aarch64/aarch64/boot.S new file mode 100644 index 00000000..85d3b944 --- /dev/null +++ b/aarch64/aarch64/boot.S @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024 Free Software Foundation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include <mach/machine/asm.h> +#include "aarch64/bits/spsr.h" +#include "aarch64/bits/hcr.h" +#include "aarch64/bits/id_aa64.h" + + .section .text.boot +.boot_header: + /* See https://docs.kernel.org/arch/arm64/booting.html#call-the-kernel-image */ + b EXT(_start) + nop + .dword 0 /* ??? */ + .dword EXT(_image_end) - .boot_header + .dword 1 + .dword 0 + .dword 0 + .dword 0 + .dword 0x644d5241 + .dword 0 + +ENTRY(_start) + /* + * See what EL we're entered in. + * + * If it's EL1, there's nothing special to do, and we can proceed. + * + * If it's EL2, see if FEAT_VHE is supported, and if so, enable + * E2H, and proceed booting in EL2. If it's unsupported, drop + * to AArch64 EL1 and boot there. + * + * If it's EL3, drop down to EL2 and do the same things. + */ + mrs x4, CurrentEL + cmp x4, #(SPSR_MAKE_EL(1)) + b.eq 2f + + cmp x4, #(SPSR_MAKE_EL(2)) + b.eq 1f + + /* Must be in EL3, drop down to EL2. */ + adr x5, 1f + msr ELR_EL3, x5 + mov x6, #(SPSR_DAIF | SPSR_MAKE_EL(2) | SPSR_SPSEL_N) + msr SPSR_EL3, x6 + eret + +0: + /* EL2 without VHE, drop down to EL1. */ + mov x7, #(HCR_RW) + msr HCR_EL2, x7 + adr x5, 2f + msr ELR_EL2, x5 + mov x6, #(SPSR_DAIF | SPSR_MAKE_EL(1) | SPSR_SPSEL_N) + msr SPSR_EL2, x6 + eret + +1: + +#define HCR_VALUE (HCR_E2H | HCR_RW | HCR_TWI | HCR_TWE | HCR_TGE) + + /* EL2, see if FEAT_VHE is supported. */ + mrs x2, ID_AA64MMFR1_EL1 + tbz x2, #(ID_AA64MMFR1_VH_SHIFT), 0b + /* Enable E2H. */ + mov x5, #(HCR_VALUE & 0xffff) + movk x5, #((HCR_VALUE >> 16) & 0xffff), lsl #16 + movk x5, #(HCR_VALUE >> 32), lsl #32 + msr HCR_EL2, x5 + isb + +2: + adr x1, .boot_stack_end + mov sp, x1 + + b EXT(c_boot_entry) +END(_start) + + .bss +.boot_stack: + .space 4096 +.boot_stack_end: + + .section .note.GNU-stack,"",%progbits diff --git a/aarch64/aarch64/conf.c b/aarch64/aarch64/conf.c new file mode 100644 index 00000000..00d374eb --- /dev/null +++ b/aarch64/aarch64/conf.c @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <device/conf.h> +#include <kern/mach_clock.h> +#include "aarch64/model_dep.h" +#ifdef MACH_KMSG +#include <device/kmsg.h> +#endif +#include <device/ramdisk.h> + +struct dev_ops dev_name_list[] = +{ + { "cn", nulldev_open, nulldev_close, nulldev_read, + nulldev_write, nulldev_getstat, nulldev_setstat, nomap, + nodev_async_in, nulldev_reset, nulldev_portdeath, 0, + nodev_info}, + { "time", timeopen, timeclose, nulldev_read, + nulldev_write, nulldev_getstat, nulldev_setstat, timemmap, + nodev_async_in, nulldev_reset, nulldev_portdeath, 0, + nodev_info}, + { "mem", nulldev_open, nulldev_close, nulldev_read, + nulldev_write, nulldev_getstat, nulldev_setstat, memmmap, + nodev_async_in, nulldev_reset, nulldev_portdeath, 0, + nodev_info }, +#ifdef MACH_KMSG + { "kmsg", kmsgopen, kmsgclose, kmsgread, + nulldev_write, kmsggetstat, nulldev_setstat, nomap, + nodev_async_in, nulldev_reset, nulldev_portdeath, 0, + nodev_info }, +#endif + RAMDISK_DEV_OPS, +}; +int dev_name_count = sizeof(dev_name_list) / sizeof(dev_name_list[0]); + +struct dev_indirect dev_indirect_list[] = +{ + { "console", &dev_name_list[0], 0 } +}; +int dev_indirect_count = sizeof(dev_indirect_list) / sizeof(dev_indirect_list[0]); diff --git a/aarch64/aarch64/cpu_number.h b/aarch64/aarch64/cpu_number.h new file mode 100644 index 00000000..df4dec0f --- /dev/null +++ b/aarch64/aarch64/cpu_number.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_CPU_NUMBER_H_ +#define _AARCH64_CPU_NUMBER_H_ + +#if NCPUS > 1 + +#ifndef __ASSEMBLER__ +static inline int cpu_number(void) +{ + int mycpu; + + asm("mrs %0, TPIDRRO_EL0" : "=r"(mycpu)); + return mycpu; +} +#endif + +#endif + +#endif /* _AARCH64_CPU_NUMBER_H_ */ diff --git a/aarch64/aarch64/cswitch.S b/aarch64/aarch64/cswitch.S new file mode 100644 index 00000000..02acca39 --- /dev/null +++ b/aarch64/aarch64/cswitch.S @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <mach/machine/asm.h> +#include "aarch64/aarch64asm.h" +#include "aarch64/vm_param.h" + +ENTRY(load_context) + ldr x1, [x0, #TH_KERNEL_STACK] /* load kernel stack bottom */ + mov x2, #(KERNEL_STACK_SIZE-AKS_SIZE-AEL_SIZE) + add x2, x1, x2 /* find kernel stack top */ + adr x0, EXT(kernel_stack) + str x2, [x0] /* store kernel stack top */ + mrs x2, TPIDR_EL1 + str x1, [x2, #PERCPU_ACTIVE_STACK] /* store stack bottom */ + mov x2, #KSS_REGS /* load registers... */ + add x1, x1, x2 /* ...relative to stack bottom */ + ldp x19, x20, [x1], #16 + ldp x21, x22, [x1], #16 + ldp x23, x24, [x1], #16 + ldp x25, x26, [x1], #16 + ldp x27, x28, [x1], #16 + ldp x29, x30, [x1], #16 + ldr x0, [x1] + mov sp, x0 + mov x0, #0 /* no old thread */ + ret +END(load_context) + +ENTRY(Switch_context) + mrs x4, TPIDR_EL1 + ldr x6, [x4, #PERCPU_ACTIVE_STACK] /* load stack bottom */ + mov x5, #KSS_REGS + add x5, x6, x5 + + stp x19, x20, [x5], #16 /* save registers */ + stp x21, x22, [x5], #16 + stp x23, x24, [x5], #16 + stp x25, x26, [x5], #16 + stp x27, x28, [x5], #16 + stp x29, x30, [x5], #16 + mov x3, sp + str x3, [x5] + + str x6, [x0, #TH_KERNEL_STACK] /* save stack bottom */ + str x1, [x0, #TH_SWAP_FUNC] /* save continuation */ + ldr x7, [x2, #TH_KERNEL_STACK] /* load new stack bottom */ + mov x5, #(KERNEL_STACK_SIZE-AKS_SIZE-AEL_SIZE) + add x8, x7, x5 /* find new stack top */ + + str x2, [x4, #PERCPU_ACTIVE_THREAD] /* update current_thread() */ + str x7, [x4, #PERCPU_ACTIVE_STACK] /* store stack bottom */ + adr x4, EXT(kernel_stack) + str x8, [x4] /* store stack top */ + + mov x5, #KSS_REGS + add x5, x7, x5 + + ldp x19, x20, [x5], #16 + ldp x21, x22, [x5], #16 + ldp x23, x24, [x5], #16 + ldp x25, x26, [x5], #16 + ldp x27, x28, [x5], #16 + ldp x29, x30, [x5], #16 + ldr x2, [x5] + mov sp, x2 + + ret +END(Switch_context) + +ENTRY(Thread_continue) + mov x29, #0 + mov x30, #0 +#ifdef __ARM_FEATURE_BTI_DEFAULT + mov x16, x19 + br x16 +#else + br x19 +#endif +END(Thread_continue) + + .section .note.GNU-stack,"",%progbits diff --git a/aarch64/aarch64/db_machdep.h b/aarch64/aarch64/db_machdep.h new file mode 100644 index 00000000..836940de --- /dev/null +++ b/aarch64/aarch64/db_machdep.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_DB_MACHDEP_H_ +#define _AARCH64_DB_MACHDEP_H_ + +typedef vm_offset_t db_addr_t; /* address - unsigned */ +typedef long db_expr_t; /* expression - signed */ + +#endif /* _AARCH64_DB_MACHDEP_H_ */ diff --git a/aarch64/aarch64/fpu.c b/aarch64/aarch64/fpu.c new file mode 100644 index 00000000..1b8b345f --- /dev/null +++ b/aarch64/aarch64/fpu.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/fpu.h" +#include "aarch64/locore.h" +#include <kern/slab.h> +#include <string.h> + +#define FPEN_TRAP_MASK 0x300000 +#define FPEN_TRAP_EL01 0x000000 /* trap FP at EL0 & EL1 */ +#define FPEN_TRAP_EL0 0x100000 /* trap FP at EL0 */ +#define FPEN_TRAP_NONE 0x300000 /* don't trap FP */ + +static struct kmem_cache fpu_cache; + +void fpu_init(void) +{ + kmem_cache_init(&fpu_cache, "fpu", sizeof(struct aarch64_float_state), alignof(struct aarch64_float_state), NULL, 0); +} + +static void fpen(boolean_t enable) +{ + uint64_t cpacr; + + asm("mrs %0, CPACR_EL1" : "=r"(cpacr)); + + if (enable) + cpacr = (cpacr & ~FPEN_TRAP_MASK) | FPEN_TRAP_NONE; + else + cpacr = (cpacr & ~FPEN_TRAP_MASK) | FPEN_TRAP_EL01; + + asm volatile("msr CPACR_EL1, %0" :: "r"(cpacr)); + asm volatile("isb"); +} + +void fpu_switch_context(thread_t new) +{ + thread_t fpu_thread; + + fpu_thread = percpu_get(thread_t, fpu_thread); + fpen(new == fpu_thread); +} + +static kern_return_t fpu_alloc(pcb_t pcb) +{ + if (pcb->afs) + return KERN_SUCCESS; + + pcb->afs = (struct aarch64_float_state *) kmem_cache_alloc(&fpu_cache); + if (!pcb->afs) + return KERN_RESOURCE_SHORTAGE; + return KERN_SUCCESS; +} + +static void fpu_save_state(void) +{ + thread_t fpu_thread; + pcb_t pcb; + + fpu_thread = percpu_get(thread_t, fpu_thread); + if (fpu_thread == THREAD_NULL) + return; + + pcb = fpu_thread->pcb; + fpu_alloc(pcb); + + _fpu_save_state(pcb->afs); +} + +static void fpu_load_state(void) +{ + thread_t thread = current_thread(); + pcb_t pcb; + + pcb = thread->pcb; + if (pcb->afs == NULL) { + fpu_alloc(pcb); + memset(pcb->afs, 0, sizeof(struct aarch64_float_state)); + } + + _fpu_load_state(pcb->afs); + percpu_assign(fpu_thread, thread); +} + +void fpu_access_trap(void) +{ + fpen(TRUE); + fpu_save_state(); + fpu_load_state(); +} + +void fpu_flush_state_read(thread_t thread) +{ + thread_t fpu_thread; + +#if NCUPS > 1 +#error "Implement" +#endif + fpu_thread = percpu_get(thread_t, fpu_thread); + if (thread != fpu_thread) + return; + + fpen(TRUE); + fpu_save_state(); + if (thread != current_thread()) + fpen(FALSE); +} + +void fpu_flush_state_write(thread_t thread) +{ + thread_t fpu_thread; + + fpu_alloc(thread->pcb); + +#if NCUPS > 1 +#error "Implement" +#endif + fpu_thread = percpu_get(thread_t, fpu_thread); + if (thread != fpu_thread) + return; + + percpu_assign(fpu_thread, THREAD_NULL); + fpen(FALSE); +} + +void fpu_free(thread_t thread) +{ + thread_t fpu_thread; + + fpu_thread = percpu_get(thread_t, fpu_thread); + + if (thread->pcb->afs == NULL) { + assert(fpu_thread != thread); + return; + } +#if NCUPS > 1 +#error "Implement" +#endif + if (fpu_thread == thread) + percpu_assign(fpu_thread, THREAD_NULL); + kmem_cache_free(&fpu_cache, (vm_offset_t) thread->pcb->afs); +} diff --git a/aarch64/aarch64/fpu.h b/aarch64/aarch64/fpu.h new file mode 100644 index 00000000..05733bae --- /dev/null +++ b/aarch64/aarch64/fpu.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <kern/thread.h> + +extern void fpu_init(void); +extern void fpu_switch_context(thread_t new); +extern void fpu_access_trap(void); +extern void fpu_free(thread_t thread); + +extern void fpu_flush_state_read(thread_t thread); +extern void fpu_flush_state_write(thread_t thread); diff --git a/aarch64/aarch64/hwcaps.c b/aarch64/aarch64/hwcaps.c new file mode 100644 index 00000000..2e2ac395 --- /dev/null +++ b/aarch64/aarch64/hwcaps.c @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/hwcaps.h" +#include "aarch64/bits/id_aa64.h" + +/* https://docs.kernel.org/arch/arm64/elf_hwcaps.html */ + +uint32_t hwcaps[HWCAPS_COUNT]; +uint32_t hwcap_internal; + +void hwcaps_init(void) +{ + uint64_t id_aa64pfr0; + uint64_t id_aa64pfr1; + uint64_t id_aa64isar0; + uint64_t id_aa64isar1; + uint64_t id_aa64mmfr1; + uint64_t id_aa64mmfr2; + + asm("mrs %0, id_aa64pfr0_el1" : "=r"(id_aa64pfr0)); + asm("mrs %0, id_aa64pfr1_el1" : "=r"(id_aa64pfr1)); + asm("mrs %0, id_aa64isar0_el1" : "=r"(id_aa64isar0)); + asm("mrs %0, id_aa64isar1_el1" : "=r"(id_aa64isar1)); + asm("mrs %0, id_aa64mmfr1_el1" : "=r"(id_aa64mmfr1)); + asm("mrs %0, id_aa64mmfr2_el1" : "=r"(id_aa64mmfr2)); + + if (ID_AA64PFR0_FP(id_aa64pfr0) != ID_AA64PFR0_FP_NONE) + hwcaps[0] |= HWCAP_FP; + if (ID_AA64PFR0_FP(id_aa64pfr0) == ID_AA64PFR0_FP_FP16) + hwcaps[0] |= HWCAP_FPHP; + if (ID_AA64PFR0_ASIMD(id_aa64pfr0) != ID_AA64PFR0_ASIMD_NONE) + hwcaps[0] |= HWCAP_ASIMD; + if (ID_AA64PFR0_ASIMD(id_aa64pfr0) == ID_AA64PFR0_ASIMD_FP16) + hwcaps[0] |= HWCAP_ASIMDHP; + /* HWCAP_EVTSTRM? */ + if (ID_AA64ISAR0_AES(id_aa64isar0) == ID_AA64ISAR0_AES_AES) + hwcaps[0] |= HWCAP_AES; + else if (ID_AA64ISAR0_AES(id_aa64isar0) == ID_AA64ISAR0_AES_PMULL) + hwcaps[0] |= HWCAP_AES | HWCAP_PMULL; + if (ID_AA64ISAR0_SHA1(id_aa64isar0) != ID_AA64ISAR0_SHA1_NONE) + hwcaps[0] |= HWCAP_SHA1; + if (ID_AA64ISAR0_SHA2(id_aa64isar0) != ID_AA64ISAR0_SHA2_NONE) + hwcaps[0] |= HWCAP_SHA2; + if (ID_AA64ISAR0_SHA2(id_aa64isar0) == ID_AA64ISAR0_SHA2_SHA512) + hwcaps[0] |= HWCAP_SHA512; + if (ID_AA64ISAR0_SHA2(id_aa64isar0) != ID_AA64ISAR0_CRC32_NONE) + hwcaps[0] |= HWCAP_CRC32; + if (ID_AA64ISAR0_ATOMIC(id_aa64isar0) == ID_AA64ISAR0_ATOMIC_LSE) + hwcaps[0] |= HWCAP_ATOMICS; + else if (ID_AA64ISAR0_ATOMIC(id_aa64isar0) == ID_AA64ISAR0_ATOMIC_LSE128) { + hwcaps[0] |= HWCAP_ATOMICS; + hwcaps[1] |= HWCAP2_LSE128; + } + /* HWCAP_CPUID not set */ + if (ID_AA64ISAR0_RDM(id_aa64isar0) != ID_AA64ISAR0_RDM_NONE) + hwcaps[0] |= HWCAP_ASIMDRDM; + if (ID_AA64ISAR1_JSCVT(id_aa64isar1) != ID_AA64ISAR1_JSCVT_NONE) + hwcaps[0] |= HWCAP_JSCVT; + if (ID_AA64ISAR1_FCMA(id_aa64isar1) != ID_AA64ISAR1_FCMA_NONE) + hwcaps[0] |= HWCAP_FCMA; + if (ID_AA64ISAR1_LRCPC(id_aa64isar1) != ID_AA64ISAR1_LRCPC_NONE) + hwcaps[0] |= HWCAP_LRCPC; + if (ID_AA64ISAR1_LRCPC(id_aa64isar1) == ID_AA64ISAR1_LRCPC_LRCPC2) + hwcaps[0] |= HWCAP_ILRCPC; + else if (ID_AA64ISAR1_LRCPC(id_aa64isar1) == ID_AA64ISAR1_LRCPC_LRCPC3) { + hwcaps[0] |= HWCAP_ILRCPC; + hwcaps[1] |= HWCAP2_LRCPC3; + } + if (ID_AA64ISAR1_DPB(id_aa64isar1) != ID_AA64ISAR1_DPB_NONE) + hwcaps[0] |= HWCAP_DCPOP; + if (ID_AA64ISAR1_DPB(id_aa64isar1) != ID_AA64ISAR1_DPB_DPB2) + hwcaps[1] |= HWCAP2_DCPODP; + if (ID_AA64ISAR0_SHA3(id_aa64isar0) != ID_AA64ISAR0_SHA3_NONE) + hwcaps[0] |= HWCAP_SHA3; + if (ID_AA64ISAR0_SM3(id_aa64isar0) != ID_AA64ISAR0_SM3_NONE) + hwcaps[0] |= HWCAP_SM3; + if (ID_AA64ISAR0_SM4(id_aa64isar0) != ID_AA64ISAR0_SM4_NONE) + hwcaps[0] |= HWCAP_SM4; + if (ID_AA64ISAR0_DP(id_aa64isar0) != ID_AA64ISAR0_DP_NONE) + hwcaps[0] |= HWCAP_ASIMDDP; + /* + if (ID_AA64PFR0_SVE(id_aa64pfr0) != ID_AA64PFR0_SVE_NONE) + hwcaps[0] |= HWCAP_SVE; + */ + if (ID_AA64ISAR0_FHM(id_aa64isar0) != ID_AA64ISAR0_FHM_NONE) + hwcaps[0] |= HWCAP_ASIMDFHM; + if (ID_AA64PFR0_DIT(id_aa64pfr0) != ID_AA64PFR0_DIT_NONE) + hwcaps[0] |= HWCAP_DIT; + if (ID_AA64MMFR2_AT(id_aa64mmfr2) != ID_AA64MMFR2_AT_NONE) + hwcaps[0] |= HWCAP_USCAT; + if (ID_AA64ISAR0_TS(id_aa64isar0) == ID_AA64ISAR0_TS_FLAGM) + hwcaps[0] |= HWCAP_FLAGM; + else if (ID_AA64ISAR0_TS(id_aa64isar0) == ID_AA64ISAR0_TS_FLAGM2) { + hwcaps[0] |= HWCAP_FLAGM; + hwcaps[1] |= HWCAP2_FLAGM2; + } + if (ID_AA64PFR1_SSBS(id_aa64pfr1) != ID_AA64PFR1_SSBS_NONE) + hwcaps[0] |= HWCAP_SSBS; + if (ID_AA64ISAR1_SB(id_aa64isar1) != ID_AA64ISAR1_SB_NONE) + hwcaps[0] |= HWCAP_SB; + if (ID_AA64ISAR1_APA(id_aa64isar1) != ID_AA64ISAR1_APA_NONE) + hwcaps[0] |= HWCAP_PACA; + if (ID_AA64ISAR1_API(id_aa64isar1) != ID_AA64ISAR1_API_NONE) + hwcaps[0] |= HWCAP_PACA; + if (ID_AA64ISAR1_GPA(id_aa64isar1) != ID_AA64ISAR1_GPA_NONE) + hwcaps[0] |= HWCAP_PACG; + if (ID_AA64ISAR1_GPI(id_aa64isar1) != ID_AA64ISAR1_GPI_NONE) + hwcaps[0] |= HWCAP_PACG; + + /* + * SME/SVE are not exposed to userland for now + * even if hardware supports them, due to lack + * of kernels-side support. + */ + + if (ID_AA64ISAR1_FRINTTS(id_aa64isar1) != ID_AA64ISAR1_FRINTTS_NONE) + hwcaps[1] |= HWCAP2_FRINT; + if (ID_AA64ISAR1_I8MM(id_aa64isar1) != ID_AA64ISAR1_I8MM_NONE) + hwcaps[1] |= HWCAP2_I8MM; + if (ID_AA64ISAR1_BF16(id_aa64isar1) == ID_AA64ISAR1_BF16_BF16) + hwcaps[1] |= HWCAP2_BF16; + else if (ID_AA64ISAR1_BF16(id_aa64isar1) == ID_AA64ISAR1_BF16_EBF16) + hwcaps[1] |= HWCAP2_BF16 | HWCAP2_EBF16; + if (ID_AA64ISAR1_DGH(id_aa64isar1) != ID_AA64ISAR1_DGH_NONE) + hwcaps[1] |= HWCAP2_DGH; + if (ID_AA64ISAR0_RNDR(id_aa64isar0) != ID_AA64ISAR0_RNDR_NONE) + hwcaps[1] |= HWCAP2_RNG; + if (ID_AA64PFR1_BT(id_aa64pfr1) != ID_AA64PFR1_BT_NONE) + hwcaps[1] |= HWCAP2_BTI; + + /* to be continued... */ + + if (ID_AA64MMFR1_PAN(id_aa64mmfr1) != ID_AA64MMFR1_PAN_NONE) { + /* + * PAN is supported, enable it. + */ + asm volatile(".word 0xd500419f"); /* msr PAN, #1 */ + hwcap_internal |= HWCAP_INT_PAN; + } + if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_PAN3) + hwcap_internal |= HWCAP_INT_EPAN; + if (ID_AA64MMFR1_ASID(id_aa64mmfr1) == ID_AA64MMFR1_ASID_16) + hwcap_internal |= HWCAP_INT_ASID16; + if (ID_AA64MMFR2_UAO(id_aa64mmfr2) != ID_AA64MMFR2_UAO_NONE) + hwcap_internal |= HWCAP_INT_UAO; + if (ID_AA64MMFR2_NV(id_aa64mmfr2) >= ID_AA64MMFR2_NV_NV2) + hwcap_internal |= HWCAP_INT_NV2; +} diff --git a/aarch64/aarch64/hwcaps.h b/aarch64/aarch64/hwcaps.h new file mode 100644 index 00000000..7d119ccd --- /dev/null +++ b/aarch64/aarch64/hwcaps.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <mach/machine/mach_aarch64_types.h> + +extern uint32_t hwcaps[HWCAPS_COUNT]; + +extern uint32_t hwcap_internal; + +#define HWCAP_INT_PAN 0x01 /* privileged access never */ +#define HWCAP_INT_EPAN 0x02 /* extended privileged access never */ +#define HWCAP_INT_ASID16 0x04 /* 16-bit ASID */ +#define HWCAP_INT_UAO 0x08 /* user access override */ +#define HWCAP_INT_NV2 0x10 /* nested virtualization v2 */ + +extern void hwcaps_init(void); diff --git a/aarch64/aarch64/ipl.h b/aarch64/aarch64/ipl.h new file mode 100644 index 00000000..daa53574 --- /dev/null +++ b/aarch64/aarch64/ipl.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_IPL_H_ +#define _AARCH64_IPL_H_ + +#include "aarch64/spl.h" + +#endif /* _AARCH64_IPL_H_ */ diff --git a/aarch64/aarch64/irq.c b/aarch64/aarch64/irq.c new file mode 100644 index 00000000..4e5223b7 --- /dev/null +++ b/aarch64/aarch64/irq.c @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/irq.h" + +struct irq_src *root_irq_src; diff --git a/aarch64/aarch64/irq.h b/aarch64/aarch64/irq.h new file mode 100644 index 00000000..bfc3472f --- /dev/null +++ b/aarch64/aarch64/irq.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_IRQ_ +#define _AARCH64_IRQ_ + +#include <device/dtb.h> + +/* Legacy intr cruft to make device/intr.h compile, do not actually use this */ +#ifdef __INTR_H__ + +#define NINTR 1234 + +typedef unsigned int irq_t; + +// FIXME move to inter.h +extern struct irqdev irqtab; + +// FIXME does it make sense on aarch64? +extern int iunit[]; +typedef void (*interrupt_handler_fn)(int); +extern interrupt_handler_fn ivect[]; + +void intnull(int unit); + +void __enable_irq (irq_t irq); +void __disable_irq (irq_t irq); + +extern void unmask_irq (unsigned int irq_nr); + +#endif /* device/intr.h legacy */ + + +struct irq_src { + void (*handle_irq)(struct irq_src *); +}; + +typedef enum { + IRQ_DESC_TYPE_DT = 0, +} irq_desc_type; + +struct irq_desc { + irq_desc_type type; +}; + +struct irq_desc_dt { + irq_desc_type type; + dtb_prop_t prop; +}; + +struct irq_ctlr { + void (*add_src)(struct irq_ctlr *ctlr, + struct irq_src *src, + const struct irq_desc *desc); +}; + +extern struct irq_src *root_irq_src; + +#endif /* _AARCH64_IRQ_ */ diff --git a/aarch64/aarch64/locore.S b/aarch64/aarch64/locore.S new file mode 100644 index 00000000..008e39cc --- /dev/null +++ b/aarch64/aarch64/locore.S @@ -0,0 +1,829 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "mach/machine/asm.h" +#include "aarch64/aarch64asm.h" +#include "aarch64/cpu_number.h" +#include "aarch64/vm_param.h" +#include <mach/kern_return.h> +#include <mach/exception.h> + +/* + * The top (highest addresses) of each kernel stack contain: + * + * struct aarch64_exception_link; + * struct aarch64_kernel_state; + * + * where aarch64_exception_link contains a single pointer into + * the PCB. + */ +#define GET_PCB_STACK(reg) \ + mov reg, sp ;\ + orr reg, reg, #(KERNEL_STACK_SIZE - 1) ;\ + ldr reg, [reg, #(1 - AKS_SIZE - AEL_SIZE)] + +/* 3 instructions (12 bytes) long */ +#define SWITCH_TO_KERNEL_STACK(scratch) \ + adr scratch, EXT(kernel_stack) ;\ + ldr scratch, [scratch] ;\ + mov sp, scratch + +/* + * We just caught an exception or an interrupt from EL0; this + * saved pc -> ELR_EL1, replacing it with the handler's address, + * and sp -> SP_EL0, replacing it with the previous EL1 stack + * pointer (SP_EL1). The other general-purpose registers are as + * they have been in EL0. We want to save them into memory, to + * be able to restore them back later (in thread_exception_return). + * + * The way we achieve this is we leave SP_EL1 pointing right to + * "struct aarch64_thread_state ats" member of the PCB; this is + * referred to as running on "the PCB stack", even though you + * cannot use PCB as a stack otherwise, e.g. you cannot call C + * code while on the PCB stack. + */ + +/* 25 instructions (100 bytes) long */ +#define SAVE_EL0_STATE() \ + stp x0, x1, [sp, #(ATS_X)] ;\ + stp x2, x3, [sp, #(ATS_X + 16)] ;\ + stp x4, x5, [sp, #(ATS_X + 32)] ;\ + stp x6, x7, [sp, #(ATS_X + 48)] ;\ + stp x8, x9, [sp, #(ATS_X + 64)] ;\ + stp x10, x11, [sp, #(ATS_X + 80)] ;\ + stp x12, x13, [sp, #(ATS_X + 96)] ;\ + stp x14, x15, [sp, #(ATS_X + 112)] ;\ + stp x16, x17, [sp, #(ATS_X + 128)] ;\ + stp x18, x19, [sp, #(ATS_X + 144)] ;\ + stp x20, x21, [sp, #(ATS_X + 160)] ;\ + stp x22, x23, [sp, #(ATS_X + 176)] ;\ + stp x24, x25, [sp, #(ATS_X + 192)] ;\ + stp x26, x27, [sp, #(ATS_X + 208)] ;\ + stp x28, x29, [sp, #(ATS_X + 224)] ;\ + str x30, [sp, #(ATS_X + 240)] ;\ + ;\ + mrs x2, SP_EL0 ;\ + mrs x3, ELR_EL1 ;\ + stp x2, x3, [sp, #(ATS_SP)] ;\ + mrs x2, TPIDR_EL0 ;\ + mrs x3, SPSR_EL1 ;\ + stp x2, x3, [sp, #(ATS_TPIDR_EL0)] ;\ + /* + * ESR/FAR are not a part of aarch64_thread_state, + * but they are stored in the PCB immediately + * following the aarch64_thread_state. + */ ;\ + mrs x2, ESR_EL1 ;\ + mrs x3, FAR_EL1 ;\ + stp x2, x3, [sp, #(ATS_SIZE)] + +/* + * CFI / DWARF magic to let GDB know about aarch64_exception_link + * & aarch64_thread_state. This makes it possible to backtrace + * through an exception taken from EL0. + */ +#define DW_CFA_def_cfa_expression 0x0f + +#define DW_OP_const1u 0x08 +#define DW_OP_const2u 0x0a +#define DW_OP_const4u 0x0c +#define DW_OP_deref 0x06 +#define DW_OP_minus 0x1c +#define DW_OP_or 0x21 +#define DW_OP_reg(n) (0x50 + (n)) +#define DW_OP_breg(n) (0x70 + (n)) + +#if ((KERNEL_STACK_SIZE - 1) > 0xffff) || (AKS_SIZE + AEL_SIZE - 1 > 0xff) +#error "Update the DWARF bytecode below" +#endif + +#define CFI_PCB_EL0_STATE() \ + /* + * CFA = *( + * (sp | (KERNEL_STACK_SIZE - 1)) + * - (AKS_SIZE + AEL_SIZE - 1) + * ) + */ ;\ + .cfi_escape DW_CFA_def_cfa_expression, 10, \ + DW_OP_breg(31), 0, \ + DW_OP_const2u, \ + (KERNEL_STACK_SIZE - 1) & 0xff, \ + ((KERNEL_STACK_SIZE - 1) >> 8) & 0xff, \ + DW_OP_or, \ + DW_OP_const1u, AKS_SIZE + AEL_SIZE - 1, \ + DW_OP_minus, \ + DW_OP_deref ;\ + .cfi_offset 0, ATS_X ;\ + .cfi_offset 1, ATS_X + 8 ;\ + .cfi_offset 2, ATS_X + 16 ;\ + .cfi_offset 3, ATS_X + 24 ;\ + .cfi_offset 4, ATS_X + 32 ;\ + .cfi_offset 5, ATS_X + 40 ;\ + .cfi_offset 6, ATS_X + 48 ;\ + .cfi_offset 7, ATS_X + 56 ;\ + .cfi_offset 8, ATS_X + 64 ;\ + .cfi_offset 9, ATS_X + 72 ;\ + .cfi_offset 10, ATS_X + 80 ;\ + .cfi_offset 11, ATS_X + 88 ;\ + .cfi_offset 12, ATS_X + 96 ;\ + .cfi_offset 13, ATS_X + 104 ;\ + .cfi_offset 14, ATS_X + 112 ;\ + .cfi_offset 15, ATS_X + 120 ;\ + .cfi_offset 16, ATS_X + 128 ;\ + .cfi_offset 17, ATS_X + 136 ;\ + .cfi_offset 18, ATS_X + 144 ;\ + .cfi_offset 19, ATS_X + 152 ;\ + .cfi_offset 20, ATS_X + 160 ;\ + .cfi_offset 21, ATS_X + 168 ;\ + .cfi_offset 22, ATS_X + 176 ;\ + .cfi_offset 23, ATS_X + 184 ;\ + .cfi_offset 24, ATS_X + 192 ;\ + .cfi_offset 25, ATS_X + 200 ;\ + .cfi_offset 26, ATS_X + 208 ;\ + .cfi_offset 27, ATS_X + 216 ;\ + .cfi_offset 28, ATS_X + 224 ;\ + .cfi_offset 29, ATS_X + 232 ;\ + .cfi_offset 30, ATS_X + 240 ;\ + .cfi_offset 31, ATS_SP ;\ + .cfi_offset 32, ATS_PC ;\ + .cfi_offset 36, ATS_TPIDR_EL0 + + +/* + * Spill caller-saved registers onto the stack, forming an + * struct aarch64_kernel_exception_state (akes). + */ + +#if (AKES_X != 0) || (AKES_CPSR != AKES_PC + 8) +#error "Update the logic below" +#endif + +/* 13 instructions (52 bytes) long */ +#define SAVE_EL1_STATE() \ + stp x0, x1, [sp, #(-AKES_SIZE)]! ;\ + .cfi_def_cfa sp, AKES_SIZE ;\ + .cfi_offset 0, AKES_X - AKES_SIZE ;\ + .cfi_offset 1, AKES_X + 8 - AKES_SIZE ;\ + stp x2, x3, [sp, #(AKES_X + 16)] ;\ + .cfi_offset 2, AKES_X + 16 - AKES_SIZE ;\ + .cfi_offset 3, AKES_X + 24 - AKES_SIZE ;\ + stp x4, x5, [sp, #(AKES_X + 32)] ;\ + .cfi_offset 4, AKES_X + 32 - AKES_SIZE ;\ + .cfi_offset 5, AKES_X + 40 - AKES_SIZE ;\ + stp x6, x7, [sp, #(AKES_X + 48)] ;\ + .cfi_offset 6, AKES_X + 48 - AKES_SIZE ;\ + .cfi_offset 6, AKES_X + 56 - AKES_SIZE ;\ + stp x8, x9, [sp, #(AKES_X + 64)] ;\ + .cfi_offset 8, AKES_X + 64 - AKES_SIZE ;\ + .cfi_offset 9, AKES_X + 72 - AKES_SIZE ;\ + stp x10, x11, [sp, #(AKES_X + 80)] ;\ + .cfi_offset 10, AKES_X + 80 - AKES_SIZE ;\ + .cfi_offset 11, AKES_X + 80 - AKES_SIZE ;\ + stp x12, x13, [sp, #(AKES_X + 96)] ;\ + .cfi_offset 12, AKES_X + 96 - AKES_SIZE ;\ + .cfi_offset 13, AKES_X + 104 - AKES_SIZE ;\ + stp x14, x15, [sp, #(AKES_X + 112)] ;\ + .cfi_offset 14, AKES_X + 112 - AKES_SIZE ;\ + .cfi_offset 15, AKES_X + 120 - AKES_SIZE ;\ + stp x16, x17, [sp, #(AKES_X + 128)] ;\ + .cfi_offset 16, AKES_X + 128 - AKES_SIZE ;\ + .cfi_offset 17, AKES_X + 136 - AKES_SIZE ;\ + stp x18, x30, [sp, #(AKES_X + 144)] ;\ + .cfi_offset 18, AKES_X + 144 - AKES_SIZE ;\ + .cfi_offset 30, AKES_X + 152 - AKES_SIZE ;\ + mrs x0, ELR_EL1 ;\ + mrs x1, SPSR_EL1 ;\ + stp x0, x1, [sp, #(AKES_PC)] ;\ + .cfi_offset 32, AKES_PC - AKES_SIZE ;\ + +/* + * Fault recovery. + */ +#define RECOVER(handler) \ + .pushsection .rodata, 2 ;\ + .xword 9f - EXT(recover_table) ;\ + .xword handler - EXT(recover_table) ;\ + .popsection ;\ +9: + + .pushsection .rodata, 2 + .global EXT(recover_table) +LEXT(recover_table) + .popsection + + +/* + * Copy memory from user's untrusted address into the + * kernel's memory. Return 0 for success, 1 for bad + * address. + */ +ENTRY(copyin) + /* + * x0: user src + * x1: kernel dst + * x2: count in bytes + * + * Make sure neither src nor dst overflow; also check + * that end of src < VM_MAX_USER_ADDRESS. + */ + adds x3, x0, x2 + b.cs .copyin_fail + adds x4, x1, x2 + b.cs .copyin_fail + cmp xzr, x3, lsr #(VM_AARCH64_T0SZ) + b.ne .copyin_fail + /* + * x3: end of src + * x4: end of dst + * + * Copy single bytes until src is 8-byte aligned. + * This is a slow path, it doesn't have to be fast. + */ +.copyin_align: + tst x0, #7 + b.eq .copyin_by_8 + RECOVER(.copyin_fail) + ldtrb w5, [x0] /* load a byte */ + add x0, x0, #1 + strb w5, [x1], #1 /* store it back */ + subs x2, x2, #1 + b.ne .copyin_align +.copyin_by_8: + cmp x2, #8 + b.lo .copyin_rest + RECOVER(.copyin_fail) + ldtr x5, [x0] /* load 8b */ + add x0, x0, #8 + str x5, [x1], #8 /* store it back */ + subs x2, x2, #8 + b.ne .copyin_by_8 +.copyin_rest: + cbz x2, .copyin_success + cmp x2, #3 + b.eq .copyin_3 + b.lo .copyin_1_or_2 + /* fallthrough */ +.copyin_4_to_7: + RECOVER(.copyin_fail) + ldtr w5, [x0] /* load first 4b */ + RECOVER(.copyin_fail) + ldtr w6, [x3, #-4] /* load last 4b */ + str w5, [x1] /* store first 4b */ + str w6, [x4, #-4] /* store last 4b */ + b .copyin_success +.copyin_3: + RECOVER(.copyin_fail) + ldtrb w5, [x0, #1] /* load middle byte */ + strb w5, [x1, #1] /* store middle byte */ + /* fallthrough */ +.copyin_1_or_2: + RECOVER(.copyin_fail) + ldtrb w5, [x0] /* load first byte */ + RECOVER(.copyin_fail) + ldtrb w6, [x3, #-1] /* load last byte */ + strb w5, [x1] /* store first byte */ + strb w6, [x4, #-1] /* store last byte */ + /* fallthrough */ +.copyin_success: + mov x0, #0 + ret +.copyin_fail: + mov x0, #1 + ret +END(copyin) + +/* + * Copy from kernel's memory to user's untrusted address. + * Return 0 for success, 1 for bad address. + */ +ENTRY(copyout) + /* + * x0: kernel src + * x1: user dst + * x2: count in bytes + * + * Make sure that neither src nor dst overflow; + * also check that end of dst < VM_MAX_USER_ADDRESS. + */ + adds x3, x0, x2 + b.cs .copyout_fail + adds x4, x1, x2 + b.cs .copyout_fail + cmp xzr, x4, lsr #(VM_AARCH64_T0SZ) + b.ne .copyout_fail + /* + * x3: end of src + * x4: end of dst + * + * Copy single bytes until dst is 8-byte aligned. + * This is a slow path, it doesn't have to be fast. + */ +.copyout_align: + tst x1, #7 + b.eq .copyout_by_8 + ldrb w5, [x0], #1 /* load a byte */ + RECOVER(.copyout_fail) + sttrb w5, [x1] /* store it back */ + add x1, x1, #1 + sub x2, x2, #1 + b.ne .copyout_align +.copyout_by_8: + cmp x2, #8 + b.lo .copyout_rest + ldr x5, [x0], #8 /* load 8b */ + RECOVER(.copyout_fail) + sttr x5, [x1] /* store it back */ + add x1, x1, #8 + subs x2, x2, #8 + b.ne .copyout_by_8 +.copyout_rest: + cbz x2, .copyout_success + cmp x2, #3 + b.eq .copyout_3 + b.lo .copyout_1_or_2 + /* fallthrough */ +.copyout_4_to_7: + ldr w5, [x0] /* load first 4b */ + ldr w6, [x3, #-4] /* load last 4b */ + RECOVER(.copyout_fail) + sttr w5, [x1] /* store first 4b */ + RECOVER(.copyout_fail) + sttr w6, [x4, #-4] /* store last 4b */ + b .copyout_success +.copyout_3: + ldrb w5, [x0, #1] /* load middle byte */ + RECOVER(.copyout_fail) + sttrb w5, [x1, #1] /* store middle byte */ + /* fallthrough */ +.copyout_1_or_2: + ldrb w5, [x0] /* load first byte */ + ldrb w6, [x3, #-1] /* load last byte */ + RECOVER(.copyout_fail) + sttrb w5, [x1] /* store first byte */ + RECOVER(.copyout_fail) + sttrb w6, [x4, #-1] /* store last byte */ + /* fallthrough */ +.copyout_success: + mov x0, #0 + ret +.copyout_fail: + mov x0, #1 + ret +END(copyout) + +/* + * Save FP/AdvSIMD state into a struct aarch64_float_state. + */ +ENTRY(_fpu_save_state) + stp q0, q1, [x0], #32 + stp q2, q3, [x0], #32 + stp q4, q5, [x0], #32 + stp q6, q7, [x0], #32 + stp q8, q9, [x0], #32 + stp q10, q11, [x0], #32 + stp q12, q13, [x0], #32 + stp q14, q15, [x0], #32 + stp q16, q17, [x0], #32 + stp q18, q19, [x0], #32 + stp q20, q21, [x0], #32 + stp q22, q23, [x0], #32 + stp q24, q25, [x0], #32 + stp q26, q27, [x0], #32 + stp q28, q29, [x0], #32 + stp q30, q31, [x0], #32 + mrs x1, fpcr + mrs x2, fpsr + stp x1, x2, [x0], #32 + stp xzr, xzr, [x0] /* FPMR ignored for now */ + ret +END(_fpu_save_state) + +/* + * Load FP/AdvSIMD state from a struct aarch64_float_state. + */ +ENTRY(_fpu_load_state) + ldp q0, q1, [x0], #32 + ldp q2, q3, [x0], #32 + ldp q4, q5, [x0], #32 + ldp q6, q7, [x0], #32 + ldp q8, q9, [x0], #32 + ldp q10, q11, [x0], #32 + ldp q12, q13, [x0], #32 + ldp q14, q15, [x0], #32 + ldp q16, q17, [x0], #32 + ldp q18, q19, [x0], #32 + ldp q20, q21, [x0], #32 + ldp q22, q23, [x0], #32 + ldp q24, q25, [x0], #32 + ldp q26, q27, [x0], #32 + ldp q28, q29, [x0], #32 + ldp q30, q31, [x0], #32 + + ldp x1, x2, [x0] + msr fpcr, x1 + msr fpsr, x2 + /* FPMR ignored for now */ + ret +END(_fpu_load_state) + +LEXT(kernel_trap_fatal_helper) + /* + * We got here after kernel_trap_sync() returned FALSE. + * AKES and ESR/FAR are on the stack. Construct an ATS + * and branch to kernel_trap_fatal(). + */ + ldp x0, x1, [sp] /* x0 <- esr, x1 <- far */ + ldp x2, x3, [sp, #(16 + AKES_PC)] /* x2 <- pc, x3 <- cpsr */ + + ldp x5, x6, [sp, #16] /* x0, x1 */ + stp x5, x6, [sp, #(AKES_SIZE + 16 - ATS_SIZE)]! + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 16)] + stp x5, x6, [sp, #16] /* x2, x3 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 32)] + stp x5, x6, [sp, #32] /* x4, x5 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 48)] + stp x5, x6, [sp, #48] /* x6, x7 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 64)] + stp x5, x6, [sp, #64] /* x8, x9 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 80)] + stp x5, x6, [sp, #80] /* x10, x11 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 96)] + stp x5, x6, [sp, #96] /* x12, x13 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 112)] + stp x5, x6, [sp, #112] /* x14, x15 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 128)] + stp x5, x6, [sp, #128] /* x16, x17 */ + ldp x5, x6, [sp, #(ATS_SIZE - AKES_SIZE + 144)] + stp x5, x19, [sp, #144] /* x18; x6 <- x30 */ + stp x20, x21, [sp, #160] + stp x22, x23, [sp, #176] + stp x24, x25, [sp, #192] + stp x26, x27, [sp, #208] + stp x28, x29, [sp, #224] + add x4, sp, #ATS_SIZE + stp x6, x4, [sp, #240] + stp x2, xzr, [sp, #256] + str x3, [sp, #272] + + mov x2, sp + b EXT(kernel_trap_fatal) +END(kernel_trap_fatal_helper) + +/* + * Called as a function, makes the current thread + * return from the kernel as if from a syscall. + * Takes the syscall's return code as an argument. + */ +ENTRY(thread_syscall_return) + GET_PCB_STACK(x1) + str x0, [x1, #(ATS_X)] + /* b EXT(thread_exception_return) */ + /* fallthrough */ +END(thread_syscall_return) + +/* + * Called as a function, makes the current thread + * return from the kernel as if from an exception. + * + * Take an AST if needed, switch to the PCB stack, restore + * the saved register values, and return to EL0. + */ +ENTRY(thread_exception_return) +ENTRY(thread_bootstrap_return) +#ifdef __ARM_FEATURE_BTI_DEFAULT + bti c +#endif + + /* + * We're about to write system registers (ELR_EL1 etc.), + * and we can't allow an interrupt to clobber their values, + * otherwise the 'eret' below wouldn't return to user, it + * would loop back here. So, mask interrupts for a short + * while. They'll get unmasked when we 'eret' to user code, + * which sets DAIF again from SPSR_EL1. + * + */ + msr DAIFSet, #15 + + /* take an AST if needed */ + adr x0, EXT(need_ast) + ldr w1, [x0] + cbnz w1, .take_ast + + /* switch to PCB stack */ + GET_PCB_STACK(x1) + mov sp, x1 + + /* restore registers */ + ldp x0, x1, [sp, #(ATS_SP)] + msr SP_EL0, x0 + msr ELR_EL1, x1 + ldp x0, x1, [sp, #(ATS_TPIDR_EL0)] + msr TPIDR_EL0, x0 + msr SPSR_EL1, x1 + + ldp x0, x1, [sp, #(ATS_X)] + ldp x2, x3, [sp, #(ATS_X + 16)] + ldp x4, x5, [sp, #(ATS_X + 32)] + ldp x6, x7, [sp, #(ATS_X + 48)] + ldp x8, x9, [sp, #(ATS_X + 64)] + ldp x10, x11, [sp, #(ATS_X + 80)] + ldp x12, x13, [sp, #(ATS_X + 96)] + ldp x14, x15, [sp, #(ATS_X + 112)] + ldp x16, x17, [sp, #(ATS_X + 128)] + ldp x18, x19, [sp, #(ATS_X + 144)] + ldp x20, x21, [sp, #(ATS_X + 160)] + ldp x22, x23, [sp, #(ATS_X + 176)] + ldp x24, x25, [sp, #(ATS_X + 192)] + ldp x26, x27, [sp, #(ATS_X + 208)] + ldp x28, x29, [sp, #(ATS_X + 224)] + ldr x30, [sp, #(ATS_X + 240)] + eret + +.take_ast: + bl EXT(ast_taken) /* take the AST */ + b EXT(thread_exception_return) +END(thread_exception_return) + +LEXT(return_to_kernel) + msr DAIFSet, #15 /* see above */ + + ldp x0, x1, [sp, #(AKES_PC)] + msr ELR_EL1, x0 + msr SPSR_EL1, x1 + + ldp x0, x1, [sp, #(AKES_X)] + ldp x2, x3, [sp, #(AKES_X + 16)] + ldp x4, x5, [sp, #(AKES_X + 32)] + ldp x6, x7, [sp, #(AKES_X + 48)] + ldp x8, x9, [sp, #(AKES_X + 64)] + ldp x10, x11, [sp, #(AKES_X + 80)] + ldp x12, x13, [sp, #(AKES_X + 96)] + ldp x14, x15, [sp, #(AKES_X + 112)] + ldp x16, x17, [sp, #(AKES_X + 128)] + ldp x18, x30, [sp, #(AKES_X + 144)] + add sp, sp, #(AKES_SIZE) + eret +END(return_to_kernel) + +/* + * Discard the current stack, and invoke the given continuation. + */ +ENTRY(call_continuation) + mov x1, sp + orr x1, x1, #(KERNEL_STACK_SIZE - 1) + sub x1, x1, #(AKS_SIZE + AEL_SIZE - 1) + mov sp, x1 /* point stack to the top */ + mov x29, #0 /* dummy frame */ + mov x30, #0 /* dummy return */ +#ifdef __ARM_FEATURE_BTI_DEFAULT + mov x16, x0 + br x16 /* goto continuation */ +#else + br x0 /* goto continuation */ +#endif +END(call_continuation) + + .balign 2048 +ENTRY(exception_vector_table) +.sync_exc_el1_sp_el0: + b . +.balign 0x80 +.irq_el1_sp_el0: + b . +.balign 0x80 +.fiq_el1_sp_el0: + b . +.balign 0x80 +.serror_el1_sp_el0: + b . +.balign 0x80 +.sync_exc_el1_sp_el1: + .cfi_startproc simple + .cfi_signal_frame + SAVE_EL1_STATE() + mrs x0, ESR_EL1 + mrs x1, FAR_EL1 + mov x2, sp + stp x0, x1, [sp, #-16]! + .cfi_adjust_cfa_offset 16 + bl EXT(kernel_trap_sync) + cbz x0, EXT(kernel_trap_fatal_helper) + add sp, sp, #16 + b EXT(return_to_kernel) + .cfi_endproc +.balign 0x80 +.irq_el1_sp_el1: + .cfi_startproc simple + .cfi_signal_frame + SAVE_EL1_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(kernel_trap_irq) + b EXT(return_to_kernel) + .cfi_endproc +.balign 0x80 +.fiq_el1_sp_el1: + .cfi_startproc simple + .cfi_signal_frame + SAVE_EL1_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(kernel_trap_fiq) + b EXT(return_to_kernel) + .cfi_endproc +.balign 0x80 +.serror_el1_sp_el1: + .cfi_startproc simple + .cfi_signal_frame + SAVE_EL1_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(kernel_trap_serror) + b EXT(return_to_kernel) + .cfi_endproc +.balign 0x80 +.sync_exc_el0_aarch64: + .cfi_startproc + .cfi_signal_frame + SAVE_EL0_STATE() + SWITCH_TO_KERNEL_STACK(x0) + CFI_PCB_EL0_STATE() + msr DAIFClr, #15 /* safe to unmask interrupts now */ + bl EXT(user_trap_sync) + nop /* mysterious nop */ + .cfi_endproc +.balign 0x80 +.irq_el0_aarch64: + .cfi_startproc + .cfi_signal_frame + SAVE_EL0_STATE() + SWITCH_TO_KERNEL_STACK(x0) + CFI_PCB_EL0_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(user_trap_irq) + nop /* mysterious nop */ + .cfi_endproc +.balign 0x80 +.fiq_el0_aarch64: + .cfi_startproc + .cfi_signal_frame + SAVE_EL0_STATE() + SWITCH_TO_KERNEL_STACK(x0) + CFI_PCB_EL0_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(user_trap_fiq) + nop /* mysterious nop */ + .cfi_endproc +.balign 0x80 +.serror_el0_aarch64: + .cfi_startproc + .cfi_signal_frame + SAVE_EL0_STATE() + SWITCH_TO_KERNEL_STACK(x0) + CFI_PCB_EL0_STATE() +#ifdef MACH_KDB + msr DAIFClr, #8 +#endif + bl EXT(user_trap_serror) + nop /* mysterious nop */ + .cfi_endproc +.balign 0x80 +.sync_exc_el0_aarch32: + SWITCH_TO_KERNEL_STACK(x0) + b EXT(user_trap_aarch32) +.balign 0x80 +.irq_el0_aarch32: + SWITCH_TO_KERNEL_STACK(x0) + b EXT(user_trap_aarch32) +.balign 0x80 +.fiq_el0_aarch32: + SWITCH_TO_KERNEL_STACK(x0) + b EXT(user_trap_aarch32) +.balign 0x80 +.serror_el0_aarch32: + SWITCH_TO_KERNEL_STACK(x0) + b EXT(user_trap_aarch32) +END(exception_vector_table) + +ENTRY(load_exception_vector_table) + adr x0, exception_vector_table + msr VBAR_EL1, x0 + ret +END(load_exception_vector_table) + +ENTRY(handle_syscall) + .cfi_startproc + stp x29, x30, [sp, #-32]! + .cfi_def_cfa_offset 32 + .cfi_offset 29, -32 + .cfi_offset 30, -24 + mov x29, sp + str x0, [sp, #16] + + mov x12, x0 + ldr x8, [x12, #(ATS_X + 8*8)] /* load syscall number (in w8) */ + neg w8, w8 /* negate it */ + adr x9, EXT(mach_trap_count) /* this should really be a compile-time constant... */ + ldr x9, [x9] + cmp x9, x8 + b.ls .bad_syscall + ubfiz x8, x8, #5, #32 /* x8 *= sizeof(mach_trap_t) */ + adr x9, EXT(mach_trap_table) + add x9, x9, x8 + ldp x10, x11, [x9] + cmp x10, #8 + b.hi .load_stack_args +.load_reg_args: + ldp x0, x1, [x12, #(ATS_X)] + ldp x2, x3, [x12, #(ATS_X + 16)] + ldp x4, x5, [x12, #(ATS_X + 32)] + ldp x6, x7, [x12, #(ATS_X + 48)] + + blr x11 + + ldr x12, [x29, #16] + str x0, [x12, #(ATS_X)] /* put return code into user's x0 */ + mov w0, #1 /* return TRUE */ + +.out: + mov sp, x29 + ldp x29, x30, [sp], #32 + .cfi_restore x29 + .cfi_restore x30 + .cfi_def_cfa_offset 0 + ret +.load_stack_args: + sub x10, x10, #8 + mov x0, x10 + add x1, x10, #1 + tst x0, #1 + lsl x0, x0, #3 + lsl x1, x1, #3 + csel x0, x0, x1, eq + ldr x13, [x12, #(ATS_SP)] + adds x13, x13, x0 + b.cs .bad_sp + cmp xzr, x13, lsr #(VM_AARCH64_T0SZ) + b.ne .bad_sp +0: + sub x13, x13, #16 + RECOVER(.bad_stack_args) + ldtr x1, [x13, #8] + RECOVER(.bad_stack_args) + ldtr x0, [x13] + stp x0, x1, [sp, #-16]! + cmp x10, #2 + b.ls .load_reg_args + sub x10, x10, #2 + b 0b +.bad_syscall: + mov w0, #0 /* return FALSE */ + b .out +.bad_sp: + mov x0, #(EXC_BAD_ACCESS) + mov x1, #(KERN_INVALID_ADDRESS) + mov x2, x13 + /* fallthrough */ +.bad_stack_args: + /* + * We were loading arguments from user's stack, + * but faulted. Restore our SP, and treat it + * as a fault from EL0. x0, x1, x2 already hold + * appropriate values for the exception() call. + */ + mov sp, x29 + ldp x29, x30, [sp], #32 + b EXT(exception) + .cfi_endproc +END(handle_syscall) + + .pushsection .rodata, 2 +END(recover_table) + .global EXT(recover_table_end) +LEXT(recover_table_end) + .popsection + + .section .note.GNU-stack,"",%progbits diff --git a/aarch64/aarch64/locore.h b/aarch64/aarch64/locore.h new file mode 100644 index 00000000..74697d9a --- /dev/null +++ b/aarch64/aarch64/locore.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_LOCORE_ +#define _AARCH64_LOCORE_ + +#include <mach/mach_types.h> +#include <kern/sched_prim.h> +#include <stddef.h> + +/* + * Fault recovery in copyin/copyout routines. + * + * Both offsets are relative to &recover_table. + */ +struct recovery { + vm_offset_t fault_addr_off; + vm_offset_t recover_addr_off; +}; +extern const struct recovery recover_table[]; +extern const struct recovery recover_table_end[]; + +int copyin(const void *userbuf, void *kernelbuf, size_t cn); +int copyout(const void *kernelbuf, void *userbuf, size_t cn); + +extern void __attribute__((noreturn)) call_continuation(continuation_t continuation); +extern boolean_t handle_syscall(struct aarch64_thread_state *ats); + +extern void load_exception_vector_table(void); + +struct aarch64_float_state; +extern void _fpu_save_state(struct aarch64_float_state *); +extern void _fpu_load_state(const struct aarch64_float_state *); + +#endif /* _AARCH64_LOCORE_ */ diff --git a/aarch64/aarch64/loose_ends.h b/aarch64/aarch64/loose_ends.h new file mode 100644 index 00000000..d45826b1 --- /dev/null +++ b/aarch64/aarch64/loose_ends.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_LOOSE_ENDS_H_ +#define _AARCH64_LOOSE_ENDS_H_ + +void delay (int n); + +#endif /* _AARCH64_LOOSE_ENDS_H_ */ diff --git a/aarch64/aarch64/mach_aarch64.c b/aarch64/aarch64/mach_aarch64.c new file mode 100644 index 00000000..078fe124 --- /dev/null +++ b/aarch64/aarch64/mach_aarch64.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/mach_aarch64.server.h" +#include "aarch64/hwcaps.h" +#include <kern/host.h> +#include <string.h> + +kern_return_t aarch64_get_hwcaps( + const host_t host, + uint32_t *out_hwcaps, + mach_msg_type_number_t *hwcapsCnt, + uint64_t *midr_el1, + uint64_t *revidr_el1) +{ + uint64_t v; + + if (host != &realhost) + return KERN_INVALID_HOST; + + *hwcapsCnt = MIN(*hwcapsCnt, HWCAPS_COUNT); + memcpy(out_hwcaps, hwcaps, sizeof(uint32_t) * (*hwcapsCnt)); + + asm("mrs %0, midr_el1" : "=r"(v)); + *midr_el1 = v; + asm("mrs %0, revidr_el1" : "=r"(v)); + *revidr_el1 = v; + + return KERN_SUCCESS; +} diff --git a/aarch64/aarch64/mach_param.h b/aarch64/aarch64/mach_param.h new file mode 100644 index 00000000..5af9a532 --- /dev/null +++ b/aarch64/aarch64/mach_param.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_MACH_PARAM_H_ +#define _AARCH64_MACH_PARAM_H_ + +#define HZ 100 + +#endif /* _AARCH64_MACH_PARAM_H_ */ diff --git a/aarch64/aarch64/machine_routines.h b/aarch64/aarch64/machine_routines.h new file mode 100644 index 00000000..c652a112 --- /dev/null +++ b/aarch64/aarch64/machine_routines.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_MACHINE_ROUTINES_H_ +#define _AARCH64_MACHINE_ROUTINES_H_ + +#define MACHINE_SERVER mach_aarch64_server +#define MACHINE_SERVER_HEADER "aarch64/aarch64/mach_aarch64.server.h" +#define MACHINE_SERVER_ROUTINE mach_aarch64_server_routine + +#endif /* _AARCH64_MACHINE_ROUTINES_H_ */ + diff --git a/aarch64/aarch64/model_dep.c b/aarch64/aarch64/model_dep.c new file mode 100644 index 00000000..1105daf8 --- /dev/null +++ b/aarch64/aarch64/model_dep.c @@ -0,0 +1,417 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/model_dep.h" +#include "aarch64/locore.h" +#include "aarch64/hwcaps.h" +#include "aarch64/fpu.h" +#include "aarch64/bits/spsr.h" +#include "arm/gic-v2.h" +#include "arm/pl011.h" +#include "arm/psci.h" +#include <device/dtb.h> +#include <mach/machine.h> +#include <kern/printf.h> +#include <kern/startup.h> +#include <kern/bootstrap.h> +#include <kern/boot_script.h> +#include <string.h> + +#include <device/intr.h> /* FIXME */ + +/* Some ELF definitions, for applying relocations. */ + +#define R_AARCH64_NONE 0 +#define R_AARCH64_RELATIVE 1027 + +typedef uint64_t Elf64_Addr; +typedef uint64_t Elf64_Xword; +typedef int64_t Elf64_Sxword; + +typedef struct +{ + Elf64_Addr r_offset; + Elf64_Xword r_info; + Elf64_Sxword r_addend; +} Elf64_Rela; + + + +const char *kernel_cmdline; + +char /*struct start_info*/ boot_info; +struct irqdev irqtab; +int iunit[1]; +interrupt_handler_fn ivect[1]; + +int spl_init; + +void machine_idle(int mycpu) +{ +#ifdef MACH_HYP + hyp_idle(); +#else + // assert(cpu == cpu_number()); + asm volatile("wfi"); +#endif +} + +void halt_cpu(void) +{ +#ifdef MACH_HYP + hyp_halt(); +#else + /* Try PSCI. */ + psci_cpu_off(); + /* Disable interrupts and WFI forever. */ + asm volatile( + "msr DAIFSet, #15\n" + "0:\n\t" + "wfi\n\t" + "b 0b" + ); + __builtin_unreachable(); +#endif +} + +void halt_all_cpus(boolean_t reboot) +{ + if (reboot) + psci_system_reset(); + else + psci_system_off(); + + /* TODO halt _all_ CPUs. */ + printf("Shutdown completed successfully, now in tight loop.\n"); + printf("You can safely power off the system or hit ctl-alt-del to reboot\n"); + halt_cpu(); +} + +/* FIXME */ +static struct irq_ctlr *interrupt_controller; + +static void zero_out_bss(void) +{ + extern char __bss_start, __bss_end; + + memset(&__bss_start, 0, &__bss_end - &__bss_start); +} + +static void apply_runtime_relocations(void) +{ + extern const Elf64_Rela __rela_start, __rela_end; + extern const char __text_start; + + const Elf64_Rela *rela; + Elf64_Addr *addr; + Elf64_Addr slide; + + /* TODO: This assumes we're linked at base address 0x0. */ + slide = (Elf64_Addr) &__text_start; + + for (rela = &__rela_start; rela != &__rela_end; rela++) { + switch (rela->r_info) { + case R_AARCH64_NONE: + /* Nothing to do. */ + break; + case R_AARCH64_RELATIVE: + addr = (Elf64_Addr *)(slide + rela->r_offset); + *addr = slide + rela->r_addend; + break; + default: + panic("Unimplemented relocation type\n"); + } + } +} + +static void print_model(const char *model) +{ + const char *c; + boolean_t seen_comma = FALSE; + + printf("Model name: "); + for (c = model; *c; c++) { + if (!seen_comma && *c == ',') { + printf(" "); + seen_comma = TRUE; + } else { + printf("%c", *c); + } + } + printf("\n"); +} + +static void walk_dtb_visit_node( + dtb_node_t node, + dtb_ranges_map_t map) +{ + struct dtb_node child; + struct dtb_ranges_map nmap; + boolean_t have_nmap = FALSE; + + if (dtb_node_is_compatible(node, "arm,pl011")) { + pl011_init(node, map); + } else if (dtb_node_is_compatible(node, "arm,armv8-timer")) { + cnt_init(node); + /* FIXME */ + if (interrupt_controller) + cnt_set_interrupt_parent(node, interrupt_controller); + } else if (gic_v2_is_compatible(node)) { + interrupt_controller = gic_v2_init(node, map); + } else if (psci_is_compatible(node)) { + psci_init(node); + } else if (dtb_node_is_compatible(node, "simple-bus")) { + nmap = dtb_node_make_ranges_map(node); + nmap.next = map; + have_nmap = TRUE; + } + + dtb_for_each_child (*node, child) { + walk_dtb_visit_node(&child, have_nmap ? &nmap : map); + } +} + +static void walk_dtb(void) +{ + struct dtb_node node; + struct dtb_prop prop; + + node = dtb_root_node(); + /* + * Look at root node's properties. + */ + dtb_for_each_prop (node, prop) { + if (!strcmp(prop.name, "model")) { + print_model(prop.data); + } + } + + /* + * Look at top-level nodes and their props. + */ + dtb_for_each_child (node, node) { + walk_dtb_visit_node(&node, NULL); + } +} + +/* + * Find devices. The system is alive. + */ +void machine_init(void) +{ + fpu_init(); + + /* Note that the kernel is entered with IRQ/FIQ masked. */ + spl7_irq(); + spl_init = TRUE; + + walk_dtb(); + + /* FIXME */ + assert(interrupt_controller != NULL); + gic_v2_enable(interrupt_controller); +} + +static void early_dtb_walk_visit_node( + dtb_node_t node, + dtb_ranges_map_t map) +{ + struct dtb_node child; + struct dtb_ranges_map nmap; + boolean_t have_nmap = FALSE; + + if (dtb_node_is_compatible(node, "arm,pl011")) { + pl011_early_init(node, map); + } else if (psci_is_compatible(node)) { + psci_init(node); + } else if (dtb_node_is_compatible(node, "simple-bus")) { + nmap = dtb_node_make_ranges_map(node); + nmap.next = map; + have_nmap = TRUE; + } + + dtb_for_each_child (*node, child) { + early_dtb_walk_visit_node(&child, have_nmap ? &nmap : map); + } +} + +static void early_dtb_walk(void) +{ + struct dtb_node node; + struct dtb_prop prop; + + node = dtb_root_node(); + + /* + * Look at top-level nodes and their props. + */ + dtb_for_each_child (node, node) { + if (!strcmp(node.name, "chosen") || !strncmp(node.name, "chosen@", 7)) { + prop = dtb_node_find_prop(&node, "bootargs"); + if (!DTB_IS_SENTINEL(prop)) + kernel_cmdline = (const char *) prop.data; + /* TODO: /chosen/kaslr-seed */ + continue; + } + dtb_for_each_prop(node, prop) { + if (!strcmp(prop.name, "device_type") + && !strcmp(prop.data, "memory")) + pmap_discover_physical_memory(&node); + } + early_dtb_walk_visit_node(&node, NULL); + } +} + +static void print_el(void) +{ + long current_el; + unsigned short el; + + asm("mrs %0, CurrentEL" : "=r"(current_el)); + el = SPSR_EL(current_el); + + printf("Booting in EL%d\n", el); +} + +void __attribute__((noreturn)) c_boot_entry(dtb_t dtb) +{ + kern_return_t kr; + extern const char version[]; + + zero_out_bss(); + + kr = dtb_load(dtb); + assert(kr == KERN_SUCCESS); + + hwcaps_init(); + early_dtb_walk(); + pmap_bootstrap(); + /* + * Now running with MMU from highmem, re-load things. + */ + asm volatile("" ::: "memory"); + apply_runtime_relocations(); + + dtb = (dtb_t) phystokv(dtb); + kr = dtb_load(dtb); + assert(kr == KERN_SUCCESS); + + pmap_bootstrap_misc(); + load_exception_vector_table(); + + /* + * We should be able to use kmsg/cnputc now, even though + * it doesn't yet go anywhere. + * So before we do anything else, print the hello message. + */ + printf("%s\n", version); + + if (kernel_cmdline == NULL) + kernel_cmdline = ""; + else + kernel_cmdline = (const char *) phystokv(kernel_cmdline); + printf("Kernel command line: %s\n", kernel_cmdline); + + print_el(); + + machine_slot[0].is_cpu = TRUE; + machine_slot[0].cpu_type = CPU_TYPE_ARM64; + init_percpu(0); + + setup_main(); + __builtin_unreachable(); +} + +void machine_exec_boot_script(void) +{ + struct dtb_node chosen, node; + struct dtb_prop prop; + unsigned short address_cells, size_cells; + struct bootstrap_module bmods[10]; + int i = 0, err, losers = 0; + const char *args; + vm_offset_t off; + + chosen = dtb_node_by_path("/chosen"); + if (DTB_IS_SENTINEL(chosen)) + panic("No chosen node in DTB\n"); + + dtb_for_each_child (chosen, node) { + if (dtb_node_is_compatible(&node, "multiboot,module")) { + assert(i < 10); /* 10 boot modules ought to be enough for anybody */ + prop = dtb_node_find_prop(&node, "bootargs"); + if (DTB_IS_SENTINEL(prop)) + panic("No bootargs for bootstrap module %d %s\n", i, node.name); + args = (const char *) prop.data; + printf("module %d: %s\n", i, args); + + prop = dtb_node_find_prop(&node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + address_cells = node.address_cells; + size_cells = node.size_cells; + + /* + * Work around an apparent QEMU guest-laoder bug, + * where it unconditionally uses address/size cell + * size of 2, yet doesn't set (or respect previously + * set) #address-cells / #size-cells properties in + * the parent node. + */ + if (prop.length == 16 && address_cells == 2 && size_cells == 1) + size_cells = 2; + + off = 0; + bmods[i].mod_start = dtb_prop_read_cells(&prop, address_cells, &off); + bmods[i].mod_end = bmods[i].mod_start + dtb_prop_read_cells(&prop, size_cells, &off); + + /* FIXME: we probably should make a copy of this string */ + err = boot_script_parse_line(&bmods[i], args); + if (err) { + printf("Error: %s\n", boot_script_error_string(err)); + losers++; + } + i++; + } + } + if (i == 0) + panic("No bootstrap modules loaded with Mach\n"); + if (losers) + panic("Failed to parse boot script\n"); + printf("%d bootstrap modules\n", i); + err = boot_script_exec(); + if (err) + panic("Failed to execute boot script: %s\n", boot_script_error_string(err)); + /* TODO free memory */ +} + +vm_offset_t timemmap(dev_t dev, vm_offset_t off, vm_prot_t prot) +{ + extern time_value_t *mtime; + + if (prot != VM_PROT_READ || off != 0) + return (vm_offset_t) -1; + return pmap_extract(kernel_pmap, (vm_offset_t) mtime); +} + +vm_offset_t memmmap(dev_t dev, vm_offset_t off, vm_prot_t prot) +{ + if (!vm_page_aligned(off)) + return -1; + + return off; +} diff --git a/aarch64/aarch64/model_dep.h b/aarch64/aarch64/model_dep.h new file mode 100644 index 00000000..5818e78d --- /dev/null +++ b/aarch64/aarch64/model_dep.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_MODEL_DEP_H_ +#define _AARCH64_MODEL_DEP_H_ + +#include <mach/std_types.h> +#include <device/dtb.h> +#include <mach/vm_prot.h> +#include "arm/timer.h" /* startrtclock() */ +#include <sys/types.h> /* dev_t */ + +/* + * Find devices. The system is alive. + */ +extern void machine_init (void); + +/* Conserve power on processor CPU. */ +extern void machine_idle (int cpu); + +extern void resettodr (void); + +/* + * Halt a cpu. + */ +extern void halt_cpu (void) __attribute__ ((noreturn)); + +/* + * Halt the system or reboot. + */ +extern void halt_all_cpus (boolean_t reboot) __attribute__ ((noreturn)); + +/* + * Make cpu pause a bit. + */ +extern void machine_relax (void); + +/* + * C boot entrypoint - called by boot_entry in boothdr.S. + */ +extern void c_boot_entry(dtb_t dtb) __attribute__ ((noreturn)); + +extern vm_offset_t timemmap(dev_t dev, vm_offset_t off, vm_prot_t prot); +extern vm_offset_t memmmap(dev_t dev, vm_offset_t off, vm_prot_t prot); + +#endif /* _AARCH64_MODEL_DEP_H_ */ diff --git a/aarch64/aarch64/mp_desc.h b/aarch64/aarch64/mp_desc.h new file mode 100644 index 00000000..4ed7f936 --- /dev/null +++ b/aarch64/aarch64/mp_desc.h @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_MP_DESC_H_ +#define _AARCH64_MP_DESC_H_ + +#endif /* _AARCH64_MP_DESC_H_ */ diff --git a/aarch64/aarch64/pcb.c b/aarch64/aarch64/pcb.c new file mode 100644 index 00000000..0443ce4e --- /dev/null +++ b/aarch64/aarch64/pcb.c @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "pcb.h" +#include "aarch64/vm_param.h" +#include "aarch64/pmap.h" +#include "aarch64/fpu.h" +#include "aarch64/hwcaps.h" +#include "aarch64/bits/spsr.h" +#include <vm/vm_map.h> +#include <kern/slab.h> +#include <kern/counters.h> +#include <string.h> + +/* Top of active stack (high address). */ +vm_offset_t kernel_stack[NCPUS]; + +struct kmem_cache pcb_cache; + +void pcb_module_init(void) +{ + kmem_cache_init(&pcb_cache, "pcb", sizeof(struct pcb), + alignof(struct pcb), NULL, 0); +} + +void stack_attach( + thread_t thread, + vm_offset_t stack, + void (*continuation)(thread_t)) +{ + counter(if (++c_stacks_current > c_stacks_max) + c_stacks_max = c_stacks_current); + + assert(thread->kernel_stack == 0); + thread->kernel_stack = stack; + STACK_AKS_REG(stack, 30) = (long) Thread_continue; + STACK_AKS_REG(stack, 19) = (long) continuation; + STACK_AKS_REG(stack, 29) = (long) 0; + STACK_AKS(stack)->k_sp = (long) STACK_AEL(stack); + + STACK_AEL(stack)->saved_state = USER_REGS(thread); +} + +vm_offset_t stack_detach(thread_t thread) +{ + vm_offset_t stack; + + counter(if (--c_stacks_current < c_stacks_min) + c_stacks_min = c_stacks_current;) + + stack = thread->kernel_stack; + thread->kernel_stack = 0; + + return stack; +} + +void stack_handoff( + thread_t old, + thread_t new) +{ + task_t old_task, new_task; + int mycpu = cpu_number(); + vm_offset_t stack; + + old_task = old->task; + new_task = new->task; + if (old_task != new_task) { + PMAP_DEACTIVATE_USER(vm_map_pmap(old_task->map), old, mycpu); + PMAP_ACTIVATE_USER(vm_map_pmap(new_task->map), new, mycpu); + } + + fpu_switch_context(new); + + assert(new->kernel_stack == 0); + stack = current_stack(); + old->kernel_stack = 0; + new->kernel_stack = stack; + + percpu_assign(active_thread, new); + + STACK_AEL(stack)->saved_state = USER_REGS(new); +} + +extern thread_t Switch_context(thread_t old, continuation_t continuation, thread_t new); + +thread_t switch_context( + thread_t old, + continuation_t continuation, + thread_t new) +{ + task_t old_task, new_task; + int mycpu = cpu_number(); + + old_task = old->task; + new_task = new->task; + if (old_task != new_task) { + PMAP_DEACTIVATE_USER(vm_map_pmap(old_task->map), old, mycpu); + PMAP_ACTIVATE_USER(vm_map_pmap(new_task->map), new, mycpu); + } + + fpu_switch_context(new); + + return Switch_context(old, continuation, new); +} + +void pcb_init(task_t parent_task, thread_t thread) +{ + pcb_t pcb; + + if (parent_task == kernel_task) { + thread->pcb = NULL; + return; + } + + pcb = (pcb_t) kmem_cache_alloc(&pcb_cache); + if (pcb == NULL) + panic("pcb_init"); + + counter(if (++c_threads_current > c_threads_max) + c_threads_max = c_threads_current); + + memset(pcb, 0, sizeof(*pcb)); + + thread->pcb = pcb; +} + +void pcb_terminate(thread_t thread) +{ + counter(if (--c_threads_current < c_threads_min) + c_threads_min = c_threads_current); + + fpu_free(thread); + kmem_cache_free(&pcb_cache, (vm_offset_t) thread->pcb); + thread->pcb = NULL; +} + +void pcb_collect(__attribute__((unused)) const thread_t thread) +{ +} + +void thread_set_syscall_return( + thread_t thread, + kern_return_t kr) +{ + USER_REGS(thread)->x[0] = kr; +} + +kern_return_t thread_getstatus( + thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int *count) +{ + switch (flavor) { + case THREAD_STATE_FLAVOR_LIST: + if (*count < 2) + return KERN_INVALID_ARGUMENT; + tstate[0] = AARCH64_THREAD_STATE; + tstate[1] = AARCH64_FLOAT_STATE; + *count = 2; + return KERN_SUCCESS; + + case AARCH64_THREAD_STATE: + if (*count < AARCH64_THREAD_STATE_COUNT) + return KERN_INVALID_ARGUMENT; + memcpy(tstate, USER_REGS(thread), sizeof(struct aarch64_thread_state)); + *count = AARCH64_THREAD_STATE_COUNT; + return KERN_SUCCESS; + + case AARCH64_FLOAT_STATE: + if (*count < AARCH64_FLOAT_STATE_COUNT) + return KERN_INVALID_ARGUMENT; + fpu_flush_state_read(thread); + if (thread->pcb->afs == NULL) + memset(tstate, 0, sizeof(struct aarch64_float_state)); + else + memcpy(tstate, thread->pcb->afs, sizeof(struct aarch64_float_state)); + *count = AARCH64_FLOAT_STATE_COUNT; + return KERN_SUCCESS; + + default: + return KERN_INVALID_ARGUMENT; + } +} + +/* + * validate_cpsr: + * + * Check the CPSR the user is trying to set for + * any disallowed/privileged/reserved bits. + */ +static boolean_t validate_cpsr(long cpsr, long old_cpsr) +{ + long res0 = SPSR_RES0; + + /* + * Make sure the CPSR indicates a valid state, + * specifically EL0 AArch64. + * + * Note that both SPSR_EL() = 0 and SPSR_NRW_64 + * have a zero bit pattern, so just initializing + * CPSR to 0 should pass the checks successfully. + */ + if (SPSR_EL(cpsr) != 0) + return FALSE; + if (SPSR_NRW(cpsr) != SPSR_NRW_64) + return FALSE; + + /* + * Let userland mask debug exceptions if they + * so want, but not IRQs, FIQs, or SErrors. + * + * TODO: ARM ARM seems to say D is ignored at EL0. + */ + if (cpsr & SPSR_AIF) + return FALSE; + if (cpsr & SPSR_ALLINT) + return FALSE; + + /* + * SPSR_BTYPE: + * OK to set if we have HWCAP2_BTI. + * SPSR_SSBS: + * OK to set if we have HWCAP_SSBS. + * Resets to 0 or 1 on exception entry + * according to SCTLR_SSBS. + * SPSR_IL: + * OK (and fun) to set. + * Will cause an immediate EXC_AARCH64_IL. + * SPSR_SS: + * OK to set. + * SPSR_PAN: + * OK to set if we have HWCAP_INT_PAN. + * Doesn't affect EL0. + * Resets to 1 (given SCTLR_SPAN is unset) + * on exception entry. + * SPSR_UAO: + * OK to set if we have HWCAP_INT_UAO. + * Doesn't affect EL0. + * Resets to 0 on exception entry. + * SPSR_DIT: + * OK to set if we have HWCAP_DIT. + * SPSR_TCO: + * OK to set if we have HWCAP2_MTE. + * SPSR_NZCV: + * OK to set. + */ + if (!(hwcaps[1] & HWCAP2_BTI)) + res0 |= SPSR_BTYPE_MASK; + if (!(hwcaps[0] & HWCAP_SSBS)) + res0 |= SPSR_SSBS; + if (!(hwcap_internal & HWCAP_INT_PAN)) + res0 |= SPSR_PAN; + if (!(hwcap_internal & HWCAP_INT_UAO)) + res0 |= SPSR_UAO; + if (!(hwcaps[0] & HWCAP_DIT)) + res0 |= SPSR_DIT; + if (!(hwcaps[1] & HWCAP2_MTE)) + res0 |= SPSR_TCO; + + /* + * Allow setting reserved bits to either 0 or + * the value it already had. In other words, + * disallow setting any new reserved bits. + */ + if (cpsr & res0 & ~old_cpsr) + return FALSE; + + return TRUE; +} + +static boolean_t validate_fpcr(long fpcr, long old_fpcr) { + /* TODO */ + return TRUE; +} + +static boolean_t validate_fpsr(long fpsr, long old_fpsr) { + /* TODO */ + return TRUE; +} + +static boolean_t validate_fpmr(long fpmr, long old_fpmr) { + /* TODO: support FPMR */ + return fpmr == 0 || fpmr == old_fpmr; +} + +#define old_fpr(thread, reg) ((thread)->pcb->afs ? (thread)->pcb->afs->reg : 0) + +kern_return_t thread_setstatus( + thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count) +{ + struct aarch64_thread_state *ats; + struct aarch64_float_state *afs; + + switch (flavor) { + case AARCH64_THREAD_STATE: + if (count < AARCH64_THREAD_STATE_COUNT) + return KERN_INVALID_ARGUMENT; + if (((vm_offset_t) tstate) % alignof(struct aarch64_thread_state)) + return KERN_INVALID_ARGUMENT; + ats = (struct aarch64_thread_state *) tstate; + + if (!validate_cpsr(ats->cpsr, USER_REGS(thread)->cpsr)) + return KERN_INVALID_ARGUMENT; + + memcpy(USER_REGS(thread), ats, sizeof(struct aarch64_thread_state)); + return KERN_SUCCESS; + + case AARCH64_FLOAT_STATE: + if (count < AARCH64_FLOAT_STATE_COUNT) + return KERN_INVALID_ARGUMENT; + if (((vm_offset_t) tstate) % alignof(struct aarch64_float_state)) + return KERN_INVALID_ARGUMENT; + afs = (struct aarch64_float_state *) tstate; + + if (!validate_fpcr(afs->fpcr, old_fpr(thread, fpcr))) + return KERN_INVALID_ARGUMENT; + if (!validate_fpsr(afs->fpsr, old_fpr(thread, fpsr))) + return KERN_INVALID_ARGUMENT; + if (!validate_fpmr(afs->fpmr, old_fpr(thread, fpmr))) + return KERN_INVALID_ARGUMENT; + + fpu_flush_state_write(thread); + memcpy(thread->pcb->afs, tstate, sizeof(struct aarch64_float_state)); + return KERN_SUCCESS; + + default: + return KERN_INVALID_ARGUMENT; + } +} + +/* + * Return preferred address of user stack. + * Always returns low address. If stack grows up, + * the stack grows away from this address; + * if stack grows down, the stack grows towards this + * address. + */ +vm_offset_t user_stack_low(vm_size_t stack_size) +{ + return (VM_MAX_USER_ADDRESS - stack_size); +} + + + +vm_offset_t set_user_regs( + vm_offset_t stack_base, + vm_offset_t stack_size, + const struct exec_info *exec_info, + vm_size_t arg_size) +{ + struct aarch64_thread_state *ats; + vm_offset_t arg_addr; + + arg_size = P2ROUND(arg_size, 16); + arg_addr = stack_base + stack_size - arg_size; + assert(P2ALIGNED(stack_base, 16)); + + ats = USER_REGS(current_thread()); + ats->pc = exec_info->entry; + ats->sp = (rpc_vm_offset_t) arg_addr; + + return arg_addr; +} diff --git a/aarch64/aarch64/pcb.h b/aarch64/aarch64/pcb.h new file mode 100644 index 00000000..1699911c --- /dev/null +++ b/aarch64/aarch64/pcb.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_PCB_ +#define _AARCH64_PCB_ + +#include <kern/task.h> +#include <kern/thread.h> +#include <mach/exec/exec.h> + +extern void pcb_module_init(void); + +extern void pcb_init(task_t parent_task, thread_t thread); +extern void pcb_terminate(thread_t thread); +extern void pcb_collect(thread_t thread); + +extern void load_context(thread_t t); +extern void Thread_continue(void); + +extern vm_offset_t user_stack_low (vm_size_t stack_size); + +extern kern_return_t thread_setstatus( + thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int count); + +extern kern_return_t thread_getstatus( + thread_t thread, + int flavor, + thread_state_t tstate, + unsigned int *count); + +extern void thread_set_syscall_return( + thread_t thread, + kern_return_t retval); + +extern vm_offset_t set_user_regs( + vm_offset_t stack_base, + vm_offset_t stack_size, + const struct exec_info *exec_info, + vm_size_t arg_size); + +extern void stack_attach( + thread_t thread, + vm_offset_t stack, + void (*continuation)(thread_t)); + +extern vm_offset_t stack_detach(thread_t thread); + +#endif /* _AARCH64_PCB_ */ diff --git a/aarch64/aarch64/percpu.c b/aarch64/aarch64/percpu.c new file mode 100644 index 00000000..c80153f0 --- /dev/null +++ b/aarch64/aarch64/percpu.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/percpu.h" + +struct percpu percpu_array[NCPUS]; + +void init_percpu(int cpu) +{ + // memset(&percpu_array[cpu], 0, sizeof(struct percpu)); + asm volatile("msr TPIDRRO_EL0, %0" :: "r"(cpu)); + asm volatile("msr TPIDR_EL1, %0" :: "r"(&percpu_array[cpu])); +} diff --git a/aarch64/aarch64/percpu.h b/aarch64/aarch64/percpu.h new file mode 100644 index 00000000..86f4ff48 --- /dev/null +++ b/aarch64/aarch64/percpu.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_PERCPU_ +#define _AARCH64_PERCPU_ + + +struct percpu; + +#if NCPUS > 1 +static inline __attribute__((const)) struct percpu *_my_percpu(void) +{ + struct percpu *p; + + asm("mrs %0, TPIDR_EL1" : "=r"(p)); + return p; +} +#else +#define _my_percpu() (&percpu_array[0]) +#endif + +#define percpu_assign(stm, val) _my_percpu()->stm = (val) +#define percpu_get(typ, stm) (_my_percpu()->stm) +#define percpu_ptr(type, stm) (&_my_percpu()->stm) + +#include <kern/processor.h> +#include <kern/kern_types.h> +#include "aarch64/pmap.h" + +struct percpu { + struct processor processor; + thread_t active_thread; + vm_offset_t active_stack; + boolean_t in_irq_from_el0 : 1; + thread_t fpu_thread; + pmap_mapwindow_t mapwindows[PMAP_NMAPWINDOWS]; +/* + struct machine_slot machine_slot; + ast_t need_ast; + ipc_kmsg_t ipc_kmsg_cache; + pmap_update_list cpu_update_list; + spl_t saved_ipl; + timer_data_t kernel_timer; + timer_t current_timer; + unsigned long in_interrupt; +*/ +}; + +extern struct percpu percpu_array[NCPUS]; + +void init_percpu(int cpu); + +#endif /* _AARCH64_PERCPU_ */ diff --git a/aarch64/aarch64/pmap.c b/aarch64/aarch64/pmap.c new file mode 100644 index 00000000..d8e558c3 --- /dev/null +++ b/aarch64/aarch64/pmap.c @@ -0,0 +1,1207 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "aarch64/pmap.h" +#include "aarch64/vm_param.h" +#include "aarch64/hwcaps.h" +#include "aarch64/bits/pte.h" +#include "aarch64/bits/mair.h" +#include "aarch64/bits/tcr.h" +#include "aarch64/bits/sctlr.h" +#include <vm/pmap.h> +#include <vm/vm_page.h> +#include <vm/vm_kern.h> +#include <kern/slab.h> +#include <kern/printf.h> +#include <device/dtb.h> +#include <string.h> + +struct pmap { + pt_entry_t *l0_base; /* TTBR0 */ + unsigned short asid; + int ref_count; + decl_simple_lock_data(,lock) /* lock on map */ + struct pmap_statistics stats; /* map statistics */ +}; + +#define TTBR_ASID(x) (((x) >> 48) & 0xff) +#define TTBR_MAKE_ASID(asid) (((vm_offset_t) asid) << 48) +#define TTBR_ASID_MASK 0xffff000000000000UL +#define TTBR_BADDR_MASK 0x0000fffffffffffeUL /* translation table base address */ + +static boolean_t pmap_initialized = FALSE; + +static struct pmap kernel_pmap_store; +pmap_t kernel_pmap; +static pt_entry_t *ttbr1_l0_base; + +static struct kmem_cache pmap_cache; +static struct kmem_cache table_cache; + +typedef struct pv_entry { + struct pv_entry *next; + pmap_t pmap; + vm_offset_t va; +} *pv_entry_t; + +#define PV_ENTRY_NULL ((pv_entry_t) 0) + +static pv_entry_t pv_head_table; +static pv_entry_t pv_free_list; +def_simple_lock_data(static, pv_free_list_lock) + +static pv_entry_t pv_alloc(void) +{ + pv_entry_t pv_e; + + simple_lock(&pv_free_list_lock); + pv_e = pv_free_list; + if (pv_e != PV_ENTRY_NULL) + pv_free_list = pv_free_list->next; + simple_unlock(&pv_free_list_lock); + + return pv_e; +} + +static void pv_free(pv_entry_t pv_e) +{ + simple_lock(&pv_free_list_lock); + pv_e->next = pv_free_list; + pv_free_list = pv_e->next; + simple_unlock(&pv_free_list_lock); +} + +static struct kmem_cache pv_list_cache; + + + +#if NCPUS > 1 + +#error "Implement" + +static lock_data_t pmap_system_lock; + +#define PMAP_READ_LOCK(pmap, spl) \ +MACRO_BEGIN \ + SPLVM(spl); \ + lock_read(&pmap_system_lock); \ + simple_lock(&(pmap)->lock); \ +MACRO_END + +#define PMAP_READ_UNLOCK(pmap, spl) \ +MACRO_BEGIN \ + simple_unlock(&(pmap)->lock); \ + lock_read_done(&pmap_system_lock); \ + SPLX(spl); \ +MACRO_END + +#define PMAP_WRITE_LOCK(spl) \ +MACRO_BEGIN \ + SPLVM(spl); \ + lock_write(&pmap_system_lock); \ +MACRO_END + +#define PMAP_WRITE_UNLOCK(spl) \ +MACRO_BEGIN \ + lock_write_done(&pmap_system_lock); \ + SPLX(spl); \ +NACRO_END + +#else + +#define SPLVM(spl) (spl) = 0 +#define SPLX(spl) (void) (spl) + +#define PMAP_READ_LOCK(pmap, spl) SPLVM(spl) +#define PMAP_READ_UNLOCK(pmap, spl) SPLX(spl) +#define PMAP_WRITE_LOCK(spl) SPLVM(spl) +#define PMAP_WRITE_UNLOCK(spl) SPLX(spl) + +#endif + + +/* + * Range of kernel virtual addresses available for kernel memory mapping. + * Does not include the virtual addresses used to map physical memory 1-1. + * Initialized by pmap_bootstrap. + */ +vm_offset_t kernel_virtual_start; +vm_offset_t kernel_virtual_end; + +/* + * The (single largest) region of physical memory. + */ +static phys_addr_t phys_mem_start; +static vm_size_t phys_mem_size; + +extern const void __text_start; +extern const void _image_end; + +/* + * Two slots for temporary physical page mapping, to allow for + * physical-to-physical transfers. + */ +#define MAPWINDOW_SIZE (PMAP_NMAPWINDOWS * NCPUS * PAGE_SIZE) +static void pmap_init_mapwindows(void); + +static boolean_t valid_page(phys_addr_t addr) +{ + if (!pmap_initialized) + return FALSE; + return vm_page_lookup_pa(addr) != VM_PAGE_NULL; +} + +#define pa_index(pa) vm_page_table_index(pa) +#define pai_to_pvh(pai) (&pv_head_table[pai]) + +/* + * Early physical memory heap. + */ +static vm_offset_t heap_start; + +vm_offset_t pmap_grab_page(void) +{ + vm_offset_t res = heap_start; + + heap_start += PAGE_SIZE; + + return res; +} + +static void __attribute__((noinline)) pmap_ungrab_page(vm_offset_t page) +{ + if (page + PAGE_SIZE == heap_start) + heap_start = page; +} + + +static inline void cache_flush(void) +{ + asm volatile( + "dsb st\n\t" + "isb sy" + ::: "memory"); +} + +/* TODO: callers of this should pass VAE1IS, once we mark memory as shareable */ +#define TLB_FLUSH(kind, arg) \ +MACRO_BEGIN \ + asm volatile("dsb ishst"); \ + asm volatile("tlbi " kind ", %0" :: "r"(arg)); \ +MACRO_END + +void pmap_discover_physical_memory(dtb_node_t node) +{ + struct dtb_prop prop; + dtb_t dtb; + phys_addr_t start, kernel_start; + vm_size_t size, dtb_size; + vm_size_t off = 0; + + prop = dtb_node_find_prop(node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + + /* + * TODO: We currently only consider a single largest + * region of memory. It appears to be a limitation + * of the vm_page module, it can only handle a single + * region at the given "seg_index", of which there are + * only 4? + */ + + while (off < prop.length) { + start = dtb_prop_read_cells(&prop, + node->address_cells, + &off); + size = dtb_prop_read_cells(&prop, + node->size_cells, + &off); + + if (size > phys_mem_size) { + phys_mem_start = start; + phys_mem_size = size; + } + } + + assert(phys_mem_size > 0); + /* TODO: is VM_PAGE_SEG_DMA appropriate here? */ + vm_page_load(VM_PAGE_SEG_DMA, + phys_mem_start, + phys_mem_start + phys_mem_size); + + /* + * If the kernel itself or the DTB is loaded in + * this region of memory, exclude them form the heap. + */ + kernel_start = (phys_addr_t) &__text_start; + heap_start = phys_mem_start; + if (kernel_start >= heap_start + && kernel_start < phys_mem_start + phys_mem_size) { + heap_start = round_page((phys_addr_t) &_image_end); + } + dtb_get_location(&dtb, &dtb_size); + if ((phys_addr_t) dtb >= heap_start + && (phys_addr_t) dtb < phys_mem_start + phys_mem_size) { + heap_start = round_page(((phys_addr_t) dtb) + dtb_size); + } +} + +_Static_assert((1UL << VM_AARCH64_T0SZ) == VM_MAX_USER_ADDRESS); +_Static_assert((1UL << VM_AARCH64_T1SZ) + VM_MIN_KERNEL_ADDRESS == 0UL); + +#define BITS_PER_LEVEL 9 /* 4K granularity */ +#define NEXT_SB(sb) (((sb) - PAGE_SHIFT - 1) / BITS_PER_LEVEL * BITS_PER_LEVEL + PAGE_SHIFT) +#define TT_INDEX(v, sb, nsb) (((v) >> (nsb)) & ((1UL << ((sb) - (nsb))) - 1)) + +/* + * Bootstrap the system enough to run with virtual memory. + * Allocate the kernel page translation tables, + * and direct-map all physical memory. + * Called with mapping off. + */ +void __attribute__((target("branch-protection=none"))) pmap_bootstrap(void) +{ +#if (VM_AARCH64_T0SZ != 48) || (VM_AARCH64_T1SZ != 48) +#error "Rework the logic below" +#endif + phys_addr_t kernel_block_t0_l1; + unsigned long kernel_block_t0_l1_index; + phys_addr_t kernel_block_t1_l1; + unsigned long kernel_block_t1_l1_index; + + uint64_t sctlr; + uintptr_t scratch1, scratch2; + pt_entry_t *phys_ttbr0_l0_base; + pt_entry_t *phys_ttbr1_l0_base; + pt_entry_t *phys_ttbr0_l1_base; + pt_entry_t *phys_ttbr1_l1_base; + pt_entry_t kernel_mapping_bti; + + /* + * When the kernel itself is compiled with BTI + * (enabled with -mbranch-protection=bti in GCC), + * enable BTI enforcement for the kernel mapping. + */ +#ifdef __ARM_FEATURE_BTI_DEFAULT + kernel_mapping_bti = AARCH64_PTE_BTI; +#else + kernel_mapping_bti = 0; +#endif + + kernel_block_t0_l1_index = TT_INDEX((phys_addr_t) &__text_start, 36, NEXT_SB(36)); + kernel_block_t0_l1 = kernel_block_t0_l1_index << NEXT_SB(36); + kernel_block_t1_l1_index = TT_INDEX((phys_addr_t) &__text_start, 36, NEXT_SB(36)); + kernel_block_t1_l1 = kernel_block_t1_l1_index << NEXT_SB(36); + + phys_ttbr1_l0_base = (pt_entry_t*)pmap_grab_page(); + phys_ttbr1_l1_base = (pt_entry_t*)pmap_grab_page(); + + ttbr1_l0_base = (pt_entry_t*)phystokv(phys_ttbr1_l0_base); + + /* Make sure to grab these ones last, so they can be then released. */ + phys_ttbr0_l0_base = (pt_entry_t*)pmap_grab_page(); + phys_ttbr0_l1_base = (pt_entry_t*)pmap_grab_page(); + + memset(phys_ttbr0_l0_base, 0, PAGE_SIZE); + memset(phys_ttbr0_l1_base, 0, PAGE_SIZE); + /* Temporary identity map. */ + phys_ttbr0_l0_base[0] = (phys_addr_t) phys_ttbr0_l1_base + | AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_TABLE + | AARCH64_PTE_VALID + | AARCH64_PTE_UXN + | kernel_mapping_bti + | AARCH64_PTE_NON_SH /* ? */; + phys_ttbr0_l1_base[kernel_block_t0_l1_index] = kernel_block_t0_l1 + | AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_BLOCK + | AARCH64_PTE_VALID + | AARCH64_PTE_UXN + | kernel_mapping_bti + | AARCH64_PTE_NON_SH /* ? */; + + /* The upper map. */ + memset(phys_ttbr1_l0_base, 0, PAGE_SIZE); + memset(phys_ttbr1_l1_base, 0, PAGE_SIZE); + /* TODO: This assumes that physical memory is in the first block. */ + phys_ttbr1_l0_base[0] = (phys_addr_t) phys_ttbr1_l1_base + | AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_TABLE + | AARCH64_PTE_VALID + | AARCH64_PTE_UXN + | AARCH64_PTE_NON_SH /* ? */; + phys_ttbr1_l1_base[0] = 0x0 + | AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_BLOCK + | AARCH64_PTE_VALID + | AARCH64_PTE_UXN + | AARCH64_PTE_PXN + | AARCH64_PTE_NON_SH /* ? */; + /* FIXME: MAIR device index... */ + phys_ttbr1_l1_base[kernel_block_t1_l1_index] = kernel_block_t1_l1 + | AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_BLOCK + | AARCH64_PTE_VALID + | AARCH64_PTE_UXN + | kernel_mapping_bti + | AARCH64_PTE_NON_SH /* ? */; + + /* + * Determine the kernel virtual address range. + * It starts at the end of the physical memory + * mapped into the kernel address space, + * and extends to a stupid arbitrary limit beyond that. + */ + kernel_virtual_start = phystokv(kernel_block_t1_l1 + (1UL << NEXT_SB(36))); + kernel_virtual_end = kernel_virtual_start + VM_KERNEL_MAP_SIZE; + + sctlr = SCTLR_M | SCTLR_SA | SCTLR_SA0 | SCTLR_UCT | SCTLR_UCI + | SCTLR_BT0 | SCTLR_BT1; + if (hwcap_internal & HWCAP_INT_EPAN) + sctlr |= SCTLR_EPAN; + if (hwcaps[0] & HWCAP_SSBS) + sctlr |= SCTLR_SSBS; + + /* Enable the MMU. */ + asm volatile("msr MAIR_EL1, %0" :: "r"(MAIR_VALUE)); + asm volatile("msr TCR_EL1, %0" :: "r"(TCR_VALUE)); + asm volatile("msr TTBR0_EL1, %0" :: "r"(phys_ttbr0_l0_base)); + asm volatile("msr TTBR1_EL1, %0" :: "r"(phys_ttbr1_l0_base)); + asm volatile( + "isb sy\n\t" + "msr SCTLR_EL1, %[sctlr]\n\t" + "dsb st\n\t" + "isb sy\n\t" + /* Adjust sp, x29 to high memory. */ + "mov %[scratch1], #%[min_addr]\n\t" + "add sp, sp, %[scratch1]\n\t" + "add x29, x29, %[scratch1]\n\t" + /* Jump to high memory. */ + "adr %[scratch2], .here\n\t" + "add %[scratch2], %[scratch2], %[scratch1]\n\t" + "br %[scratch2]\n" + ".here:\n\t" +#ifdef __ARM_FEATURE_BTI_DEFAULT + "bti j\n\t" +#endif + /* Now adjust the saved x29 / x30. */ + "ldp %[scratch2], x30, [x29]\n\t" + "add %[scratch2], %[scratch2], %[scratch1]\n\t" + "add x30, x30, %[scratch1]\n\t" + "stp %[scratch2], x30, [x29]\n\t" + /* + * Unload the identity mapping, just to make + * sure we crash if anything still references it. + */ + "mov %[scratch2], #-1\n\t" + "msr TTBR0_EL1, %[scratch2]" + : + [scratch1] "=&r"(scratch1), + [scratch2] "=&r"(scratch2) + : + [sctlr] "r"(sctlr), + [min_addr] "i"(VM_MIN_KERNEL_ADDRESS) + : "memory" + ); + + /* + * Release the pages used for the identity mapping, + * since we no longer need them. This must be done + * in the reverse order compared to how we grabbed + * them above. + */ + pmap_ungrab_page((vm_offset_t) phys_ttbr0_l1_base); + pmap_ungrab_page((vm_offset_t) phys_ttbr0_l0_base); +} + +void pmap_bootstrap_misc(void) +{ + extern boolean_t vm_fault_dirty_handling; + + kernel_pmap = &kernel_pmap_store; +#if NCPUS > 1 + lock_init(&pmap_system_lock, FALSE); +#endif + simple_lock_init(&kernel_pmap->lock); + kernel_pmap->ref_count = 1; + kernel_pmap->l0_base = PT_ENTRY_NULL; + kernel_pmap->asid = 0; + + vm_page_load_heap(VM_PAGE_SEG_DMA, heap_start, phys_mem_start + phys_mem_size); + pmap_init_mapwindows(); + + vm_fault_dirty_handling = TRUE; +} + +void pmap_virtual_space( + vm_offset_t *startp, + vm_offset_t *endp) +{ + *startp = kernel_virtual_start; + *endp = kernel_virtual_end - MAPWINDOW_SIZE; +} + +void pmap_activate_user(pmap_t p) +{ + vm_offset_t ttbr0, asid_hi; + + asid_hi = TTBR_MAKE_ASID(1); + ttbr0 = kvtophys(p->l0_base) | asid_hi; + asm volatile("msr TTBR0_EL1, %0" :: "r"(ttbr0)); + TLB_FLUSH("aside1", asid_hi); + cache_flush(); +} + +void pmap_init(void) +{ + kern_return_t kr; + unsigned long npages; + vm_offset_t addr; + + npages = vm_page_table_size(); + + kr = kmem_alloc_wired(kernel_map, &addr, sizeof(struct pv_entry) * npages); + assert(kr == KERN_SUCCESS); + pv_head_table = (pv_entry_t) addr; + + kmem_cache_init(&pmap_cache, "pmap", sizeof(struct pmap), alignof(struct pmap), NULL, 0); + kmem_cache_init(&table_cache, "page table", PAGE_SIZE, PAGE_SIZE, NULL, KMEM_CACHE_PHYSMEM); + kmem_cache_init(&pv_list_cache, "pv_entry", sizeof(struct pv_entry), alignof(struct pv_entry), NULL, 0); + + pmap_initialized = TRUE; +} + +#ifdef notyet +static unsigned short next_asid; +#endif + +pmap_t pmap_create(vm_size_t size) +{ + pmap_t p; + + if (size != 0) + return PMAP_NULL; + + p = (pmap_t) kmem_cache_alloc(&pmap_cache); + if (p == PMAP_NULL) + return PMAP_NULL; + + p->l0_base = (pt_entry_t*)kmem_cache_alloc(&table_cache); + if (p->l0_base == PT_ENTRY_NULL) { + kmem_cache_free(&pmap_cache, (vm_offset_t)p); + return PMAP_NULL; + } + memset(p->l0_base, 0, PAGE_SIZE); + +#ifdef notyet +#if NCPUS > 1 +#error "Need some synch here" +#endif + p->asid = ++next_asid; + if (likely(hwcap_internal & HWCAP_INT_ASID16)) + p->asid &= 0xffff; + else + p->asid &= 0xff; +#else + p->asid = 1; +#endif + + p->ref_count = 1; + simple_lock_init(&p->lock); + + p->stats.resident_count = 0; + p->stats.wired_count = 0; + + return p; +} + +integer_t pmap_resident_count(pmap_t pmap) +{ + return pmap->stats.resident_count; +} + +void pmap_reference(pmap_t pmap) +{ + simple_lock(&pmap->lock); + pmap->ref_count++; + simple_unlock(&pmap->lock); +} + +static void pmap_destroy_table( + pt_entry_t *table, + unsigned char significant_bits) +{ + unsigned index, table_size; + unsigned char next_sb; + boolean_t last_level; + pt_entry_t entry; + pt_entry_t *next_table; + + assert(significant_bits > PAGE_SHIFT); + next_sb = NEXT_SB(significant_bits); + last_level = (next_sb == PAGE_SHIFT); + table_size = 1U << (significant_bits - next_sb); + + if (!last_level) { + for (index = 0; index < table_size; index++) { + entry = table[index]; + if (!(entry & AARCH64_PTE_VALID)) { + assert(entry == 0); + continue; + } + if (entry & AARCH64_PTE_TABLE) { + next_table = (pt_entry_t*)(entry & AARCH64_PTE_ADDR_MASK); + assert(next_table != PT_ENTRY_NULL); + next_table = (pt_entry_t*)phystokv(next_table); + pmap_destroy_table(next_table, next_sb); + } + } + } + + kmem_cache_free(&table_cache, (vm_offset_t) table); +} + +void pmap_destroy(pmap_t pmap) +{ + int c; + + simple_lock(&pmap->lock); + c = --pmap->ref_count; + simple_unlock(&pmap->lock); + + if (c != 0) + return; + + assert(pmap != kernel_pmap); + assert(pmap->stats.wired_count == 0); + + pmap_destroy_table(pmap->l0_base, VM_AARCH64_T0SZ); + kmem_cache_free(&pmap_cache, (vm_offset_t) pmap); +} + +static pt_entry_t pmap_prot(vm_offset_t v, vm_prot_t prot) +{ + pt_entry_t entry = 0; + + if (!(prot & VM_PROT_WRITE)) + entry |= AARCH64_PTE_READ_ONLY; + + if (v >= VM_MIN_KERNEL_ADDRESS) { + assert(!(prot & VM_PROT_EXECUTE)); + entry |= AARCH64_PTE_PXN; + entry |= AARCH64_PTE_UXN; + } else { + if (prot & (VM_PROT_READ | VM_PROT_WRITE)) + entry |= AARCH64_PTE_EL0_ACCESS; + entry |= AARCH64_PTE_PXN; + if (!(prot & VM_PROT_EXECUTE)) + entry |= AARCH64_PTE_UXN; + /* + * With EPAN, we can have truly execute-only + * mappings in EL0. But if EPAN is not available, + * forcefully enable EL0 read access. + */ + else if (!(hwcap_internal & HWCAP_INT_EPAN)) + entry |= AARCH64_PTE_EL0_ACCESS; + } + + return entry; +} + +static void pmap_reduce_prot( + pt_entry_t *entry, + pt_entry_t prot_mask) +{ + *entry |= AARCH64_PTE_PXN; + + if (prot_mask & AARCH64_PTE_UXN) + *entry |= AARCH64_PTE_UXN; + if (prot_mask & AARCH64_PTE_READ_ONLY) + *entry |= AARCH64_PTE_READ_ONLY; + + if (!(prot_mask & AARCH64_PTE_EL0_ACCESS)) + *entry &= ~AARCH64_PTE_EL0_ACCESS; + + /* + * XXX: Convince myself that this cannot lead to + * execute-only mappings unless we have EPAN. + */ +} + +static kern_return_t pmap_walk( + pmap_t pmap, + pt_entry_t *table, + vm_offset_t v, + unsigned char significant_bits, + int *spl, + boolean_t create, + vm_prot_t prot, + pt_entry_t **out_entry) +{ + unsigned index; + pt_entry_t entry; + unsigned char next_sb; + boolean_t last_level; + pt_entry_t *next_table; + + assert(significant_bits > PAGE_SHIFT); + next_sb = NEXT_SB(significant_bits); + last_level = (next_sb == PAGE_SHIFT); + + index = TT_INDEX(v, significant_bits, next_sb); + assert(index < PAGE_SIZE / sizeof(pt_entry_t)); + +Retry: + entry = table[index]; + if (!(entry & AARCH64_PTE_VALID)) { + assert(entry == 0); + if (!create) + return KERN_INVALID_ADDRESS; + + if (!last_level) { + /* + * This PTE will point to the next-level table. + * Allocate and clear that now. Unlock pmap + * while trying to allocate. + */ + PMAP_READ_UNLOCK(pmap, *spl); + if (!pmap_initialized) { + next_table = (pt_entry_t*)phystokv(vm_page_bootalloc(PAGE_SIZE)); + } else { + while (!(next_table = (pt_entry_t*)kmem_cache_alloc(&table_cache))) + VM_PAGE_WAIT(0); + } + memset(next_table, 0, PAGE_SIZE); + PMAP_READ_LOCK(pmap, *spl); + entry = table[index]; + if (entry & AARCH64_PTE_VALID) { + /* + * Someone else has got to entering the + * table before we did. No big deal, + * proceed as if nothing happened. We + * assume that concurrent pmap calls don't + * otherwise conflict with each other, i.e. + * they use appropriate locking at a higher + * level (vm_map). + */ + PMAP_READ_UNLOCK(pmap, *spl); + assert(pmap_initialized); + kmem_cache_free(&table_cache, (vm_offset_t) next_table); + PMAP_READ_LOCK(pmap, *spl); + goto Retry; + } + entry = kvtophys(next_table) | AARCH64_PTE_TABLE; + } else { + /* + * This PTE will point to a block; its address + * will be filled in by our caller. + */ + entry = AARCH64_PTE_BLOCK | AARCH64_PTE_LEVEL3; + } + entry |= AARCH64_PTE_MAIR_INDEX(MAIR_NORMAL_INDEX) + | AARCH64_PTE_ACCESS + | AARCH64_PTE_VALID + | AARCH64_PTE_NON_SH /* ?? */; + if (v < VM_MAX_USER_ADDRESS) + entry |= AARCH64_PTE_NG; + + entry |= pmap_prot(v, prot); + table[index] = entry; + } + + if (!(entry & AARCH64_PTE_TABLE) || last_level) { + *out_entry = &table[index]; + return KERN_SUCCESS; + } + + next_table = (pt_entry_t*)(entry & AARCH64_PTE_ADDR_MASK); + assert(next_table != PT_ENTRY_NULL); + next_table = (pt_entry_t*)phystokv(next_table); + return pmap_walk(pmap, next_table, v, next_sb, spl, create, prot, out_entry); +} + +static void pmap_walk_range( + pt_entry_t *table, + vm_offset_t sva, + vm_offset_t eva, + unsigned char significant_bits, + void (*callback)(vm_offset_t, pt_entry_t*, void*), + void *data) +{ + pt_entry_t entry, *next_table; + unsigned char next_sb; + unsigned index; + unsigned table_size; + boolean_t last_level; + vm_offset_t v; + + assert(significant_bits > PAGE_SHIFT); + next_sb = NEXT_SB(significant_bits); + last_level = (next_sb == PAGE_SHIFT); + index = TT_INDEX(sva, significant_bits, next_sb); + table_size = 1U << (significant_bits - next_sb); + v = sva; + + for (v = sva; v < eva && index < table_size; v += (1UL << next_sb), index++) { + entry = table[index]; + if (!(entry & AARCH64_PTE_VALID)) { + assert(entry == 0); + callback(v, PT_ENTRY_NULL, data); + } else if (!(entry & AARCH64_PTE_TABLE) || last_level) { + callback(v, &table[index], data); + } else { + next_table = (pt_entry_t*)(entry & AARCH64_PTE_ADDR_MASK); + assert(next_table != PT_ENTRY_NULL); + next_table = (pt_entry_t*)phystokv(next_table); + pmap_walk_range(next_table, v, eva, next_sb, callback, data); + } + } +} + +static pt_entry_t *pmap_table(pmap_t pmap, vm_offset_t v) +{ + if (v >= VM_MIN_KERNEL_ADDRESS) + return ttbr1_l0_base; + assert(pmap != kernel_pmap); + return pmap->l0_base; +} + +static unsigned char pmap_root_sb(vm_offset_t v) +{ + if (v >= VM_MIN_KERNEL_ADDRESS) + return VM_AARCH64_T1SZ; + return VM_AARCH64_T0SZ; +} + +static void pv_link( + pmap_t pmap, + phys_addr_t pa, + vm_offset_t v, + spl_t *spl) +{ + pv_entry_t pv_e, pv_h; + unsigned long pai; + + if (!valid_page(pa)) + return; + + pv_e = PV_ENTRY_NULL; +Again: + pai = pa_index(pa); + // LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + if (pv_h->pmap == NULL) { + pv_h->pmap = pmap; + pv_h->va = v; + pv_h->next = PV_ENTRY_NULL; + } else { + if (pv_e == PV_ENTRY_NULL) + pv_e = pv_alloc(); + if (pv_e == PV_ENTRY_NULL) { + // UNLOCK_VH(pai); + PMAP_READ_UNLOCK(pmap, *spl); + pv_e = (pv_entry_t) kmem_cache_alloc(&pv_list_cache); + PMAP_READ_LOCK(pmap, *spl); + goto Again; + } + pv_e->pmap = pmap; + pv_e->va = v; + pv_e->next = pv_h->next; + pv_h->next = pv_e; + pv_e = PV_ENTRY_NULL; + } + // UNLOCK_VH(pai); + if (pv_e != PV_ENTRY_NULL) + pv_free(pv_e); +} + +static void pv_unlink( + pmap_t pmap, + phys_addr_t pa, + vm_offset_t v) +{ + pv_entry_t pv_h; + pv_entry_t pv_e; + pv_entry_t pv_prev; + unsigned long pai; + + if (!valid_page(pa)) + return; + + pai = pa_index(pa); + // LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + + assert(pv_h->pmap != PMAP_NULL); + if (pv_h->pmap == pmap && pv_h->va == v) { + pv_e = pv_h->next; + if (pv_e != PV_ENTRY_NULL) { + *pv_h = *pv_e; + pv_free(pv_e); + } else { + pv_h->pmap = PMAP_NULL; + pv_h->va = 0; + } + } else { + pv_e = pv_h; + do { + pv_prev = pv_e; + assert(pv_e->next != PV_ENTRY_NULL); + pv_e = pv_e->next; + } while (pv_e->pmap != pmap || pv_e->va != v); + pv_prev->next = pv_e->next; + pv_free(pv_e); + } + // UNLOCK_PVH(pai); +} + +/* + * Insert the given physical page (p) at + * the specified virtual address (v) in the + * target physical map with the protection requested. + * + * If specified, the page will be wired down, meaning + * that the related pte can not be reclaimed. + * + * NB: This is the only routine which MAY NOT lazy-evaluate + * or lose information. That is, this routine must actually + * insert this page into the given map NOW. + */ +void pmap_enter( + pmap_t pmap, + vm_offset_t v, + phys_addr_t pa, + vm_prot_t prot, + boolean_t wired) +{ + kern_return_t kr; + pt_entry_t *entry; + int spl; + phys_addr_t prev_pa; + + assert(pmap != NULL); + assert(pa != vm_page_fictitious_addr); + + if (pmap == kernel_pmap && (v < kernel_virtual_start || v >= kernel_virtual_end)) + panic("pmap_enter(%#016.lx, %#llx) falls in physical memory area!\n", (unsigned long) v, (unsigned long long) pa); + + if (pmap != kernel_pmap && v >= kernel_virtual_start) + panic("pmap_enter(%#016.lx, %#llx) for a non-kernel pmap?\n", (unsigned long) v, (unsigned long long) pa); + + PMAP_READ_LOCK(pmap, spl); + + kr = pmap_walk(pmap, pmap_table(pmap, v), v, pmap_root_sb(v), &spl, TRUE, prot, &entry); + assert(kr == KERN_SUCCESS); + prev_pa = (*entry) & AARCH64_PTE_ADDR_MASK; + *entry = ((*entry) & ~AARCH64_PTE_ADDR_MASK & ~AARCH64_PTE_PROT_MASK) | (pt_entry_t)pa | pmap_prot(v, prot); + + if (!prev_pa) { /* FIXME what if it was pa 0x0 */ + pmap->stats.resident_count++; + pv_link(pmap, pa, v, &spl); + } else { + assert(prev_pa == pa); + } + + PMAP_READ_UNLOCK(pmap, spl); + + TLB_FLUSH("vae1", (v >> PAGE_SHIFT) | TTBR_MAKE_ASID(pmap->asid)); + cache_flush(); +} + + +phys_addr_t pmap_extract( + pmap_t pmap, + vm_offset_t v) +{ + kern_return_t kr; + pt_entry_t *entry; + phys_addr_t pa; + int spl; + + SPLVM(spl); + simple_lock(&pmap->lock); + + kr = pmap_walk(pmap, pmap_table(pmap, v), v, pmap_root_sb(v), &spl, FALSE, VM_PROT_NONE, &entry); + if (kr != KERN_SUCCESS) + pa = 0; + pa = (*entry) & AARCH64_PTE_ADDR_MASK; + simple_unlock(&pmap->lock); + SPLX(spl); + return pa; +} + +static void pmap_protect_callback( + vm_offset_t v, + pt_entry_t *entry, + void *data) +{ + pt_entry_t prot = (pt_entry_t) data; + + if (entry == PT_ENTRY_NULL || !(*entry & AARCH64_PTE_VALID)) + return; + + pmap_reduce_prot(entry, prot); +} + +void pmap_protect( + pmap_t pmap, + vm_offset_t sva, + vm_offset_t eva, + vm_prot_t prot) +{ + pt_entry_t *table; + vm_offset_t v; + int spl; + + assert(sva < eva); + + SPLVM(spl); + simple_lock(&pmap->lock); + + table = pmap_table(pmap, sva); + assert(pmap_table(pmap, eva - 1) == table); + + pmap_walk_range(table, sva, eva, pmap_root_sb(sva), pmap_protect_callback, (void *) pmap_prot(sva, prot)); + + simple_unlock(&pmap->lock); + SPLX(spl); + + for (v = sva; v < eva; v += PAGE_SIZE) + TLB_FLUSH("vae1", (v >> PAGE_SHIFT) | TTBR_MAKE_ASID(pmap->asid)); + cache_flush(); +} + +static void pmap_remove_callback( + vm_offset_t v, + pt_entry_t *entry, + void *data) +{ + pmap_t pmap; + phys_addr_t pa; + boolean_t was_present; + + was_present = (entry != PT_ENTRY_NULL) && ((*entry) & AARCH64_PTE_VALID); + if (!was_present) + return; + + pa = (phys_addr_t) ((*entry) & AARCH64_PTE_ADDR_MASK); + *entry = 0; + pmap = (pmap_t) data; + pmap->stats.resident_count--; + + pv_unlink(pmap, pa, v); +} + +void pmap_remove( + pmap_t pmap, + vm_offset_t sva, + vm_offset_t eva) +{ + int spl; + pt_entry_t *table; + vm_offset_t v; + + PMAP_READ_LOCK(pmap, spl); + + table = pmap_table(pmap, sva); + assert(pmap_table(pmap, eva - 1) == table); + + pmap_walk_range(table, sva, eva, pmap_root_sb(sva), pmap_remove_callback, pmap); + + PMAP_READ_UNLOCK(pmap, spl); + + for (v = sva; v < eva; v += PAGE_SIZE) + TLB_FLUSH("vae1", (v >> PAGE_SHIFT) | TTBR_MAKE_ASID(pmap->asid)); + cache_flush(); +} + +void pmap_page_protect( + phys_addr_t phys, + vm_prot_t prot) +{ + kern_return_t kr; + int spl; + unsigned long pai; + pv_entry_t pv_h, pv_e, pv_next; + pmap_t pmap; + pt_entry_t *entry; + vm_offset_t v; + + assert(phys != vm_page_fictitious_addr); + + if (!valid_page(phys)) + return; + + PMAP_WRITE_LOCK(spl); + + pai = pa_index(phys); + // LOCK_PVH(pai); + pv_h = pai_to_pvh(pai); + if (pv_h->pmap == PMAP_NULL) + goto Out; + + pv_e = pv_h; + + do { + pv_next = pv_e->next; + pmap = pv_e->pmap; + simple_lock(&pmap->lock); + v = pv_e->va; + kr = pmap_walk(pmap, pmap_table(pmap, v), v, pmap_root_sb(v), &spl, FALSE, VM_PROT_NONE, &entry); + assert(kr == KERN_SUCCESS); + assert((*entry) & AARCH64_PTE_VALID); + assert(((*entry) & AARCH64_PTE_ADDR_MASK) == phys); + + if (prot == VM_PROT_NONE) { + /* + * We're removing the physical page from all pmaps. + * Don't bother with updating previous pv_e. + */ + pmap->stats.resident_count--; + *entry = 0; + if (pv_e == pv_h) { + pv_e->pmap = PMAP_NULL; + pv_e->va = 0; + pv_e->next = PV_ENTRY_NULL; + } else { + pv_free(pv_e); + } + } else { + /* + * Just reduce the protection. + */ + pmap_reduce_prot(entry, pmap_prot(v, prot)); + } + + TLB_FLUSH("vae1", (v >> PAGE_SHIFT) | TTBR_MAKE_ASID(pmap->asid)); + + pv_e = pv_next; + } while (pv_e != PV_ENTRY_NULL); + +Out: + // UNLOCK_PVH(pai); + PMAP_WRITE_UNLOCK(spl); + + cache_flush(); +} + +static void pmap_init_mapwindows(void) +{ + kern_return_t kr; + pmap_mapwindow_t *mw; + vm_offset_t v; + int i, cpu, spl = SPL7; + + v = kernel_virtual_end - MAPWINDOW_SIZE; + for (cpu = 0; cpu < NCPUS; cpu++) { + for (i = 0; i < PMAP_NMAPWINDOWS; i++) { + mw = &percpu_array[cpu].mapwindows[i]; + mw->vaddr = v; + kr = pmap_walk(kernel_pmap, ttbr1_l0_base, v, VM_AARCH64_T1SZ, &spl, TRUE, VM_PROT_NONE, &mw->entry); + assert(kr == KERN_SUCCESS); + *mw->entry &= ~AARCH64_PTE_VALID; + v += PAGE_SIZE; + } + } +} + +pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry) +{ + int i; + boolean_t found = FALSE; + pmap_mapwindow_t *mw; + + assert(!(entry & ~(AARCH64_PTE_ADDR_MASK | AARCH64_PTE_PROT_MASK))); + + /* Find an empty one. */ + for (i = 0; i < PMAP_NMAPWINDOWS; i++) { + mw = &percpu_get(pmap_mapwindow_t, mapwindows[i]); + if (!(*mw->entry)) { + found = TRUE; + break; + } + } + assert(found); + *mw->entry |= entry | AARCH64_PTE_VALID | AARCH64_PTE_PXN | AARCH64_PTE_UXN; + cache_flush(); + return mw; +} + +void pmap_put_mapwindow(pmap_mapwindow_t *mw) +{ + *mw->entry &= ~AARCH64_PTE_ADDR_MASK & ~AARCH64_PTE_PROT_MASK & ~AARCH64_PTE_VALID; + // cache_flush(); +} + +void pmap_zero_page(phys_addr_t p) +{ + vm_offset_t v; + pmap_mapwindow_t *mw; + boolean_t direct; + + assert(p != vm_page_fictitious_addr); + direct = p < VM_PAGE_DIRECTMAP_LIMIT; + + if (direct) { + v = phystokv(p); + } else { + mw = pmap_get_mapwindow((pt_entry_t) p); + v = mw->vaddr; + } + + memset((void *) v, 0, PAGE_SIZE); + + if (!direct) + pmap_put_mapwindow(mw); +} + +void pmap_copy_page(phys_addr_t src, phys_addr_t dst) +{ + vm_offset_t src_v, dst_v; + pmap_mapwindow_t *src_mw = NULL, *dst_mw = NULL; + boolean_t src_direct, dst_direct; + + assert(src != vm_page_fictitious_addr); + src_direct = src < VM_PAGE_DIRECTMAP_LIMIT; + dst_direct = dst < VM_PAGE_DIRECTMAP_LIMIT; + + if (src_direct) { + src_v = phystokv(src); + } else { + src_mw = pmap_get_mapwindow((pt_entry_t) src | AARCH64_PTE_READ_ONLY); + src_v = src_mw->vaddr; + } + if (dst_direct) { + dst_v = phystokv(dst); + } else { + dst_mw = pmap_get_mapwindow((pt_entry_t) dst); + dst_v = dst_mw->vaddr; + } + + memcpy((void *) dst_v, (const void *) src_v, PAGE_SIZE); + + if (!src_direct) + pmap_put_mapwindow(src_mw); + if (!dst_direct) + pmap_put_mapwindow(dst_mw); +} diff --git a/aarch64/aarch64/pmap.h b/aarch64/aarch64/pmap.h new file mode 100644 index 00000000..71d3d710 --- /dev/null +++ b/aarch64/aarch64/pmap.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_PMAP_ +#define _AARCH64_PMAP_ + +#include <kern/lock.h> +#include <mach/machine/vm_param.h> +#include <mach/vm_statistics.h> +#include <mach/kern_return.h> + +typedef phys_addr_t pt_entry_t; +#define PT_ENTRY_NULL ((pt_entry_t *) 0) + +typedef struct pmap *pmap_t; + +#define pmap_attribute(pmap,addr,size,attr,value) (KERN_INVALID_ADDRESS) +#define PMAP_NULL ((pmap_t) 0) + +typedef struct { + pt_entry_t *entry; + vm_offset_t vaddr; +} pmap_mapwindow_t; + +extern pmap_mapwindow_t *pmap_get_mapwindow(pt_entry_t entry); +extern void pmap_put_mapwindow(pmap_mapwindow_t *map); + +#define PMAP_NMAPWINDOWS 2 /* per CPU */ + +extern vm_offset_t kernel_virtual_start; +extern vm_offset_t kernel_virtual_end; + +extern void pmap_activate_user(pmap_t pmap); + +#define PMAP_ACTIVATE_KERNEL(my_cpu) +#define PMAP_DEACTIVATE_KERNEL(my_cpu) +#define PMAP_DEACTIVATE_USER(pmap, th, my_cpu) + +#define PMAP_ACTIVATE_USER(pmap, th, my_cpu) \ +MACRO_BEGIN \ + (void) (th); \ + (void) (my_cpu); \ + if (likely((pmap) != kernel_pmap)) \ + pmap_activate_user(pmap); \ +MACRO_END + +#define pmap_kernel() (kernel_pmap) +#define pmap_phys_address(frame) (frame) +#define pmap_phys_to_frame(phys) (phys) +#define pmap_copy(dst_pmap,src_pmap,dst_addr,len,src_addr) + +extern integer_t pmap_resident_count(pmap_t pmap); + +static inline boolean_t pmap_is_modified(phys_addr_t) { return FALSE; } +static inline boolean_t pmap_is_referenced(phys_addr_t) { return FALSE; } +static inline void pmap_clear_modify(phys_addr_t) {} +static inline void pmap_clear_reference(phys_addr_t) {} + + +struct dtb_node; +extern void pmap_discover_physical_memory(const struct dtb_node *node); +extern void pmap_bootstrap(void); +extern void pmap_bootstrap_misc(void); + +extern void pmap_zero_page(phys_addr_t); +extern void pmap_copy_page(phys_addr_t, phys_addr_t); + +#endif /* _AARCH64_PMAP_ */ diff --git a/aarch64/aarch64/setjmp.h b/aarch64/aarch64/setjmp.h new file mode 100644 index 00000000..8cbd9b8e --- /dev/null +++ b/aarch64/aarch64/setjmp.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* TODO */ diff --git a/aarch64/aarch64/smp.h b/aarch64/aarch64/smp.h new file mode 100644 index 00000000..b04ca834 --- /dev/null +++ b/aarch64/aarch64/smp.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_SMP_H_ +#define _AARCH64_SMP_H_ + +int smp_init(void); +void smp_remote_ast(unsigned apic_id); +void smp_pmap_update(unsigned apic_id); +void smp_startup_cpu(unsigned apic_id, unsigned vector); + +#define cpu_pause() asm volatile("yield" ::: "memory"); + +#endif /* _AARCH64_SMP_H_ */ diff --git a/aarch64/aarch64/spl.h b/aarch64/aarch64/spl.h new file mode 100644 index 00000000..e7369f5f --- /dev/null +++ b/aarch64/aarch64/spl.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _MACHINE_SPL_H_ +#define _MACHINE_SPL_H_ + +/* + * This file defines the interrupt priority levels used by + * machine-dependent code. + */ + +typedef int spl_t; + +#define SPL0 0 +#define SPL7 7 + +/* Enable interrupts. */ +static inline void spl0(void) +{ + __atomic_signal_fence(__ATOMIC_RELEASE); + asm volatile("msr DAIFClr, #7"); +} + +/* Disable interrupts, returning previous SPL. */ +static inline spl_t spl7(void) +{ + long daif; + + asm volatile("mrs %0, DAIF" : "=r"(daif)); + asm volatile("msr DAIFSet, #7"); + __atomic_signal_fence(__ATOMIC_ACQUIRE); + + /* + * 0x3c0 is SPSR_DAIF, but we'd rather avoid + * including "aarch64/bits/spsr.h" into this + * widely-used header. + */ + return (daif & 0x3c0) ? SPL7 : SPL0; +} + +static inline void splx(spl_t spl) +{ + if (spl == SPL0) + spl0(); +} + +static inline void spl0_irq(void) +{ + __atomic_signal_fence(__ATOMIC_RELEASE); +} + +static inline void spl7_irq(void) +{ + __atomic_signal_fence(__ATOMIC_ACQUIRE); +} + +#define splhigh spl7 +#define splsoftclock spl7 +#define splnet spl7 +#define splhdw spl7 +#define splbio spl7 +#define spldcm spl7 +#define spltty spl7 +#define splimp spl7 +#define splvm spl7 +#define splclock spl7 +#define splsched spl7 + +#define spl1 spl7 +#define spl2 spl7 +#define spl3 spl7 +#define spl4 spl7 +#define spl5 spl7 +#define spl6 spl7 + +#define assert_splsched() assert(splsched() == SPL7) + +extern int spl_init; + +static inline void setsoftclock(void) +{ + __builtin_unreachable(); +} + +#endif /* _MACHINE_SPL_H_ */ diff --git a/aarch64/aarch64/strings.c b/aarch64/aarch64/strings.c new file mode 100644 index 00000000..7bf47bc6 --- /dev/null +++ b/aarch64/aarch64/strings.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <string.h> +#include <sys/types.h> + +/* Nothing Aarch64-specific about these. */ + +void *memset(void *_s, int c, size_t n) +{ + char *s = _s; + size_t i; + + for (i = 0; i < n ; i++) + s[i] = c; + + return _s; +} + +void *memcpy(void *_d, const void *_s, size_t n) +{ + char *d = _d; + const char *s = _s; + size_t i; + + for (i = 0; i < n; i++) + d[i] = s[i]; + + return _d; +} + +int memcmp(const void *_s1, const void *_s2, size_t n) +{ + const char *s1 = _s1; + const char *s2 = _s2; + size_t i; + + for (i = 0; i < n; i++) { + if (s1[i] != s2[i]) + return s1[i] - s2[i]; + } + + return 0; +} diff --git a/aarch64/aarch64/task.c b/aarch64/aarch64/task.c new file mode 100644 index 00000000..8b71c3fa --- /dev/null +++ b/aarch64/aarch64/task.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <machine/task.h> + +void machine_task_module_init (void) +{ +} + +void machine_task_init (task_t) +{ + /* FIXME: inherit from parent? */ +} + +void machine_task_terminate (task_t) +{ + /* Nothing to do here */ +} + +void machine_task_collect (task_t) +{ + /* Nor here */ +} diff --git a/aarch64/aarch64/task.h b/aarch64/aarch64/task.h new file mode 100644 index 00000000..6638d784 --- /dev/null +++ b/aarch64/aarch64/task.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_TASK_ +#define _AARCH64_TASK_ + +#include <kern/kern_types.h> + +struct aarch64_pac_keys +{ + /* FIXME: are PAC keys even 64-bit? */ + uint64_t ia; + uint64_t ib; + uint64_t da; + uint64_t db; +}; + +/* The machine specific data of a task. */ +struct machine_task +{ + struct aarch64_pac_keys apk; +}; +typedef struct machine_task machine_task_t; + +/* Initialize the machine task module. The function is called once at + start up by task_init in kern/task.c. */ +void machine_task_module_init (void); + +/* Initialize the machine specific part of task TASK. */ +void machine_task_init (task_t); + +/* Destroy the machine specific part of task TASK and release all + associated resources. */ +void machine_task_terminate (task_t); + +/* Try to release as much memory from the machine specific data in + task TASK. */ +void machine_task_collect (task_t); + +#endif /* _AARCH64_TASK_ */ diff --git a/aarch64/aarch64/thread.h b/aarch64/aarch64/thread.h new file mode 100644 index 00000000..2d53f2e5 --- /dev/null +++ b/aarch64/aarch64/thread.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_THREAD_ +#define _AARCH64_THREAD_ + +#include "aarch64/vm_param.h" +#include "mach/machine/thread_status.h" + +/* + * The state of a blocked kernel thread, see Switch_context(). + * Since blocking looks like a function call, we only have to + * save callee-saved registers. + */ +struct aarch64_kernel_state { + long k_regs[12]; /* x19 to x30 */ + long k_sp; +}; +#define AKS_REG(aks, reg) ((aks)->k_regs[(reg) - 19]) + +/* + * The state saved on the stack when taking an exception from + * EL1. We only have to save the caller-saved registers. + */ +struct aarch64_kernel_exception_state { + long x[19]; /* x0 to x18 */ + void *x30; + void *pc; + long cpsr; +}; + +typedef struct pcb { + struct aarch64_float_state *afs; + _Alignas(16) struct aarch64_thread_state ats; + long esr; + long far; +} *pcb_t; + +#define USER_REGS(thread) (&(thread)->pcb->ats) + +struct aarch64_exception_link { + /* points to pcb->ats */ + struct aarch64_thread_state *saved_state; +}; + +#define STACK_AKS(stack) \ + ((struct aarch64_kernel_state *)((stack) + KERNEL_STACK_SIZE) - 1) +#define STACK_AEL(stack) \ + ((struct aarch64_exception_link *)STACK_AKS(stack) - 1) +#define STACK_AKS_REG(stack, reg) AKS_REG(STACK_AKS(stack), reg) + +#endif /* _AARCH64_THREAD_ */ diff --git a/aarch64/aarch64/trap.c b/aarch64/aarch64/trap.c new file mode 100644 index 00000000..712985e0 --- /dev/null +++ b/aarch64/aarch64/trap.c @@ -0,0 +1,683 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "machine/trap.h" +#include "aarch64/locore.h" +#include "aarch64/fpu.h" +#include "aarch64/irq.h" +#include "aarch64/bits/esr.h" +#include <mach/exception.h> +#include <vm/vm_fault.h> +#include <vm/vm_kern.h> +#include <kern/printf.h> +#include <kern/thread.h> +#include <kern/exception.h> + +static inline vm_prot_t fault_prot(unsigned long esr) +{ + /* Instruction aborts only need execute permission. */ + if (ESR_EC(esr) == ESR_EC_IABT_LOWER_EL) + return VM_PROT_EXECUTE; + + assert(ESR_EC(esr) == ESR_EC_DABT_LOWER_EL + || ESR_EC(esr) == ESR_EC_DABT_SAME_EL + || ESR_EC(esr) == ESR_EC_WATCHPT_LOWER_EL + || ESR_EC(esr) == ESR_EC_WATCHPT_SAME_EL); + + /* If WNR is unset, it's a read fault. */ + if (!(esr & ESR_WNR)) + return VM_PROT_READ; + + /* + * WNR is set if this was either an actual write fault, + * or a cache maintenance operation. So if it's not one, + * which is the common case, it's a write fault. + */ + if (likely(!(esr & ESR_CM))) + return VM_PROT_WRITE; + + /* + * For cache maintenance operations, check if DFSC + * indicates a permission fault, and only treat it as + * a write fault in that case. For watchpoints, we can't + * exactly know, so let's guess it's a write fault. + * + * Note that mistaking a read fault for a write fault is + * likely to be fatal for data aborts, and likely to not + * matter much for watchpoints. + */ + if ((ESR_DABT_DFSC(esr) >= ESR_DABT_DFSC_PERM_L0 + && ESR_DABT_DFSC(esr) <= ESR_DABT_DFSC_PERM_L3) + || ESR_DABT_DFSC(esr) == ESR_WATCHPT_DFSC) + return VM_PROT_WRITE; + return VM_PROT_READ; +} + +static void user_page_fault_continue(kern_return_t kr) +{ + pcb_t pcb; + + if (likely(kr == KERN_SUCCESS)) + thread_exception_return(); + + pcb = current_thread()->pcb; + exception(EXC_BAD_ACCESS, kr, pcb->far); +} + +void user_trap_aarch32(void) +{ + panic("Unexpected trap from AArch32 EL0\n"); +} + +void user_trap_irq(void) +{ + spl7_irq(); + percpu_assign(in_irq_from_el0, TRUE); + + assert(root_irq_src); + root_irq_src->handle_irq(root_irq_src); + + spl0_irq(); + percpu_assign(in_irq_from_el0, FALSE); + + thread_exception_return(); +} + +void user_trap_fiq(void) +{ + printf("Got FIQ while in EL0, ignoring for now\n"); + thread_exception_return(); +} + +/* + * TODO: what should we do when FAR crosses a page boundary? + * Can this happen, or do unaligned accesses get reporter as + * two separate ones? + */ + +/* + * Routine: user_trap_sync + * Purpose: + * Handle a synchronous exception from EL0. + * Conditions: + * Exceptions are unmasked. Nothing is locked. User's + * general-purpose registers have been saved into the PCB, + * along with the ESR and FAR. + * Returns: + * Doesn't return, must exit via exception() or + * thread_exception_return() and similar. + */ +void user_trap_sync(void) +{ + pcb_t pcb = current_thread()->pcb; + unsigned long esr = pcb->esr; + vm_offset_t far = pcb->far; + int imm16; + +#if 0 + printf("Sync exc from EL0!\n"); + printf("ESR: %#lx, FAR: %#lx\n", esr, far); +#endif + + switch (ESR_EC(esr)) { + case ESR_EC_UNK: + /* + * Unknown (unallocated) instruction; or one of + * the many misc causes. "This EC code is used + * for all exceptions that are not covered by + * any other EC value." + * + * We get absolutely no additional information + * encoded in ESR (or FAR), so there isn't really + * much we can do here (short of fetching and + * decoding the culprit instruction). So just + * report it to user and hope they can figure + * out what went wrong. + */ + exception(EXC_BAD_INSTRUCTION, EXC_AARCH64_UNK, 0); + + case ESR_EC_WF: + /* + * WFE, WFI, WFET, and WFIT instructions. When + * executed at EL1, these put the CPU into a low- + * power state until an exception happens. + * + * Emulate the same semantics for the user by + * blocking the thread until something aborts it. + * This is primarily meant for VMs, since they + * can't block by performing explicit syscalls, + * and do expect WF* to put the virtual CPU into + * a low-power state. We also allow it for normal + * tasks. + * + * Unlike for SVC & friends, we must advance PC + * explicitly. Do this first, before we go to + * sleep, so it looks like we're blocking on the + * next instruction, same as for SVC. + * + * We don't currently support WFxT properly (it's + * unclear which timer it should work on), so for + * them just make a single attempt to switch to + * another thread, and return without waiting + * otherwise. + */ + pcb->ats.pc += 4; + + switch (ESR_WF_TI(esr)) { + case ESR_WF_TI_WFI: + thread_will_wait(current_thread()); + thread_block(thread_exception_return); + __builtin_unreachable(); + + case ESR_WF_TI_WFE: + case ESR_WF_TI_WFIT: + case ESR_WF_TI_WFET: + default: + // TODO: thread_will_wait_with_timeout + thread_block(thread_exception_return); + __builtin_unreachable(); + } + + case ESR_EC_FP_ACCESS: + /* + * Userland accessed floating point registers + * (typically for SIMD) while floating-point + * access was trapped, because other thread's + * FP state is loaded in the FP registers. + * + * Take the appropriate action to allow the + * current thread to use FP registers, and + * continue as if nothing has happened. + */ + fpu_access_trap(); + thread_exception_return(); + + case ESR_EC_BTI: + /* + * Branch target indentification failure: an + * indirect branch to an instruction that wasn't + * intended to be a target of indirect branches. + * + * We treat it much like a branch to a non- + * executable memory region, so file it under + * EXC_BAD_ACCESS. glibc maps this to SIGILL + * however. + * + * We get BTYPE (indirect branch type) indicated + * in ESR, so pass that to user. + */ + exception(EXC_BAD_ACCESS, EXC_AARCH64_BTI, ESR_BTI_BTYPE(esr)); + + case ESR_EC_IL: + /* + * Illegal execution state. + * + * We would get one of these *from EL1* if we + * attempt to return from an exception with a bad + * execution state indicated in SPSR_EL1. + * + * The only way to get one of these *from EL0* is + * to ask for it explicitly by setting the IL flag + * in PSTATE (CPSR) with a thread_set_state() call + * (see "Legal returns that set PSTATE.IL to 1"). + */ + exception(EXC_BAD_INSTRUCTION, EXC_AARCH64_IL, 0); + + case ESR_EC_SVC: + /* + * The "SVC" (syscall) instruction, with a 16-bit + * immediate argument. Note that ELR (ats.pc) + * gets pointed to the following instruction by + * hardware, so simply returning from the + * exception resumes execution after the sycall. + * + * We recognize "SVC #0" with a valid negated trap + * index (in w8) as a Mach trap, and report a + * dedicated exception otherwise. This could be + * used by a user-level exception handler to + * implement foreign syscall emulation. + * + * Note that handle_syscall() doesn't necesserily + * return normally; some syscalls return to user + * by themselves. + */ + imm16 = ESR_SVC_IMM(esr); + if (likely(imm16 == 0) && handle_syscall(&pcb->ats)) + thread_exception_return(); + exception(EXC_SOFTWARE, EXC_AARCH64_SVC, imm16); + + case ESR_EC_HVC: + /* + * The "HVC" (hypervisor call) instruction, very + * similar to SVC. + * + * We always report these as exceptions; a + * hypervisor running as a user task should know + * what to do with them. + */ +#ifdef notyet + imm16 = ESR_HVC_IMM(esr); + exception(EXC_SOFTWARE, EXC_AARCH64_HVC, imm16); +#else + panic("Virtualization not supported yet\n"); +#endif + + case ESR_EC_SMC: + /* + * The "SMC" (secure monitor call) instruction, + * very similar to SVC/SMC. + */ +#ifdef notyet + imm16 = ESR_SMC_IMM(esr); + exception(EXC_SOFTWARE, EXC_AARCH64_SMC, imm16); +#else + panic("Virtualization not supported yet\n"); +#endif + + case ESR_EC_MRS: + /* We may add a special code for this (EXC_AARCH64_MRS?) */ + exception(EXC_BAD_INSTRUCTION, 0, 0); + + case ESR_EC_PAC: + /* + * Pointer authentication failure. This is only + * generated when we have FEAT_FPAC, otherwise an + * authentication failure simply results in an + * intentionally invalid pointer, which will then + * trap normally if dereferenced. + * + * We treat this like a bad pointer dereference + * (which it will be without FEAT_FPAC), but with + * a special code indicating its PAC nature. We + * get two bits of information (whether A or B and + * whether instruction or data key has been used), + * which we pass on to the user. + */ + exception(EXC_BAD_ACCESS, EXC_AARCH64_PAC, ESR_PAC_INFO(esr)); + + case ESR_EC_IABT_LOWER_EL: + /* + * Instruction abort: invalid address, non- + * executable region, or something of that sort. + */ + if (ESR_IABT_IFSC(esr) >= ESR_IABT_IFSC_SYNC_EXT) + exception(EXC_BAD_INSTRUCTION, 0, 0); /* huh? */ + /* + * If this is not a valid user address, do not + * even bother trying to resolve it. There's + * nothing there as far as user is concerned. + */ + if (unlikely(far >= VM_MAX_USER_ADDRESS)) + exception(EXC_BAD_ACCESS, KERN_INVALID_ADDRESS, far); + + /* + * Resolve the fault against the user's map. + * We pass user_page_fault_continue() as the + * continuation, so this call never returns. + */ + (void) vm_fault(current_map(), trunc_page(far), + fault_prot(esr), + FALSE, FALSE, + user_page_fault_continue); + __builtin_unreachable(); + + case ESR_EC_IABT_SAME_EL: + panic("Same EL exception in EL0 handler\n"); + + case ESR_EC_AL_PC: + /* + * Misaligned PC. FAR and ats.pc both hold the + * misaligned PC value. + */ + exception(EXC_BAD_ACCESS, EXC_AARCH64_AL_PC, far); + + case ESR_EC_DABT_LOWER_EL: + /* + * Data fault. + */ + + /* + * If this is not a valid user address, do not + * even bother trying to resolve it. There's + * nothing there as far as user is concerned. + */ + if (unlikely(far >= VM_MAX_USER_ADDRESS)) + exception(EXC_BAD_ACCESS, KERN_INVALID_ADDRESS, far); + + if (ESR_DABT_DFSC(esr) == ESR_DABT_DFSC_MTE) +#ifdef notyet + exception(EXC_BAD_ACCESS, EXC_AARCH64_MTE, far); +#else + panic("MTE is not supported yet\n"); +#endif + + if (ESR_DABT_DFSC(esr) == ESR_DABT_DFSC_AL) + exception(EXC_BAD_ACCESS, EXC_AARCH64_AL, far); + + /* + * Resolve the fault against the user's map. + * We pass user_page_fault_continue() as the + * continuation, so this call never returns. + */ + (void) vm_fault(current_map(), trunc_page(far), + fault_prot(esr), + FALSE, FALSE, + user_page_fault_continue); + __builtin_unreachable(); + + case ESR_EC_DABT_SAME_EL: + panic("Same EL exception in EL0 handler\n"); + + case ESR_EC_AL_SP: + /* + * SP was not 16-aligned on an SP-relative memory + * access attempt. QEMU doesn't report this. + * + * This doesn't set FAR. + */ + exception(EXC_BAD_ACCESS, EXC_AARCH64_AL_SP, pcb->ats.sp); + + case ESR_EC_FP_EXC: + /* + * Some sort of floating-point exception, but not + * an attempt to access floating-point registers + * when trapped (see the ESR_EC_FP_ACCESS case for + * that), and not missing floating-point support + * altogether (that gets reported as ESR_EC_UNK). + * + * There are a bunch of flags that indicate what + * exactly has happened, but we can only look at + * them if TFV is set. + */ + if (!ESR_FP_EXC_TFV(esr)) + exception(EXC_ARITHMETIC, 0, 0); + + if (ESR_FP_EXC_IDF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_IDF, 0); + if (ESR_FP_EXC_IXF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_IXF, 0); + if (ESR_FP_EXC_UFF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_UFF, 0); + if (ESR_FP_EXC_OFF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_OFF, 0); + if (ESR_FP_EXC_DZF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_DZF, 0); + if (ESR_FP_EXC_IOF(esr)) + exception(EXC_ARITHMETIC, EXC_AARCH64_IOF, 0); + /* Huh? */ + exception(EXC_ARITHMETIC, 0, 0); + + case ESR_EC_SERROR: + panic("SError in sync exc handler\n"); + + case ESR_EC_BREAKPT_LOWER_EL: + /* + * A hardware breakpoint triggered. We don't get + * any more details, but the user should be able + * to match the faulting PC to a previously set + * breakpoint. + */ + exception(EXC_BREAKPOINT, EXC_AARCH64_BREAKPT, 0); + + case ESR_EC_BREAKPT_SAME_EL: + panic("Same EL exception in EL0 handler\n"); + + case ESR_EC_SS_LOWER_EL: + /* + * TODO: need a userspace API to set/unset the MDSCR_EL1.SS bit, + * perhaps as a part of aarch64_debug_state. + * Unset it here before taking the exception. + * https://lore.kernel.org/all/CAFEAcA8QmsHfxAdUQET2Oab_xXa7x4i4C4+_6Y-J8ZNs1t5pPg@mail.gmail.com/ + */ + exception(EXC_BREAKPOINT, EXC_AARCH64_SS, + ESR_SS_ISV(esr) ? -1L : !!ESR_SS_EX(esr)); + + case ESR_EC_SS_SAME_EL: + panic("Same EL exception in EL0 handler\n"); + + case ESR_EC_WATCHPT_LOWER_EL: +#ifdef notyet + exception(EXC_BREAKPOINT, + (fault_prot(esr) == VM_PROT_READ) ? + EXC_AARCH64_WATCHPT_READ : EXC_AARCH64_WATCHPT_WRITE, + far); +#else + panic("Watchpoints not supported yet\n"); +#endif + + case ESR_EC_WATCHPT_SAME_EL: + panic("Same EL exception in EL0 handler\n"); + + case ESR_EC_BRK: + /* + * The "BRK" (software breakpoint) instruction, + * with a 16-bit immediate argument. + */ + exception(EXC_BREAKPOINT, EXC_AARCH64_BRK, ESR_BRK_IMM(esr)); + + default: + printf("Unhandled exception! esr = %lx\n", esr); + exception(EXC_BAD_INSTRUCTION, EXC_AARCH64_UNK, far); + } +} + +void user_trap_serror(void) +{ + panic("SError in EL0\n"); +} + +void kernel_trap_irq(void) +{ + spl7_irq(); + + assert(root_irq_src); + root_irq_src->handle_irq(root_irq_src); + + spl0_irq(); +} + +void kernel_trap_fiq(void) +{ + printf("Got FIQ while in EL1, ignoring for now\n"); +} + +/* + * Routine: look_up_recovery + * Purpose: + * Look up the recovery address for a kernel fault. + * Returns: + * NULL no recovery. + * addr recovery address. + */ +static void *look_up_recovery(void *pc) { + const struct recovery *rp; + vm_offset_t recover_base = (vm_offset_t) &recover_table; + + for (rp = recover_table; rp < recover_table_end; rp++) { + if ((vm_offset_t) pc == recover_base + rp->fault_addr_off) { + return (void *) (recover_base + rp->recover_addr_off); + } + } + + return NULL; +} + +/* + * Routine: kernel_trap_sync + * Purpose: + * Handle a synchronous exception from EL1. + * Conditions: + * Exceptions are masked. Nothing is locked. Caller-saved + * registers have been saved, and their values can be + * modified by this routine to return to a different state. + * Returns: + * TRUE if the exception has been handled, and execution + * should continue from where it got interrupted. + * FALSE if the exception is fatal, and execution should + * abort after dumping registers. + * Parameters: + * esr exception syndrome register + * far fault address register + * akes caller-saved registers at the time of the fault + */ +boolean_t kernel_trap_sync( + unsigned long esr, + vm_offset_t far, + struct aarch64_kernel_exception_state *akes) +{ + unsigned short ec = ESR_EC(esr); + kern_return_t kr; + vm_map_t map; + void *recovery = NULL; + +#if 0 + printf("Sync exc from EL1!\n"); + printf("ESR: %#lx, FAR: %#lx, PC: %#lx\n", esr, far, akes->pc); +#endif + +#if MACH_KDB + /* + * First thing, unmask debug exceptions, unless this is + * a debug exception that we're handling. + */ + switch (ec) { + case ESR_EC_BREAKPT_SAME_EL: + case ESR_EC_SS_SAME_EL: + case ESR_EC_WATCHPT_SAME_EL: + case ESR_EC_BRK: + break; + default: + asm volatile("msr DAIFClr, #8"); + break; + } +#endif + + switch (ec) { + case ESR_EC_DABT_SAME_EL: + /* Faulted on an address while in kernelspace. */ + + if ((far >= VM_MIN_KERNEL_ADDRESS && far < kernel_virtual_start) + || far >= kernel_virtual_end) { + printf("Kernel segfault in physical memory area!\n"); + return FALSE; + } + + if (far <= VM_MAX_USER_ADDRESS) { + /* + * Faulted on a user address. + * This could be a PAN failure, or the fault + * may be benign if there's a recovery handler + * at this address. We can detect would-be PAN + * failures even if hardware PAN is not present + * (SoftPAN). + */ + if (current_task() != kernel_task) + recovery = look_up_recovery(akes->pc); + if (unlikely(!recovery)) { + printf("SoftPAN failure\n"); + return FALSE; + } + map = current_map(); + } else { + map = kernel_map; + } + + kr = vm_fault(map, trunc_page(far), + fault_prot(esr), + FALSE, FALSE, NULL); + if (likely(kr == KERN_SUCCESS)) + return TRUE; + + /* + * If we have a recovery handler for this address, + * jump there. + */ + + if (likely(recovery)) { + /* + * Set things up for an appropriate exception() call, + * if that's what the handler wants to do. + */ + akes->pc = recovery; + akes->x[0] = (long) EXC_BAD_ACCESS; + akes->x[1] = (long) kr; + akes->x[2] = (long) far; + return TRUE; + } + + printf("Kernel segfault\n"); + return FALSE; + + default: + printf("Unexpected exception in EL1\n"); + return FALSE; + } +} + +void kernel_trap_fatal( + unsigned long esr, + vm_offset_t far, + struct aarch64_thread_state *ats) +{ + printf("=====================\n"); + printf("Dump of kernel state:\n"); + printf(" x0 %016lx x1 %016lx x2 %016lx x3 %016lx\n", + ats->x[0], ats->x[1], + ats->x[2], ats->x[3]); + printf(" x4 %016lx x5 %016lx x6 %016lx x7 %016lx\n", + ats->x[4], ats->x[5], + ats->x[6], ats->x[7]); + printf(" x8 %016lx x9 %016lx x10 %016lx x11 %016lx\n", + ats->x[8], ats->x[9], + ats->x[10], ats->x[11]); + printf("x12 %016lx x13 %016lx x14 %016lx x15 %016lx\n", + ats->x[12], ats->x[13], + ats->x[14], ats->x[15]); + printf("x16 %016lx x17 %016lx x18 %016lx x19 %016lx\n", + ats->x[16], ats->x[17], + ats->x[18], ats->x[19]); + printf("x20 %016lx x21 %016lx x22 %016lx x23 %016lx\n", + ats->x[20], ats->x[21], + ats->x[22], ats->x[23]); + printf("x24 %016lx x25 %016lx x26 %016lx x27 %016lx\n", + ats->x[24], ats->x[25], + ats->x[26], ats->x[27]); + printf("x28 %016lx x29 %016lx x30 %016lx sp %016lx\n", + ats->x[28], ats->x[29], + ats->x[30], ats->sp); + printf(" pc %016lx psr %016lx esr %016lx far %016lx\n", + ats->pc, ats->cpsr, + esr, far); + + panic("Fatal exception"); +} + +void kernel_trap_serror(void) +{ + panic("SError in EL1\n"); +} + +#if MACH_PCSAMPLE > 0 +/* + * Return saved state for interrupted user thread. + */ +unsigned interrupted_pc(const thread_t t) +{ + return USER_REGS(t)->pc; +} +#endif /* MACH_PCSAMPLE > 0 */ diff --git a/aarch64/aarch64/trap.h b/aarch64/aarch64/trap.h new file mode 100644 index 00000000..92d036d4 --- /dev/null +++ b/aarch64/aarch64/trap.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_TRAP_H_ +#define _AARCH64_TRAP_H_ + +#include <kern/kern_types.h> +#include "aarch64/thread.h" + +unsigned int interrupted_pc(thread_t); + +void __attribute__((noreturn)) user_trap_aarch32(void); +void __attribute__((noreturn)) user_trap_sync(void); +void __attribute__((noreturn)) user_trap_irq(void); +void __attribute__((noreturn)) user_trap_fiq(void); +void __attribute__((noreturn)) user_trap_serror(void); + +boolean_t kernel_trap_sync( + unsigned long esr, + vm_offset_t far, + struct aarch64_kernel_exception_state *akes); + +void kernel_trap_irq(void); +void kernel_trap_fiq(void); +void kernel_trap_serror(void); + +void __attribute__((noreturn)) kernel_trap_fatal( + unsigned long esr, + vm_offset_t far, + struct aarch64_thread_state *ats); + +#endif /* _AARCH64_TRAP_H_ */ diff --git a/aarch64/aarch64/undef.c b/aarch64/aarch64/undef.c new file mode 100644 index 00000000..834cb454 --- /dev/null +++ b/aarch64/aarch64/undef.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* FIXME */ + +void constab() {} +void delay() {} +void __disable_irq() {} +void __enable_irq() {} +void intnull() {} +void pmap_change_wiring() {} +void pmap_collect() {} +void pmap_map_bd() {} +void pmap_pageable() {} +void resettodr() {} +void unmask_irq() {} diff --git a/aarch64/aarch64/vm_param.h b/aarch64/aarch64/vm_param.h new file mode 100644 index 00000000..06e346b4 --- /dev/null +++ b/aarch64/aarch64/vm_param.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_VM_PARAM_ +#define _AARCH64_VM_PARAM_ + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1 << PAGE_SHIFT) +#define KERNEL_STACK_SIZE (4*PAGE_SIZE) + +#define VM_MIN_USER_ADDRESS VM_MIN_ADDRESS +#define VM_MAX_USER_ADDRESS VM_MAX_ADDRESS + +#define VM_MIN_KERNEL_ADDRESS (0xffff000000000000ULL) +#define VM_MAX_KERNEL_ADDRESS (0xFFFFFFFFFFFF0000ULL) + +#define VM_AARCH64_T0SZ 48 +#define VM_AARCH64_T1SZ 48 + +#define CPU_L1_SHIFT 6 + +#define VM_PAGE_DMA_LIMIT DECL_CONST(0x1000000, UL) +#define VM_PAGE_DMA32_LIMIT DECL_CONST(0x100000000, UL) +#define VM_PAGE_DIRECTMAP_LIMIT DECL_CONST(0x400000000000, UL) +#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, ULL) + +#define VM_PAGE_SEG_DMA 0 +#define VM_PAGE_SEG_DMA32 (VM_PAGE_SEG_DMA+1) +#define VM_PAGE_SEG_DIRECTMAP (VM_PAGE_SEG_DMA32+1) +#define VM_PAGE_SEG_HIGHMEM (VM_PAGE_SEG_DIRECTMAP+1) + +#define VM_PAGE_MAX_SEGS 4 + +#define phystokv(a) ((vm_offset_t)(a) + VM_MIN_KERNEL_ADDRESS) +#define kvtophys(a) ((vm_offset_t)(a) - VM_MIN_KERNEL_ADDRESS) + +#define VM_KERNEL_MAP_SIZE (512 * 1024 * 1024) + +#include <mach/vm_param.h> + +#endif /* _AARCH64_VM_PARAM_ */ diff --git a/aarch64/aarch64/xpr.h b/aarch64/aarch64/xpr.h new file mode 100644 index 00000000..1e01a7a1 --- /dev/null +++ b/aarch64/aarch64/xpr.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _AARCH64_XPR_ +#define _AARCH64_XPR_ + +// TODO needed? +#define XPR_TIMESTAMP 0 + +#endif /* _AARCH64_XPR_ */ diff --git a/aarch64/arm/gic-v2.c b/aarch64/arm/gic-v2.c new file mode 100644 index 00000000..545fcbe6 --- /dev/null +++ b/aarch64/arm/gic-v2.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "arm/gic-v2.h" +#include "aarch64/irq.h" +#include "aarch64/vm_param.h" +#include <kern/kalloc.h> +#include <kern/printf.h> +#include <string.h> +#include <stddef.h> + +#define GICD_CTLR 0x000 +#define GICD_ISENABLER 0x100 +#define GICD_ICENABLER 0x180 +#define GICD_ISPENDR 0x200 +#define GICD_ICPENDR 0x280 + +#define GICD_CTLR_DISABLE 0x0 +#define GICD_CTLR_ENABLE 0x1 + +#define GICC_CTLR 0x000 +#define GICC_PMR 0x004 +#define GICC_BPR 0x008 +#define GICC_IAR 0x00c +#define GICC_EOIR 0x010 + +#define GICC_CTLR_DISABLE 0x0 +#define GICC_CTLR_ENABLE 0x1 + +#define GICC_IAR_IRQ_MASK 0x7ff + +#define GIC_IRQ_SPURIOUS 1023 + +struct gic_v2 { + struct irq_src src; + struct irq_ctlr ctlr; + + void *distributor_base; + vm_size_t distributor_size; + void *cpu_base; + vm_size_t cpu_size; + + vm_size_t nsrcs; + struct irq_src **srcs; +}; + +#define GICD_REG(gic, off) *(volatile uint32_t *) ((gic)->distributor_base + (off)) +#define GICC_REG(gic, off) *(volatile uint32_t *) ((gic)->cpu_base + (off)) + +static void gic_v2_enable_irq(struct gic_v2 *gic, int irq) +{ + GICD_REG(gic, GICD_ISENABLER + (irq / 32)) = 1 << (irq % 32); +} + +static void gic_v2_disable_irq(struct gic_v2 *gic, int irq) +{ + GICD_REG(gic, GICD_ICENABLER + (irq / 32)) = 1 << (irq % 32); +} + +static void gic_v2_handle_irq(struct irq_src *data) +{ + struct gic_v2 *gic = structof(data, struct gic_v2, src); + struct irq_src *src; + uint32_t iar; + uint32_t irq; + + iar = GICC_REG(gic, GICC_IAR); + irq = iar & GICC_IAR_IRQ_MASK; + + if (unlikely(irq == GIC_IRQ_SPURIOUS)) + return; + + if (irq < 16 || irq - 16 >= gic->nsrcs) + goto BadIrq; + src = gic->srcs[irq - 16]; + if (src == NULL) + goto BadIrq; + + src->handle_irq(src); + + GICC_REG(gic, GICC_EOIR) = iar; + + /* TODO: loop perhaps? */ + + return; + +BadIrq: + printf("GIC v2: unexpected IRQ %u\n", irq); + gic_v2_disable_irq(gic, irq); +} + +static void gic_v2_add_src( + struct irq_ctlr *ctlr, + struct irq_src *src, + const struct irq_desc *desc) +{ + struct gic_v2 *gic = structof(ctlr, struct gic_v2, ctlr); + dtb_prop_t prop; + vm_size_t off; + + assert(desc->type == IRQ_DESC_TYPE_DT); + prop = ((const struct irq_desc_dt *) desc)->prop; + assert(prop->length % 12 == 0); + + for (off = 0; off < prop->length;) { + uint32_t cell, flags; + boolean_t is_ppi; + int irq; + + cell = dtb_prop_read_cells(prop, 1, &off); + is_ppi = (cell == 1); + cell = dtb_prop_read_cells(prop, 1, &off); + irq = cell + (is_ppi ? 16 : 32); + cell = dtb_prop_read_cells(prop, 1, &off); + flags = cell; + + if (irq - 16 >= gic->nsrcs) { + struct irq_src **srcs; + + srcs = (struct irq_src **) kalloc(sizeof(struct irq_src *) * (irq - 16 + 1)); + if (gic->nsrcs > 0) + memcpy(srcs, gic->srcs, sizeof(struct irq_src *) * gic->nsrcs); + memset(srcs + gic->nsrcs, sizeof(struct irq_src *) * (irq - 16 + 1 - gic->nsrcs), 0); + if (gic->srcs != NULL) + kfree((vm_offset_t) gic->srcs, sizeof(struct irq_src *) * gic->nsrcs); + gic->srcs = srcs; + gic->nsrcs = irq - 16 + 1; + } + assert(gic->srcs[irq - 16] == NULL); + gic->srcs[irq - 16] = src; + + gic_v2_enable_irq(gic, irq); + /* TODO: flags */ + } +} + +struct irq_ctlr *gic_v2_init(dtb_node_t node, dtb_ranges_map_t map) +{ + struct gic_v2 *gic; + struct dtb_prop prop; + uint64_t tmp; + vm_offset_t off = 0; + + gic = (struct gic_v2 *) kalloc(sizeof(struct gic_v2)); + gic->nsrcs = 0; + gic->srcs = NULL; + gic->src.handle_irq = gic_v2_handle_irq; + gic->ctlr.add_src = gic_v2_add_src; + + prop = dtb_node_find_prop(node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + + tmp = dtb_prop_read_cells(&prop, node->address_cells, &off); + tmp = dtb_map_address(map, tmp); + gic->distributor_base = (void *) phystokv(tmp); + + gic->distributor_size = dtb_prop_read_cells(&prop, node->size_cells, &off); + + tmp = dtb_prop_read_cells(&prop, node->address_cells, &off); + tmp = dtb_map_address(map, tmp); + gic->cpu_base = (void *) phystokv(tmp); + + gic->cpu_size = dtb_prop_read_cells(&prop, node->size_cells, &off); + + /* TODO: support GICv2 not being the root controller */ + root_irq_src = &gic->src; + + return &gic->ctlr; +} + +void gic_v2_enable(struct irq_ctlr *ctlr) +{ + struct gic_v2 *gic = structof(ctlr, struct gic_v2, ctlr); + + GICD_REG(gic, GICD_CTLR) = GICD_CTLR_ENABLE; + GICC_REG(gic, GICC_CTLR) = GICC_CTLR_ENABLE; + GICC_REG(gic, GICC_PMR) = 0xff; + GICC_REG(gic, GICC_BPR) = 0; +} diff --git a/aarch64/arm/gic-v2.h b/aarch64/arm/gic-v2.h new file mode 100644 index 00000000..41b0dee1 --- /dev/null +++ b/aarch64/arm/gic-v2.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <mach/boolean.h> +#include <device/dtb.h> + +#define gic_v2_is_compatible(node) \ + (dtb_node_is_compatible(node, "arm,arm1176jzf-devchip-gic") \ + || dtb_node_is_compatible(node, "arm,arm11mp-gic") \ + || dtb_node_is_compatible(node, "arm,cortex-a15-gic") \ + || dtb_node_is_compatible(node, "arm,cortex-a7-gic") \ + || dtb_node_is_compatible(node, "arm,cortex-a9-gic") \ + || dtb_node_is_compatible(node, "arm,eb11mp-gic") \ + || dtb_node_is_compatible(node, "arm,gic-400") \ + || dtb_node_is_compatible(node, "arm,pl390") \ + || dtb_node_is_compatible(node, "arm,tc11mp-gic") \ + || dtb_node_is_compatible(node, "brcm,brahma-b15-gic") \ + || dtb_node_is_compatible(node, "nvidia,tegra210-agic") \ + || dtb_node_is_compatible(node, "qcom,msm-8660-qgic") \ + || dtb_node_is_compatible(node, "qcom,msm-qgic2")) + +extern struct irq_ctlr *gic_v2_init(dtb_node_t node, dtb_ranges_map_t map); +extern void gic_v2_enable(struct irq_ctlr *); diff --git a/aarch64/arm/pl011.c b/aarch64/arm/pl011.c new file mode 100644 index 00000000..d21e2696 --- /dev/null +++ b/aarch64/arm/pl011.c @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "arm/pl011.h" +#include <device/dtb.h> +#include <device/cons.h> +#include <device/io_req.h> +#include <device/ds_routines.h> /* device_io_map */ +#include <vm/vm_user.h> +#include "aarch64/smp.h" /* cpu_pause() */ +#include <string.h> + +/* + * Mach driver for PrimeCell UART (PL011). + * + * https://developer.arm.com/documentation/ddi0183/g/programmers-model/summary-of-registers + * + * The following are offsets of the memory-mapped registers + * relative to a base address that is discoverable via the + * device tree. + */ + +#define UARTDR 0x000 /* data */ +#define UARTRSR 0x004 /* receive status / error clear */ + /* 0x008 to 0x014 are reserved */ +#define UARTFR 0x018 /* flags */ + /* 0x01c is reserved */ +#define UARTILPR 0x020 /* IrDA low-power counter */ +#define UARTIBRD 0x024 /* integer baud rate */ +#define UARTFBRD 0x028 /* fractional baud rate */ +#define UARTLCR_H 0x02C /* line control */ +#define UARTCR 0x030 /* control */ +#define UARTIFLS 0x034 /* interrupt FIFO level select */ +#define UARTIMSC 0x038 /* interrupt mask set/clear */ +#define UARTRIS 0x03C /* raw interrupt status */ +#define UARTMIS 0x040 /* masked interrupt status */ +#define UARTICR 0x044 /* interrrupt clear */ +#define UARTDMACR 0x048 /* DMA control */ + +#define UARTDR_DATA_MASK 0x000f +#define UARTDR_FE 0x0010 /* framing error */ +#define UARTDR_PE 0x0020 /* parity error */ +#define UARTDR_BE 0x0040 /* break error */ +#define UARTDR_OE 0x0080 /* overrun error */ + +#define UARTFR_CTS 0x0001 /* clear to send */ +#define UARTFR_DSR 0x0002 /* data set ready */ +#define UARTFR_DCD 0x0004 /* data carrier detect */ +#define UARTFR_BUSY 0x0008 /* busy */ +#define UARTFR_RXFE 0x0010 /* receive FIFO empty */ +#define UARTFR_TXFF 0x0020 /* transmit FIFO full */ +#define UARTFR_RXFF 0x0040 /* receive FIFO full */ +#define UARTFR_TXFE 0x0080 /* transmit FIFO empty */ +#define UARTFR_RI 0x0100 /* ring indicator */ + +#define UARTLCR_H_BRK 0x0001 /* send break */ +#define UARTLCR_H_PEN 0x0002 /* parity enable */ +#define UARTLCR_H_EPS 0x0004 /* even parity select */ +#define UARTLCR_H_STP2 0x0008 /* two stop bits select */ +#define UARTLCR_H_FEN 0x0010 /* enable FIFOs */ +#define UARTLCR_H_WLEN_5 0x0000 /* 5-bit words */ +#define UARTLCR_H_WLEN_6 0x0020 /* 6-bit words */ +#define UARTLCR_H_WLEN_7 0x0040 /* 7-bit words */ +#define UARTLCR_H_WLEN_8 0x0060 /* 8-bit words */ +#define UARTLCR_H_SPS 0x0080 /* stick parity select */ + +#define UARTCR_UARTEN 0x0001 /* UART enable */ +#define UARTCR_SIREN 0x0002 /* SIR enable */ +#define UARTCR_SIRLP 0x0004 /* SIR low-power mode */ + /* 6-3 reserved */ +#define UARTCR_LBE 0x0080 /* loopback enable */ +#define UARTCR_TXE 0x0100 /* transmit enable */ +#define UARTCR_RXE 0x0200 /* receive enable */ +#define UARTCR_DTR 0x0400 /* data transmit ready */ +#define UARTCR_RTS 0x0800 /* request to send */ +#define UARTCR_OUT1 0x1000 /* Out1 */ +#define UARTCR_OUT2 0x2000 /* Out2 */ +#define UARTCR_RTSEN 0x4000 /* RTS hardware flow control enable */ +#define UARTCR_CTSEN 0x8000 /* CTS hardware flow control enable */ + +struct pl011 { + struct mach_device dev; + void *base; +}; + +#define UART_REG(uart, off, tp) *(volatile tp *) ((uart)->base + (off)) + +static vm_offset_t pl011_rom_base; +static void pl011_romputc(char c) +{ + while (unlikely((*(volatile uint32_t *) (pl011_rom_base + UARTFR)) & UARTFR_TXFF)) + cpu_pause(); + *(volatile uint32_t *) (pl011_rom_base + UARTDR) = c; +} + +void pl011_early_init(dtb_node_t node, dtb_ranges_map_t map) +{ + struct dtb_prop prop; + vm_size_t off; + uint64_t addr = 0x1; + + if (romputc != NULL) + return; + + dtb_for_each_prop (*node, prop) { + if (!strcmp(prop.name, "status")) { + if (strncmp(prop.data, "ok", 2)) + return; + } else if (!strcmp(prop.name, "reg")) { + off = 0; + addr = dtb_prop_read_cells(&prop, node->address_cells, &off); + addr = dtb_map_address(map, addr); + } + } + + assert(addr != 0x1); + pl011_rom_base = phystokv(addr); /* FIXME can't call phystokv at this point */ + romputc = (void (*)(char)) phystokv(pl011_romputc); +} + +/* FIXME more than one */ +static struct pl011 the_uart; + +void pl011_init(dtb_node_t node, dtb_ranges_map_t map) +{ + struct dtb_prop prop; + uint64_t addr; + vm_size_t off = 0; + + assert(dtb_node_is_compatible(node, "arm,pl011")); + + prop = dtb_node_find_prop(node, "reg"); + assert(!DTB_IS_SENTINEL(prop)); + + addr = dtb_prop_read_cells(&prop, node->address_cells, &off); + addr = dtb_map_address(map, addr); + the_uart.base = (void *) phystokv(addr); +} + +static boolean_t pl011_txff(struct pl011 *uart) +{ + return !!(UART_REG(uart, UARTFR, uint32_t) & UARTFR_TXFF); +} + +#if 0 +void pl011_putc(char c) +{ + while (pl011_txff(&the_uart)) + cpu_pause(); + UART_REG(&the_uart, UARTDR, char) = c; +} +#endif + +static io_return_t pl011_write(dev_t dev, io_req_t ior) +{ + io_return_t kr; + vm_offset_t map_addr; + const char *data; + struct pl011 *uart = structof(ior->io_device, struct pl011, dev); + + if (ior->io_total == 0) + return D_SUCCESS; + + if (!(UART_REG(uart, UARTFR, uint32_t) & UARTFR_DCD)) + return D_IO_ERROR; + + if (ior->io_op & IO_INBAND) { + data = ior->io_data; + } else { + kr = vm_map_copyout(device_io_map, &map_addr, (vm_map_copy_t) ior->io_data); + if (kr != KERN_SUCCESS) + return kr; + data = (char *) map_addr; + } + + /* TODO: don't block */ + ior->io_residual = ior->io_total; + while (ior->io_residual > 0) { + while (pl011_txff(uart)) + cpu_pause(); + UART_REG(uart, UARTDR, char) = *data; + data++; + ior->io_residual--; + } + + if (!(ior->io_op & IO_INBAND)) { + kr = vm_deallocate(device_io_map, map_addr, ior->io_count); + assert(kr == KERN_SUCCESS); + } + + return D_SUCCESS; +} diff --git a/aarch64/arm/pl011.h b/aarch64/arm/pl011.h new file mode 100644 index 00000000..27f5ab46 --- /dev/null +++ b/aarch64/arm/pl011.h @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <device/dtb.h> + +void pl011_early_init(dtb_node_t node, dtb_ranges_map_t map); +void pl011_init(dtb_node_t node, dtb_ranges_map_t map); + +void pl011_putc(char c); diff --git a/aarch64/arm/psci.c b/aarch64/arm/psci.c new file mode 100644 index 00000000..9e20b82f --- /dev/null +++ b/aarch64/arm/psci.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "arm/psci.h" +#include <device/dtb.h> +#include <kern/debug.h> +#include <string.h> + +#define PSCI_VERSION 0x84000000 +#define CPU_SUSPEND 0xc4000001 +#define CPU_OFF 0x84000002 +#define CPU_ON 0xc4000003 +/* ...MIGRATE stuff... */ +#define SYSTEM_OFF 0x84000008 +#define SYSTEM_RESET 0x84000009 +#define SYSTEM_RESET2 0c40000012 +#define PSCI_FEATURES 0x8400000a +#define CPU_FREEZE 0x8400000b +#define CPU_DEFAULT_SUSPEND 0xc400000c +#define SYSTEM_SUSPEND 0xc400000e + +static uint32_t cpu_off_id = CPU_OFF; + +static enum { + PSCI_METHOD_NONE, + PSCI_METHOD_HVC, + PSCI_METHOD_SMC +} psci_method = PSCI_METHOD_NONE; + +static int smc32( + uint32_t function, + uint32_t arg1, + uint32_t arg2, + uint32_t arg3) +{ + register uint32_t w0 asm("w0") = function; + register uint32_t w1 asm("w1") = arg1; + register uint32_t w2 asm("w2") = arg2; + register uint32_t w3 asm("w3") = arg3; + + asm volatile("smc #0" + : "+r"(w0), "+r"(w1), "+r"(w2), "+r"(w3)); + + return w0; +} + +static int hvc32( + uint32_t function, + uint32_t arg1, + uint32_t arg2, + uint32_t arg3) +{ + register uint32_t w0 asm("w0") = function; + register uint32_t w1 asm("w1") = arg1; + register uint32_t w2 asm("w2") = arg2; + register uint32_t w3 asm("w3") = arg3; + + asm volatile("hvc #0" + : "+r"(w0), "+r"(w1), "+r"(w2), "+r"(w3)); + + return w0; +} + +kern_return_t psci_cpu_off(void) +{ + switch (psci_method) { + case PSCI_METHOD_NONE: + /* Not a chance. */ + return KERN_FAILURE; + case PSCI_METHOD_SMC: + smc32(cpu_off_id, 0, 0, 0); + return KERN_FAILURE; + case PSCI_METHOD_HVC: + hvc32(cpu_off_id, 0, 0, 0); + return KERN_FAILURE; + default: + panic("Bad PSCI method\n"); + } +} + +kern_return_t psci_system_off(void) +{ + switch (psci_method) { + case PSCI_METHOD_NONE: + /* Not a chance. */ + return KERN_FAILURE; + case PSCI_METHOD_SMC: + smc32(SYSTEM_OFF, 0, 0, 0); + return KERN_FAILURE; + case PSCI_METHOD_HVC: + hvc32(SYSTEM_OFF, 0, 0, 0); + return KERN_FAILURE; + default: + panic("Bad PSCI method\n"); + } +} + +kern_return_t psci_system_reset(void) +{ + switch (psci_method) { + case PSCI_METHOD_NONE: + /* Not a chance. */ + return KERN_FAILURE; + case PSCI_METHOD_SMC: + smc32(SYSTEM_RESET, 0, 0, 0); + return KERN_FAILURE; + case PSCI_METHOD_HVC: + hvc32(SYSTEM_RESET, 0, 0, 0); + return KERN_FAILURE; + default: + panic("Bad PSCI method\n"); + } +} + +void psci_init(dtb_node_t node) +{ + struct dtb_prop prop; + vm_size_t off; + + dtb_for_each_prop (*node, prop) { + if (!strcmp(prop.name, "method")) { + if (!strcmp(prop.data, "smc")) { + psci_method = PSCI_METHOD_SMC; + } else if (!strcmp(prop.data, "hvc")) { + psci_method = PSCI_METHOD_HVC; + } else { + panic("Unexpected PSCI method %s\n", (const char *) prop.data); + } + } else if (!strcmp(prop.name, "cpu_off")) { + off = 0; + cpu_off_id = dtb_prop_read_cells(&prop, 1, &off); + } + /* could add cpu_on etc here */ + } + + assert(psci_method != PSCI_METHOD_NONE); +} diff --git a/aarch64/arm/psci.h b/aarch64/arm/psci.h new file mode 100644 index 00000000..19b171ec --- /dev/null +++ b/aarch64/arm/psci.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <device/dtb.h> + +#define psci_is_compatible(node) \ + (dtb_node_is_compatible(node, "arm,psci") \ + || dtb_node_is_compatible(node, "arm,psci-0.2") \ + || dtb_node_is_compatible(node, "arm,psci-1.0")) + +extern void psci_init(dtb_node_t node); +extern kern_return_t psci_cpu_off(void); +extern kern_return_t psci_system_off(void); +extern kern_return_t psci_system_reset(void); diff --git a/aarch64/arm/timer.c b/aarch64/arm/timer.c new file mode 100644 index 00000000..ac0114f0 --- /dev/null +++ b/aarch64/arm/timer.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "arm/timer.h" +#include "aarch64/irq.h" +#include "aarch64/percpu.h" +#include "aarch64/mach_param.h" /* HZ */ +#include <kern/thread.h> +#include <kern/assert.h> +#include <kern/mach_clock.h> +#include <string.h> + +static unsigned cnt_freq; /* frequency, timer ticks per second */ +static volatile unsigned long long cnt_pct; + +static void set_up_next_interrupt(void) +{ + asm volatile( + "msr CNTP_CVAL_EL0, %0" + :: "r"(cnt_pct + cnt_freq / HZ)); +} + +void startrtclock(void) +{ + asm( + "mrs %0, CNTFRQ_EL0\n\t" + "mrs %1, CNTPCT_EL0" + : "=r"(cnt_freq), "=r"(cnt_pct)); + assert(cnt_freq > 10); + + asm volatile( + "msr CNTP_CTL_EL0, %0" + :: "r"(1) + ); + + set_up_next_interrupt(); +} + +static void cnt_handle_irq(struct irq_src *) +{ + unsigned long last_pct = cnt_pct; + unsigned int usec; + boolean_t from_el0; + + asm volatile( + "isb\n\t" + "mrs %0, CNTPCT_EL0" + : "=r"(cnt_pct)); + usec = (cnt_pct - last_pct) * 1000000 / cnt_freq; + + from_el0 = percpu_get(boolean_t, in_irq_from_el0); + if (from_el0) + clock_interrupt(usec, TRUE, TRUE, current_thread()->pcb->ats.pc); + else + clock_interrupt(usec, FALSE, TRUE, 0); + + set_up_next_interrupt(); +} + +void cnt_set_interrupt_parent(dtb_node_t node, struct irq_ctlr *ctlr) +{ + struct dtb_prop prop; + struct irq_desc_dt desc; + static struct irq_src src; + + prop = dtb_node_find_prop(node, "interrupts"); + assert(!DTB_IS_SENTINEL(prop)); + + desc.type = IRQ_DESC_TYPE_DT; + desc.prop = ∝ + + src.handle_irq = cnt_handle_irq; + ctlr->add_src(ctlr, &src, (struct irq_desc *) &desc); +} + +void cnt_init(dtb_node_t node) +{ + struct dtb_prop prop; + + asm("mrs %0, CNTFRQ_EL0" : "=r"(cnt_freq)); + + dtb_for_each_prop (*node, prop) { + if (!strcmp(prop.name, "clock-frequency")) { + vm_offset_t off = 0; + cnt_freq = dtb_prop_read_cells(&prop, 1, &off); + } + } +} diff --git a/aarch64/arm/timer.h b/aarch64/arm/timer.h new file mode 100644 index 00000000..afdfde02 --- /dev/null +++ b/aarch64/arm/timer.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <device/dtb.h> + +struct irq_ctlr; + +extern void cnt_init(dtb_node_t node); + +extern void cnt_set_interrupt_parent( + dtb_node_t node, + struct irq_ctlr *ctlr); + +extern void startrtclock(void); diff --git a/aarch64/configfrag.ac b/aarch64/configfrag.ac index 03f980bf..2a927f42 100644 --- a/aarch64/configfrag.ac +++ b/aarch64/configfrag.ac @@ -13,10 +13,14 @@ dnl "AS IS" CONDITION. THE FREE SOFTWARE FOUNDATION DISCLAIMS ANY dnl LIABILITY OF ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE dnl USE OF THIS SOFTWARE. +# Nothing here at the moment. + [case $host_cpu in aarch64)] AM_CONDITIONAL([HOST_aarch64], [true]) + # Determines the size of the CPU cache line. + AC_DEFINE([CPU_L1_SHIFT], [6], [CPU_L1_SHIFT]) [ # Does the architecture provide machine-specific interfaces? mach_machine_routines=1 diff --git a/aarch64/include/mach/aarch64/exception.h b/aarch64/include/mach/aarch64/exception.h index 2e96e09a..b064c18c 100644 --- a/aarch64/include/mach/aarch64/exception.h +++ b/aarch64/include/mach/aarch64/exception.h @@ -47,11 +47,10 @@ #define EXC_AARCH64_SVC 1 /* SVC that's not a valid syscall, subcode contains immediate */ -/* -Not yet: -#define EXC_AARCH64_HVC 2 HVC, subcode contains immediate -#define EXC_AARCH64_SMC 3 SMC, subcode contains immediate -*/ +#ifdef notyet +#define EXC_AARCH64_HVC 2 /* HVC, subcode contains immediate */ +#define EXC_AARCH64_SMC 3 /* SMC, subcode contains immediate */ +#endif /* * EXC_BAD_ACCESS @@ -68,10 +67,9 @@ Not yet: #define EXC_AARCH64_PAC 103 /* PAC failure, subcode describes the key */ #define EXC_AARCH64_BTI 104 /* BTI failure, subcode contains BTYPE */ -/* -Not yet: -#define EXC_AARCH64_MTE 105 MTE failure -*/ +#ifdef notyet +#define EXC_AARCH64_MTE 105 /* MTE failure */ +#endif /* * EXC_BREAKPOINT @@ -81,10 +79,9 @@ Not yet: #define EXC_AARCH64_SS 2 /* software single step, subcode contains EX flag, or -1 if unknown */ #define EXC_AARCH64_BREAKPT 3 /* hardware breakpoint */ -/* -Not yet: -#define EXC_AARCH64_WATCHPT_READ 4 hardware watchpoint (read), subcode contains accessed address -#define EXC_AARCH64_WATCHPT_WRITE 5 hardware watchpoint (write), subcode contains accessed address -*/ +#ifdef notyet +#define EXC_AARCH64_WATCHPT_READ 4 /* hardware watchpoint (read), subcode contains accessed address */ +#define EXC_AARCH64_WATCHPT_WRITE 5 /* hardware watchpoint (write), subcode contains accessed address */ +#endif #endif /* _MACH_AARCH64_EXCEPTION_H_ */ diff --git a/aarch64/include/mach/aarch64/exec/elf.h b/aarch64/include/mach/aarch64/exec/elf.h new file mode 100644 index 00000000..9904876a --- /dev/null +++ b/aarch64/include/mach/aarch64/exec/elf.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023-2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +/* TODO: This should not be a public header. */ + +#ifndef _MACH_AARCH64_EXEC_ELF_H_ +#define _MACH_AARCH64_EXEC_ELF_H_ + +typedef unsigned int Elf32_Addr; +typedef unsigned short Elf32_Half; +typedef unsigned int Elf32_Off; +typedef signed int Elf32_Sword; +typedef unsigned int Elf32_Word; + +typedef uint64_t Elf64_Addr; +typedef uint64_t Elf64_Off; +typedef int32_t Elf64_Shalf; +typedef int32_t Elf64_Sword; +typedef uint32_t Elf64_Word; +typedef int64_t Elf64_Sxword; +typedef uint64_t Elf64_Xword; +typedef uint16_t Elf64_Half; + + +#define MY_ELF_CLASS ELFCLASS64 +#define MY_EI_DATA ELFDATA2LSB +#define MY_E_MACHINE EM_AARCH64 + +#endif /* _MACH_AARCH64_EXEC_ELF_H_ */ diff --git a/aarch64/include/mach/aarch64/mach_aarch64.defs b/aarch64/include/mach/aarch64/mach_aarch64.defs index da1e9498..0fe1eb62 100644 --- a/aarch64/include/mach/aarch64/mach_aarch64.defs +++ b/aarch64/include/mach/aarch64/mach_aarch64.defs @@ -43,7 +43,7 @@ import <mach/machine/mach_aarch64_types.h>; * versions can add more items and more bits (HWCAP3_* and * so forth). */ -type hwcaps_t = array[*:16] of uint64_t; +type hwcaps_t = array[*:16] of uint32_t; routine aarch64_get_hwcaps( host : host_t; diff --git a/aarch64/include/mach/aarch64/mach_aarch64_types.h b/aarch64/include/mach/aarch64/mach_aarch64_types.h index efb9f897..98fd6c4b 100644 --- a/aarch64/include/mach/aarch64/mach_aarch64_types.h +++ b/aarch64/include/mach/aarch64/mach_aarch64_types.h @@ -25,7 +25,7 @@ #ifndef __ASSEMBLER__ #include <stdint.h> -typedef uint64_t *hwcaps_t; +typedef uint32_t *hwcaps_t; #endif /* These definitions are meant to match those in diff --git a/aarch64/include/mach/aarch64/machine_types.defs b/aarch64/include/mach/aarch64/machine_types.defs index d2f5e31e..eab022fb 100644 --- a/aarch64/include/mach/aarch64/machine_types.defs +++ b/aarch64/include/mach/aarch64/machine_types.defs @@ -16,31 +16,40 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -/* +/* * Mach Operating System * Copyright (c) 1992 Carnegie Mellon University * All Rights Reserved. - * + * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. - * + * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. - * + * * Carnegie Mellon requests users of this software to return to - * + * * Software Distribution Coordinator or [email protected] * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 - * + * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ +/* + * File: mach/machine/machine_types.defs + * Author: Alessandro Forin + * Date: 7/92 + * + * Header file for the basic, machine-dependent data types. + * Version for 32 bit architectures. + * + */ #ifndef _MACHINE_MACHINE_TYPES_DEFS_ #define _MACHINE_MACHINE_TYPES_DEFS_ 1 diff --git a/aarch64/include/mach/aarch64/thread_status.h b/aarch64/include/mach/aarch64/thread_status.h index c0c7773e..a095af2b 100644 --- a/aarch64/include/mach/aarch64/thread_status.h +++ b/aarch64/include/mach/aarch64/thread_status.h @@ -40,4 +40,6 @@ struct aarch64_float_state { }; #define AARCH64_FLOAT_STATE_COUNT (sizeof(struct aarch64_float_state) / sizeof(unsigned int)) +#define AARCH64_REGS_SEGS_STATE 1 + #endif /* _MACH_AARHC64_THREAD_STATUS_H_ */ diff --git a/aarch64/include/mach/aarch64/vm_param.h b/aarch64/include/mach/aarch64/vm_param.h index d7b1e281..04fbafb3 100644 --- a/aarch64/include/mach/aarch64/vm_param.h +++ b/aarch64/include/mach/aarch64/vm_param.h @@ -21,6 +21,10 @@ #include <mach/machine/vm_types.h> +#ifdef KERNEL +#include "aarch64/vm_param.h" +#endif + #define BYTE_SIZE 8 /* byte size in bits */ /* diff --git a/aarch64/ldscript b/aarch64/ldscript new file mode 100644 index 00000000..236fc6f8 --- /dev/null +++ b/aarch64/ldscript @@ -0,0 +1,32 @@ +ENTRY(_start) + +SECTIONS +{ + . = 0x0; + .text : + { + __text_start = .; + *(.text.boot) *(.text); + } + .rodata : + { + *(.rodata); + } + .rela : + { + __rela_start = .; + *(.rela.*); + __rela_end = .; + } + .data : + { + *(.data); + } + .bss : + { + __bss_start = .; + *(.bss); + __bss_end = .; + } + _image_end = .; +} diff --git a/device/dtb.c b/device/dtb.c new file mode 100644 index 00000000..529dae5e --- /dev/null +++ b/device/dtb.c @@ -0,0 +1,429 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <device/dtb.h> +#include <kern/assert.h> +#include <kern/debug.h> +#include <string.h> + +#define DTB_MAGIC 0xd00dfeed + +#define DTB_BEGIN_NODE 0x1 +#define DTB_END_NODE 0x2 +#define DTB_PROP 0x3 +#define DTB_NOP 0x4 +#define DTB_END 0x9 + +/* + * "If missing, a client program should assume a default + * value of 2 for #address-cells, and a value of 1 for + * #size-cells." + */ +#define DEFAULT_ADDRESS_CELLS 2 +#define DEFAULT_SIZE_CELLS 1 + +static dtb_t global_dtb; +#define DTB_PTR(offset) ((const char *) global_dtb + (offset)) + +static uint32_t be32toh(dtb_uint32_t arg) +{ + /* Assumes little-endian. */ + uint32_t raw; + __builtin_memcpy(&raw, &arg, 4); + return __builtin_bswap32(raw); +} + +static const char *dt_strings_str(vm_offset_t offset) +{ + offset += be32toh(global_dtb->offset_dt_strings); + return DTB_PTR(offset); +} + +static uint32_t uint32_at_offset(vm_offset_t offset) +{ + return be32toh(*(const dtb_uint32_t *) DTB_PTR(offset)); +} + +static void __attribute__((noreturn)) panic_unexpected_token(vm_offset_t offset) +{ + assert((offset & 3) == 0); + panic("Unexpected DTB token at %#x: %#x\n", + (unsigned) offset, uint32_at_offset(offset)); +} + +static void skip_nops(vm_offset_t *offset) +{ + while (uint32_at_offset(*offset) == DTB_NOP) + (*offset) += 4; +} + +static void skip_padding(vm_offset_t *offset) +{ + *offset = (*offset + 3) / 4 * 4; +} + +static boolean_t skip_prop(vm_offset_t *offset) +{ + assert(((*offset) & 0x3) == 0); + assert(uint32_at_offset(*offset) != DTB_NOP); + + if (uint32_at_offset(*offset) != DTB_PROP) + return FALSE; + *offset += 12 + uint32_at_offset(*offset + 4); + return TRUE; +} + +static void skip_node_header(vm_offset_t *offset) +{ + assert(((*offset) & 0x3) == 0); + assert(uint32_at_offset(*offset) == DTB_BEGIN_NODE); + + *offset += 4; + while (*DTB_PTR((*offset)++)); /* skip node name */ +} + +static struct dtb_node make_node_at_offset(vm_offset_t offset) +{ + struct dtb_node node; + + skip_padding(&offset); + skip_nops(&offset); + + switch (uint32_at_offset(offset)) { + case DTB_BEGIN_NODE: + node.offset = offset; + node.name = DTB_PTR(node.offset) + 4; + node.address_cells = node.size_cells = 0; + break; + + case DTB_END_NODE: + /* + * No more (sub)nodes. + */ + node.offset = DTB_SENTINEL_OFFSET; + node.name = NULL; + node.address_cells = node.size_cells = 0; + break; + + default: + panic_unexpected_token(offset); + } + + return node; +} + +static struct dtb_prop make_prop_at_offset(vm_offset_t offset) +{ + struct dtb_prop prop; + + skip_padding(&offset); + skip_nops(&offset); + + switch (uint32_at_offset(offset)) { + case DTB_BEGIN_NODE: + case DTB_END_NODE: + /* + * No more properties. + */ + prop.offset = DTB_SENTINEL_OFFSET; + prop.name = NULL; + prop.data = NULL; + prop.length = 0; + break; + + case DTB_PROP: + prop.offset = offset; + prop.length = uint32_at_offset(offset + 4); + prop.name = dt_strings_str(uint32_at_offset(offset + 8)); + prop.data = DTB_PTR(offset + 12); + break; + + default: + panic_unexpected_token(offset); + } + + return prop; +} + +kern_return_t dtb_load(dtb_t dtb) +{ + global_dtb = dtb; + + if (be32toh(dtb->magic) != DTB_MAGIC) + return KERN_INVALID_VALUE; + + return KERN_SUCCESS; +} + +void dtb_get_location(dtb_t *out_dtb, vm_size_t *out_dtb_size) +{ + *out_dtb = global_dtb; + *out_dtb_size = be32toh(global_dtb->total_size); +} + +struct dtb_node dtb_root_node(void) +{ + return make_node_at_offset(be32toh(global_dtb->offset_dt_struct)); +} + +struct dtb_prop dtb_node_first_prop(dtb_node_t node) +{ + vm_offset_t offset = node->offset; + + assert(offset != DTB_SENTINEL_OFFSET); + skip_node_header(&offset); + + return make_prop_at_offset(offset); +} + +struct dtb_prop dtb_node_next_prop(dtb_prop_t prev_prop) +{ + boolean_t skipped; + vm_offset_t offset = prev_prop->offset; + + assert(offset != DTB_SENTINEL_OFFSET); + skipped = skip_prop(&offset); + assert(skipped); + return make_prop_at_offset(offset); +} + +struct dtb_node dtb_node_first_child(dtb_node_t parent) +{ + vm_offset_t offset = parent->offset; + struct dtb_prop prop; + struct dtb_node node; + + unsigned short address_cells = DEFAULT_ADDRESS_CELLS; + unsigned short size_cells = DEFAULT_SIZE_CELLS; + + assert(offset != DTB_SENTINEL_OFFSET); + + skip_node_header(&offset); + while (TRUE) { + skip_padding(&offset); + skip_nops(&offset); + if (uint32_at_offset(offset) != DTB_PROP) + break; + prop = make_prop_at_offset(offset); + if (!strcmp(prop.name, "#address-cells")) { + assert(prop.length == 4); + address_cells = be32toh(*(const dtb_uint32_t *) prop.data); + } else if (!strcmp(prop.name, "#size-cells")) { + assert(prop.length == 4); + size_cells = be32toh(*(const dtb_uint32_t *) prop.data); + } + offset += 12 + prop.length; + } + + node = make_node_at_offset(offset); + node.address_cells = address_cells; + node.size_cells = size_cells; + return node; +} + +struct dtb_node dtb_node_next_sibling(dtb_node_t node) +{ + vm_offset_t offset = node->offset; + unsigned int depth = 0; + struct dtb_node sibling; + + assert(offset != DTB_SENTINEL_OFFSET); + assert(uint32_at_offset(offset) == DTB_BEGIN_NODE); + + do { + switch (uint32_at_offset(offset)) { + case DTB_BEGIN_NODE: + depth++; + skip_node_header(&offset); + break; + case DTB_PROP: + skip_prop(&offset); + break; + case DTB_END_NODE: + offset += 4; + depth--; + break; + default: + panic_unexpected_token(offset); + } + skip_padding(&offset); + skip_nops(&offset); + } while (depth > 0); + + sibling = make_node_at_offset(offset); + sibling.address_cells = node->address_cells; + sibling.size_cells = node->size_cells; + return sibling; +} + +struct dtb_node dtb_node_by_path(const char *node_path) +{ + boolean_t found; + const char *c = node_path + 1, *c2; + struct dtb_node node; + + assert(node_path[0] == '/'); + node = dtb_root_node(); + + while (TRUE) { + c2 = strchr(c, '/'); + if (c2 == NULL) + c2 = c + strlen(c); + found = FALSE; + dtb_for_each_child (node, node) { + if (!memcmp(node.name, c, c2 - c)) { + found = TRUE; + break; + } + } + if (!found) { + node.offset = DTB_SENTINEL_OFFSET; + node.name = NULL; + node.address_cells = node.size_cells = 0; + return node; + } + if (*c2 == 0) + return node; + c = c2 + 1; + } +} + +struct dtb_prop dtb_node_find_prop( + dtb_node_t node, + const char *prop_name) +{ + struct dtb_prop prop; + + dtb_for_each_prop (*node, prop) { + if (!strcmp(prop.name, prop_name)) + return prop; + } + + prop.offset = DTB_SENTINEL_OFFSET; + prop.name = NULL; + prop.data = NULL; + prop.length = 0; + return prop; +} + +boolean_t dtb_node_is_compatible( + dtb_node_t node, + const char *model) +{ + struct dtb_prop prop; + vm_size_t off; + + prop = dtb_node_find_prop(node, "compatible"); + if (DTB_IS_SENTINEL(prop)) + return FALSE; + + for (off = 0; off < prop.length;) { + if (!strcmp(model, prop.data + off)) + return TRUE; + off += strlen(prop.data + off) + 1; + } + + return FALSE; +} + +static uint64_t read_cells( + const void *addr, + unsigned short size, + vm_size_t *off) +{ + uint64_t tmp; + + addr = (const unsigned char *) addr + *off; + *off += size * 4; + + switch (size) { + case 0: + return 0; + case 1: + return be32toh(*(const dtb_uint32_t *) addr); + case 2: + __builtin_memcpy(&tmp, addr, 8); + return __builtin_bswap64(tmp); + default: + panic("Unimplemented cell size: %d\n", size); + } +} + +extern uint64_t dtb_prop_read_cells( + dtb_prop_t prop, + unsigned short size, + vm_size_t *off) +{ + assert((*off) + (size * 4) <= prop->length); + return read_cells(prop->data, size, off); +} + +struct dtb_ranges_map dtb_node_make_ranges_map(dtb_node_t node) +{ + struct dtb_ranges_map m; + struct dtb_prop prop; + + m.parent_address_cells = node->address_cells; + m.child_address_cells = DEFAULT_ADDRESS_CELLS; + m.child_size_cells = DEFAULT_SIZE_CELLS; + m.ranges = NULL; + m.ranges_length = 0; + m.next = NULL; + + dtb_for_each_prop (*node, prop) { + if (!strcmp(prop.name, "#address-cells")) { + assert(prop.length == 4); + m.child_address_cells = be32toh(*(const dtb_uint32_t *) prop.data); + } else if (!strcmp(prop.name, "#size-cells")) { + assert(prop.length == 4); + m.child_size_cells = be32toh(*(const dtb_uint32_t *) prop.data); + } else if (!strcmp(prop.name, "ranges")) { + m.ranges = prop.data; + m.ranges_length = prop.length; + } + } + + return m; +} + +vm_offset_t dtb_map_address( + dtb_ranges_map_t map, + vm_offset_t address) +{ + boolean_t found; + vm_offset_t child_addr, parent_addr; + vm_size_t size, off; + + for (; map != NULL; map = map->next) { + found = FALSE; + for (off = 0; off < map->ranges_length;) { + child_addr = read_cells(map->ranges, map->child_address_cells, &off); + parent_addr = read_cells(map->ranges, map->parent_address_cells, &off); + size = read_cells(map->ranges, map->child_size_cells, &off); + + if (child_addr <= address && address < child_addr + size) { + found = TRUE; + address = address - child_addr + parent_addr; + break; + } + } + assert(found || map->ranges_length == 0); + } + + return address; +} diff --git a/device/dtb.h b/device/dtb.h new file mode 100644 index 00000000..3a090b2c --- /dev/null +++ b/device/dtb.h @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024 Free Software Foundation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef _DEVICE_DTB_H_ +#define _DEVICE_DTB_H_ + +#include <mach/boolean.h> +#include <mach/kern_return.h> +#include <mach/machine/vm_types.h> +#include <device/device_types.h> + +#define DTB_SENTINEL_OFFSET ((vm_offset_t) -1) +#define DTB_IS_SENTINEL(s) ((s).offset == DTB_SENTINEL_OFFSET) + +/* + * Big endian 4-byte integer. + */ +typedef struct { + unsigned char bytes[4]; +} dtb_uint32_t; + +struct dtb_header { + dtb_uint32_t magic; + dtb_uint32_t total_size; + dtb_uint32_t offset_dt_struct; + dtb_uint32_t offset_dt_strings; + dtb_uint32_t offset_mem_rsvmap; + dtb_uint32_t version; + dtb_uint32_t last_compatible_version; + dtb_uint32_t boot_cpuid_phys; + dtb_uint32_t sizeof_dt_strings; + dtb_uint32_t sizeof_dt_struct; +}; + +typedef const struct dtb_header *dtb_t; + +extern kern_return_t dtb_load(dtb_t dtb); +extern void dtb_get_location(dtb_t *out_dtb, vm_size_t *out_dtb_size); + +struct dtb_node { + vm_offset_t offset; + const char *name; + unsigned short address_cells; + unsigned short size_cells; +}; + +typedef const struct dtb_node *dtb_node_t; + +struct dtb_prop { + vm_offset_t offset; + const char *name; + const void *data; + vm_size_t length; +}; + +typedef const struct dtb_prop *dtb_prop_t; + +struct dtb_ranges_map { + unsigned short child_address_cells; + unsigned short parent_address_cells; + unsigned short child_size_cells; + const void *ranges; + vm_size_t ranges_length; + + const struct dtb_ranges_map *next; +}; + +typedef const struct dtb_ranges_map *dtb_ranges_map_t; + +extern struct dtb_node dtb_root_node(void); +extern struct dtb_prop dtb_node_first_prop(dtb_node_t node); +extern struct dtb_prop dtb_node_next_prop(dtb_prop_t prop); +extern struct dtb_node dtb_node_first_child(dtb_node_t node); +extern struct dtb_node dtb_node_next_sibling(dtb_node_t node); + +#define dtb_for_each_child(parent, child) \ + for (child = dtb_node_first_child(&(parent)); \ + !DTB_IS_SENTINEL(child); \ + child = dtb_node_next_sibling(&child)) + +#define dtb_for_each_prop(node, prop) \ + for (prop = dtb_node_first_prop(&(node)); \ + !DTB_IS_SENTINEL(prop); \ + prop = dtb_node_next_prop(&prop)) + +extern struct dtb_node dtb_node_by_path(const char *node_path); + +extern struct dtb_prop dtb_node_find_prop( + dtb_node_t node, + const char *prop_name); + +extern boolean_t dtb_node_is_compatible( + dtb_node_t node, + const char *model); + +extern uint64_t dtb_prop_read_cells( + dtb_prop_t prop, + unsigned short size, + vm_size_t *off); + +extern struct dtb_ranges_map dtb_node_make_ranges_map( + dtb_node_t node); + +extern vm_offset_t dtb_map_address( + dtb_ranges_map_t map, + vm_offset_t address); + +#endif /* _DEVICE_DTB_H_ */ diff --git a/device/ramdisk.c b/device/ramdisk.c new file mode 100644 index 00000000..daf70436 --- /dev/null +++ b/device/ramdisk.c @@ -0,0 +1,160 @@ +#include <mach/vm_param.h> +#include <machine/vm_param.h> +#include <vm/vm_kern.h> +#include <vm/vm_user.h> +#include <device/device_types.h> +#include <device/ds_routines.h> +#include <device/conf.h> +#include <device/ramdisk.h> +#include <kern/printf.h> +#include <string.h> + +static struct ramdisk { + void *data; + vm_size_t size; +} ramdisk[RAMDISK_MAX]; + +static int ramdisk_num = 0; + +/* Initial ramdisks are created from the boot scripts */ +int ramdisk_create(vm_size_t size, const void *initdata, int *out_no) +{ + struct ramdisk *rd = &ramdisk[ramdisk_num]; + int err; + + if(ramdisk_num >= RAMDISK_MAX) + return -1; + + /* allocate the memory */ + rd->size = round_page(size); + err = kmem_alloc(kernel_map, (vm_offset_t *) &rd->data, rd->size); + if(err != KERN_SUCCESS) + return err; + + /* initialize */ + if(initdata) + memcpy(rd->data, initdata, rd->size); + else + memset(rd->data, 0, rd->size); + + /* report */ + if(out_no) *out_no = ramdisk_num; + printf("%s%d: %lu bytes @%p\n", RAMDISK_NAME, ramdisk_num, + (unsigned long) rd->size, rd->data); + + ramdisk_num++; + return KERN_SUCCESS; +} + +/* On d_open() we just check whether the ramdisk exists */ +int ramdisk_open(dev_t dev, int mode, io_req_t ior) +{ + return (dev < ramdisk_num) ? D_SUCCESS : D_NO_SUCH_DEVICE; +} + +/* d_getstat() is used to query the device characteristics */ +int ramdisk_getstat(dev_t dev, dev_flavor_t flavor, dev_status_t status, + mach_msg_type_number_t *status_count) +{ + switch(flavor) { + case DEV_GET_SIZE: + status[DEV_GET_SIZE_DEVICE_SIZE] = ramdisk[dev].size; + status[DEV_GET_SIZE_RECORD_SIZE] = RAMDISK_BLOCKSZ; + *status_count = DEV_GET_SIZE_COUNT; + return D_SUCCESS; + + case DEV_GET_RECORDS: + status[DEV_GET_RECORDS_DEVICE_RECORDS] + = ramdisk[dev].size / RAMDISK_BLOCKSZ; + status[DEV_GET_RECORDS_RECORD_SIZE] = RAMDISK_BLOCKSZ; + *status_count = DEV_GET_RECORDS_COUNT; + return D_SUCCESS; + } + return D_INVALID_OPERATION; +} + +/* TODO: implement freeramdisk with setstat() ? */ + +/* Check the given io request and compute a pointer to the ramdisk data and the + * amount to be handled. */ +static int ramdisk_ioreq(int dev, io_req_t ior, void **data, int *amt) +{ + vm_offset_t ofs = ior->io_recnum * RAMDISK_BLOCKSZ; + if(ofs >= ramdisk[dev].size) + return D_INVALID_RECNUM; + + *data = (char*) ramdisk[dev].data + ofs; + *amt = ior->io_count; + if(ofs + *amt > ramdisk[dev].size) + *amt = ramdisk[dev].size - ofs; + + return KERN_SUCCESS; +} + +/* Copy data from a vm_map_copy by mapping it temporarily. */ +static int mem_map_cpy(void *dst, vm_map_copy_t src, int amt) +{ + vm_offset_t srcaddr; + int err; + + err = vm_map_copyout(device_io_map, &srcaddr, src); + if (err != KERN_SUCCESS) + return err; + + memcpy(dst, (void *) srcaddr, amt); + vm_deallocate(device_io_map, srcaddr, amt); + return KERN_SUCCESS; +} + +int ramdisk_read(dev_t dev, io_req_t ior) +{ + void *data; + int amt, err; + + err = ramdisk_ioreq(dev, ior, &data, &amt); + if(err != KERN_SUCCESS) + return err; + + err = device_read_alloc (ior, ior->io_count); + if (err != KERN_SUCCESS) + return err; + + memcpy(ior->io_data, data, amt); + ior->io_residual = ior->io_count - amt; + + return D_SUCCESS; +} + +int ramdisk_write(dev_t dev, io_req_t ior) +{ + void *data; + int amt, err; + + err = ramdisk_ioreq(dev, ior, &data, &amt); + if(err != KERN_SUCCESS) + return err; + + if (!(ior->io_op & IO_INBAND)) { + /* Out-of-band data is transmitted as a vm_map_copy */ + err = mem_map_cpy(data, (vm_map_copy_t) ior->io_data, amt); + if(err != KERN_SUCCESS) + return err; + } else { + /* In-band data can be accessed directly */ + memcpy(data, ior->io_data, amt); + } + + ior->io_residual = ior->io_count - amt; + return D_SUCCESS; +} + +vm_offset_t ramdisk_mmap(dev_t dev, vm_offset_t off, vm_prot_t prot) +{ + if(dev >= ramdisk_num) + return -1; + if(off >= ramdisk[dev].size) + return -1; + + return pmap_phys_to_frame(kvtophys((vm_offset_t) ramdisk[dev].data + off)); +} + diff --git a/device/ramdisk.h b/device/ramdisk.h new file mode 100644 index 00000000..ac71f084 --- /dev/null +++ b/device/ramdisk.h @@ -0,0 +1,47 @@ +#ifndef _KERN_RAMDISK_H_ +#define _KERN_RAMDISK_H_ + +#include <vm/pmap.h> +#include <device/io_req.h> +#include <device/conf.h> + +/* Maximum number of ramdisk devices */ +#define RAMDISK_MAX 4 + +/* The block size used (userspace requires 512) */ +#define RAMDISK_BLOCKSZ 512 + +/* Name associated to the ramdisk major */ +#define RAMDISK_NAME "rd" +#define RAMDISK_NAMESZ (sizeof RAMDISK_NAME + sizeof (int) * 3 + 1) + +/* Create a new ramdisk of the given size. On success, if out_no and/or out_ptr + * are not NULL, the device number and pointer to the ramdisk's data are stored + * there. Returns D_SUCCESS or D_NO_MEMORY. */ +int ramdisk_create(vm_size_t size, const void *initdata, int *out_no); + +/* Device operations */ +int ramdisk_open(dev_t, int, io_req_t); +int ramdisk_getstat(dev_t, dev_flavor_t, dev_status_t, mach_msg_type_number_t *); +int ramdisk_read(dev_t, io_req_t); +int ramdisk_write(dev_t, io_req_t); +vm_offset_t ramdisk_mmap(dev_t, vm_offset_t, vm_prot_t); + +/* dev_ops initializer to be used from <machine>/conf.c */ +#define RAMDISK_DEV_OPS { \ + .d_name = RAMDISK_NAME, \ + .d_open = ramdisk_open, \ + .d_close = nulldev_close, \ + .d_read = ramdisk_read, \ + .d_write = ramdisk_write, \ + .d_getstat = ramdisk_getstat, \ + .d_setstat = nulldev_setstat, \ + .d_mmap = ramdisk_mmap, \ + .d_async_in = nodev_async_in, \ + .d_reset = nulldev_reset, \ + .d_port_death = nulldev_portdeath, \ + .d_subdev = 0, \ + .d_dev_info = nodev_info, \ + } + +#endif -- 2.54.0