[RFC PATCH v3 3/5] mpt: Add Smsdid and Smmpt supervisor domain core
Rahul Pathak <[email protected]>
| Newsgroups | org.infradead.lists.opensbi |
|---|---|
| Message-ID | <[email protected]> |
Introduce the supervisord omain memory-protection (Smmpt) table management core. This adds the mode agnostic core layer and the per mode table walkers/builder. Add defines for the mmpt CSR encoding, MPTE bit layout, permission flags, structures, fence helpers and the public interfaces. It implements the RV64 supported modes (Smmpt43/52/64) and the RV32 mode (Smmpt34), its MPT table support. Also implements mpt core which integrates Smmpt with SBI domains and other core layers. Signed-off-by: Rahul Pathak <[email protected]> --- include/sbi/sbi_mpt.h | 336 ++++++++++++++ lib/sbi/objects.mk | 2 + lib/sbi/sbi_mpt.c | 955 +++++++++++++++++++++++++++++++++++++++ lib/sbi/sbi_mpt_mode.c | 997 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 2290 insertions(+) create mode 100644 include/sbi/sbi_mpt.h create mode 100644 lib/sbi/sbi_mpt.c create mode 100644 lib/sbi/sbi_mpt_mode.c diff --git a/include/sbi/sbi_mpt.h b/include/sbi/sbi_mpt.h new file mode 100644 index 00000000..f4e3a623 --- /dev/null +++ b/include/sbi/sbi_mpt.h @@ -0,0 +1,336 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Authors: Rahul Pathak <[email protected]> + */ + +#ifndef __SBI_MPT_H__ +#define __SBI_MPT_H__ + +#include <sbi/sbi_types.h> +#include <sbi/riscv_locks.h> +#include <sbi/sbi_scratch.h> +#include <sbi/sbi_bitmap.h> +#include <sbi/riscv_asm.h> + +struct sbi_domain; +struct sbi_mpt_domain; +struct sbi_mpt_ctrl; +struct sbi_domain_memregion; + +/* CSR addresses */ +#define SBI_CSR_MMPT 0x382 +#define SBI_CSR_MSDCFG 0x74E + +/* mmpt register field definitions */ +#if __riscv_xlen == 64 +# define SBI_MMPT_MODE_SHIFT 60UL +# define SBI_MMPT_MODE_MASK (UL(0xF) << SBI_MMPT_MODE_SHIFT) +# define SBI_MMPT_SDID_SHIFT 52UL +# define SBI_MMPT_SDID_MASK (UL(0x3F) << SBI_MMPT_SDID_SHIFT) +# define SBI_MMPT_PPN_MASK UL(0x00000FFFFFFFFFFF) +#else +# define SBI_MMPT_MODE_SHIFT 30UL +# define SBI_MMPT_MODE_MASK (UL(0x3) << SBI_MMPT_MODE_SHIFT) +# define SBI_MMPT_SDID_SHIFT 22UL +# define SBI_MMPT_SDID_MASK (UL(0x3F) << SBI_MMPT_SDID_SHIFT) +# define SBI_MMPT_PPN_MASK UL(0x3FFFFF) +#endif + +#define SBI_MMPT_MODE_BARE 0UL +#if __riscv_xlen == 64 +# define SBI_MMPT_MODE_SMMPT43 1UL +# define SBI_MMPT_MODE_SMMPT52 2UL +# define SBI_MMPT_MODE_SMMPT64 3UL +#else +# define SBI_MMPT_MODE_SMMPT34 1UL +#endif + +/* + * sbi_mmpt_encode() — assemble the full mmpt CSR value. + */ +static inline unsigned long sbi_mmpt_encode(u32 mode, u32 sdid, + unsigned long ppn) +{ + return (((unsigned long)mode << SBI_MMPT_MODE_SHIFT) & SBI_MMPT_MODE_MASK) | + (((unsigned long)sdid << SBI_MMPT_SDID_SHIFT) & SBI_MMPT_SDID_MASK) | + (ppn & SBI_MMPT_PPN_MASK); +} + +/* + * MPTE bit definitions + */ +#define SBI_MPTE_V (1UL << 0) +#define SBI_MPTE_L (1UL << 1) +#define SBI_MPTE_N (1UL << 2) +#define SBI_MPTE_PPN_SHIFT 10U +#define SBI_MPT_PAGE_SHIFT 12U +#define SBI_MPT_PAGE_SIZE 4096UL +#define SBI_MPT_PAGE_MASK (SBI_MPT_PAGE_SIZE - 1UL) +#define SBI_MPTE_XWR_BASE 8U +#define SBI_MPTE_XWR_WIDTH 3U +#define SBI_MPTE_XWR_MASK 7UL + +static inline u32 sbi_mpte_xwr_shift(u32 n) +{ + return (SBI_MPTE_XWR_BASE + n * SBI_MPTE_XWR_WIDTH); +} + +static inline unsigned long sbi_mpte_leaf_set_xwr(unsigned long mpte, + u32 n, u8 xwr) +{ + u32 sh = sbi_mpte_xwr_shift(n); + + mpte &= ~(SBI_MPTE_XWR_MASK << sh); + mpte |= (xwr & SBI_MPTE_XWR_MASK) << sh; + return mpte; +} + +static inline unsigned long sbi_mpte_nonleaf(unsigned long table_pa) +{ + return (((table_pa >> SBI_MPT_PAGE_SHIFT) << SBI_MPTE_PPN_SHIFT) + | SBI_MPTE_V); +} + +static inline unsigned long sbi_mpte_nonleaf_table_pa(unsigned long mpte) +{ + return ((mpte >> SBI_MPTE_PPN_SHIFT) << SBI_MPT_PAGE_SHIFT); +} + +/* + * XWR permission constants (XWR=000 means no access) + */ +#define SBI_MPT_PERM_NONE 0U +#define SBI_MPT_PERM_R 1U +#define SBI_MPT_PERM_W 2U +#define SBI_MPT_PERM_X 4U +#define SBI_MPT_PERM_RW (SBI_MPT_PERM_R | SBI_MPT_PERM_W) +#define SBI_MPT_PERM_RX (SBI_MPT_PERM_R | SBI_MPT_PERM_X) +#define SBI_MPT_PERM_RWX (SBI_MPT_PERM_R | SBI_MPT_PERM_W | SBI_MPT_PERM_X) + +/* + * SDID constants + */ +#define SBI_MPT_SDIDMAX 6U +#define SBI_MPT_MAX_DOMAINS (1UL << SBI_MPT_SDIDMAX) +#define SBI_MPT_SDID_INVALID SBI_MPT_MAX_DOMAINS + +/* + * SBI_MPT_MAX_REGIONS_DOMAIN: Number of regions a domain may have + * Used for tracking of regions + */ +#define SBI_MPT_MAX_REGIONS_DOMAIN 32U + +/* + * SMSDID mfence and minval raw encodings + * + * R-type, opcode=SYSTEM(0x73), funct3=0, rd=x0, rs1=PADDR, rs2=SDID + * mfence.pa funct7=0x19 -> 0x32000073 where rs1=rs2=x0 + * minval.pa funct7=0x1b -> 0x36000073 where rs1=rs2=x0 + * + * .insn r opcode, func3, func7, rd, rs1, rs2 + */ +#define SBI_MPT_MFENCE_PA(rs1, rs2) ".insn r 0x73, 0, 0x19, x0, " rs1 ", " rs2 + +#define SBI_MPT_MINVAL_PA(rs1, rs2) ".insn r 0x73, 0, 0x1b, x0, " rs1 ", " rs2 + +/* + * Fence and invalidation helpers + */ +static inline void sbi_mpt_fence_all(void) +{ + __asm__ volatile(SBI_MPT_MFENCE_PA("x0", "x0") ::: "memory"); +} + +static inline void sbi_mpt_fence_sdid(u32 sdid) +{ + unsigned long s = (unsigned long)sdid & (SBI_MPT_MAX_DOMAINS - 1UL); + + __asm__ volatile(SBI_MPT_MFENCE_PA("x0", "%0") :: "r"(s) : "memory"); +} + +/* + * rs1=paddr and rs2=x0 + */ +static inline void sbi_mpt_mfence_addr(unsigned long paddr) +{ + __asm__ volatile(SBI_MPT_MFENCE_PA("%0", "x0") :: "r"(paddr) : "memory"); +} + +/* + * rs1=paddr and rs2=sdid + */ +static inline void sbi_mpt_mfence_pa(unsigned long paddr, u32 sdid) +{ + unsigned long s = (unsigned long)sdid & (SBI_MPT_MAX_DOMAINS - 1UL); + + __asm__ volatile(SBI_MPT_MFENCE_PA("%0", "%1") + :: "r"(paddr), "r"(s) : "memory"); +} + +/* + * rs1=0 and rs2=0 + */ +static inline void sbi_mpt_minval_all(void) +{ + /* Prior stores globally visible before invalidation */ + __asm__ volatile("sfence.w.inval" ::: "memory"); + __asm__ volatile(SBI_MPT_MINVAL_PA("x0", "x0") ::: "memory"); + /* Invalidation completes before subsequent implicit accesses */ + __asm__ volatile("sfence.inval.ir" ::: "memory"); +} + +/* + * rs1=paddr and rs2=sdid + */ +static inline void sbi_mpt_minval_sdid(unsigned long pa, u32 sdid) +{ + unsigned long s = (unsigned long)sdid & (SBI_MPT_MAX_DOMAINS - 1UL); + + __asm__ volatile("sfence.w.inval" ::: "memory"); + __asm__ volatile(SBI_MPT_MINVAL_PA("%0", "%1") + :: "r"(pa), "r"(s) : "memory"); + __asm__ volatile("sfence.inval.ir" ::: "memory"); +} + +/* + * Core data structures + */ +struct sbi_mpt_region { + unsigned long pa; + unsigned long size; + u8 xwr; + /* region is locked and deny modification via add/remove_region */ + bool locked; + /* mapped into more than one domain */ + bool shared; +}; + +/* + * struct sbi_mpt_mode — SMMPT mode description. + */ +struct sbi_mpt_mode { + const char *name; + u32 mode_val; + + /* Assemble the MMPT CSR value */ + unsigned long (*encode_mmpt)(unsigned long ppn, u32 sdid); + + /* map a range in MPT table */ + int (*map_range)(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, u8 xwr); + + /* number bytes for root table allocation */ + unsigned long (*root_table_size)(void); + + /* required root table physical address alignment */ + unsigned long (*root_table_align)(void); + + /* check if [pa,pa+size] fits in smmpt mode supported address space */ + bool (*pa_in_range)(unsigned long pa, unsigned long size); + + /* map full range of address space which is supported by the mode */ + int (*map_full_range)(struct sbi_mpt_domain *dom, u8 xwr); + + /* read the xwr for a PA */ + u8 (*get_xwr)(struct sbi_mpt_domain *dom, unsigned long pa); +}; + +struct sbi_mpt_domain { + u32 sdid; + bool valid; + unsigned long root_pa; + struct sbi_mpt_mode *mode; + struct sbi_domain *sbi_dom; + unsigned int nregions; + struct sbi_mpt_region regions[SBI_MPT_MAX_REGIONS_DOMAIN]; + spinlock_t lock; +}; + +/* + * struct sbi_mpt_ctrl — RDSM MPT state + */ +struct sbi_mpt_ctrl { + bool ready; + struct sbi_mpt_mode *mode; + /* detected sdidlen from mmpt probe */ + u32 sdid_len; + /* 2^sdid_len — runtime limit for all domain operations */ + u32 max_domains; + unsigned long fw_pa; + unsigned long fw_size; + struct sbi_mpt_domain domains[SBI_MPT_MAX_DOMAINS]; + u32 ndomain; + /* sdid offset in scratch */ + unsigned long sdid_offset; + /* bitmap to track assigned/free SDID (bit N set = SDID N free) */ + DECLARE_BITMAP(sdid_bitmap, SBI_MPT_MAX_DOMAINS); +}; + +struct sbi_mpt_ctrl *sbi_mpt_ctrl_get(void); + +static inline u32 sbi_mpt_thishart_sdid(void) +{ + struct sbi_mpt_ctrl *ctrl = sbi_mpt_ctrl_get(); + u32 *p; + + if (!ctrl->ready || !ctrl->sdid_offset) + return (u32)SBI_MPT_SDID_INVALID; + p = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), + ctrl->sdid_offset); + return *p; +} + +/* + * Domain creation config + * + * sbi_dom and regions are mutually exclusive. + * when a supervisor domain is backed by a SBI domain + * then the regions list and access permissions are derived + * from the SBI domain structures. + */ +struct sbi_mpt_domain_config { + /* SBI domain whose S/U memregions are mapped. */ + struct sbi_domain *sbi_dom; + /* permission for regions inherited from sbi_dom */ + u8 xwr; + /* protect firmware regions for the domain */ + bool fw_protect; + /* + * explicit region list mapped in MPT table. + * NULL when sbi_dom covers everything + */ + struct sbi_mpt_region *regions; + u32 nregions; +}; + +int sbi_mpt_init(void); + +int sbi_mpt_domain_create(const struct sbi_mpt_domain_config *cfg, u32 *out_sdid); + +int sbi_mpt_domain_add_region(u32 sdid, unsigned long pa, unsigned long size, u8 xwr); + +int sbi_mpt_domain_remove_region(u32 sdid, unsigned long pa, unsigned long size); + +int sbi_mpt_share_region(u32 src_sdid, u32 dst_sdid, unsigned long pa, unsigned long size, u8 dst_xwr); + +int sbi_mpt_unshare_region(u32 sdid, unsigned long pa, unsigned long size); + +int sbi_mpt_hart_activate(u32 sdid); + +void sbi_mpt_hart_deactivate(void); + +int sbi_mpt_hart_activate_for_domain(struct sbi_domain *sbi_dom); + +struct sbi_mpt_domain *sbi_mpt_domain_get(u32 sdid); + +unsigned long sbi_mpt_pool_alloc(unsigned long size, unsigned long align); + +void sbi_mpt_sdid_free(u32 sdid); + +int sbi_mpt_query_access(u32 sdid, unsigned long pa, u8 *out_xwr); + +void sbi_mpt_dump(void); + +#endif /* __SBI_MPT_H__ */ diff --git a/lib/sbi/objects.mk b/lib/sbi/objects.mk index c29c888f..3b3c3ea3 100644 --- a/lib/sbi/objects.mk +++ b/lib/sbi/objects.mk @@ -108,3 +108,5 @@ libsbi-objs-y += sbi_expected_trap.o libsbi-objs-y += sbi_cppc.o libsbi-objs-$(CC_SUPPORT_VECTOR) += sbi_vector.o libsbi-objs-y += sbi_fp.o +libsbi-objs-y += sbi_mpt.o +libsbi-objs-y += sbi_mpt_mode.o diff --git a/lib/sbi/sbi_mpt.c b/lib/sbi/sbi_mpt.c new file mode 100644 index 00000000..4d909591 --- /dev/null +++ b/lib/sbi/sbi_mpt.c @@ -0,0 +1,955 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Authors: Rahul Pathak <[email protected]> + */ + +#include <sbi/sbi_mpt.h> +#include <sbi/sbi_domain.h> +#include <sbi/sbi_error.h> +#include <sbi/sbi_hart.h> +#include <sbi/sbi_heap.h> +#include <sbi/sbi_scratch.h> +#include <sbi/sbi_string.h> +#include <sbi/sbi_console.h> +#include <sbi/sbi_hart_protection.h> +#include <sbi/riscv_locks.h> +#include <sbi/riscv_asm.h> + +static struct sbi_mpt_ctrl mpt_ctrl; + +struct sbi_mpt_ctrl *sbi_mpt_ctrl_get(void) +{ + return &mpt_ctrl; +} + +struct sbi_mpt_domain *sbi_mpt_domain_get(u32 sdid) +{ + struct sbi_mpt_domain *sd; + + if (sdid >= mpt_ctrl.max_domains) + return NULL; + + sd = mpt_ctrl.domains[sdid].valid ? &mpt_ctrl.domains[sdid] : NULL; + + return sd; +} + +/* Memory allocator for MPT tables from heap */ +unsigned long sbi_mpt_pool_alloc(unsigned long size, unsigned long align) +{ + unsigned long ptr = (unsigned long)sbi_aligned_alloc(align, size); + + if (!ptr) { + sbi_printf("sbi_mpt: alloc failed (size=0x%lx align=0x%lx)\n", + size, align); + return 0; + } + + /* MPT tables must be zeroed to mark them invalid (mpte.V = 0) */ + sbi_memset((void *)ptr, 0, size); + + return ptr; +} + +/* Check if two regions overlap */ +static bool regions_overlap(unsigned long pa1, unsigned long size1, + unsigned long pa2, unsigned long size2) +{ + return (pa1 < pa2 + size2) && (pa2 < pa1 + size1); +} + +/* Check if a region is completely contained inside another region */ +static bool region_contained(unsigned long child_pa, unsigned long child_size, + unsigned long parent_pa, unsigned long parent_size) +{ + return (child_pa >= parent_pa) && + (child_pa + child_size <= parent_pa + parent_size); +} + +/* + * Check if a region [pa, pa+size] overlaps with any mapped + * region which is locked + */ +static bool region_overlaps_locked(const struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size) +{ + unsigned int i; + + for (i = 0; i < dom->nregions; i++) + if (dom->regions[i].locked && + regions_overlap(pa, size, dom->regions[i].pa, dom->regions[i].size)) + return true; + return false; +} + +/* Probe SDIDLEN from the mmpt CSR. */ +static u32 probe_sdidlen(u32 mode_val) +{ + unsigned long saved, sdid_val; + u32 sdidlen = 0; + + saved = csr_read(SBI_CSR_MMPT); + + /* Program a legal non-Bare MODE with all-ones in the SDID field */ + csr_write(SBI_CSR_MMPT, + (((unsigned long)mode_val << SBI_MMPT_MODE_SHIFT) & SBI_MMPT_MODE_MASK) | + SBI_MMPT_SDID_MASK); + + sdid_val = (csr_read(SBI_CSR_MMPT) & SBI_MMPT_SDID_MASK) >> SBI_MMPT_SDID_SHIFT; + + while (sdid_val) { + sdidlen += 1; + sdid_val >>= 1; + } + + if (sdidlen > SBI_MPT_SDIDMAX) + sdidlen = SBI_MPT_SDIDMAX; + + csr_write(SBI_CSR_MMPT, saved); + + return sdidlen; +} + +#if __riscv_xlen == 64 +extern struct sbi_mpt_mode smmpt43_mode; +extern struct sbi_mpt_mode smmpt52_mode; +extern struct sbi_mpt_mode smmpt64_mode; +#else +extern struct sbi_mpt_mode smmpt34_mode; +#endif + +static bool probe_mode(u32 mode_val) +{ + u32 m; + csr_write(SBI_CSR_MMPT, + ((unsigned long)mode_val << SBI_MMPT_MODE_SHIFT) & SBI_MMPT_MODE_MASK); + + m = (csr_read(SBI_CSR_MMPT) & SBI_MMPT_MODE_MASK) >> SBI_MMPT_MODE_SHIFT; + + return (m == mode_val); +} + + +/* + * Probe SMMPT mode and initialize the related mode ops + * + * Deliberately prefer coverage over latency, because latency can be + * improved by the MPT cache which hardware may implement but the + * coverage is not. So pick the widest mode which is supported by the + * hardware. + */ +static struct sbi_mpt_mode *init_mpt_mode(u32 *out_mode_val) +{ + unsigned long saved = csr_read(SBI_CSR_MMPT); + struct sbi_mpt_mode *s = NULL; + u32 mode_val = 0; + +#if __riscv_xlen == 64 + if (probe_mode(SBI_MMPT_MODE_SMMPT64)) { + s = &smmpt64_mode; + mode_val = SBI_MMPT_MODE_SMMPT64; + } else if (probe_mode(SBI_MMPT_MODE_SMMPT52)) { + s = &smmpt52_mode; + mode_val = SBI_MMPT_MODE_SMMPT52; + } else if (probe_mode(SBI_MMPT_MODE_SMMPT43)) { + s = &smmpt43_mode; + mode_val = SBI_MMPT_MODE_SMMPT43; + } +#else + if (probe_mode(SBI_MMPT_MODE_SMMPT34)) { + s = &smmpt34_mode; + mode_val = SBI_MMPT_MODE_SMMPT34; + } +#endif + + csr_write(SBI_CSR_MMPT, saved); + + if (out_mode_val) + *out_mode_val = mode_val; + + return s; +} + +/* + * mpt_map_range_perm(): Maps [pa, pa+size] region into supervisor domain MPT table + * with its access permissions(xwr). + * + * locked=true: means entry is immutable and it cannot be modified. Used for + * firmware deny-all and other such regions for permanenet access policy. + * + * locked=false: region entry is modifiable at runtime via add/remove_region. + * Used for S/U-mode accessible domain regions. + */ + +static int mpt_map_range_perm(struct sbi_mpt_domain *dom, struct sbi_mpt_mode *sch, + unsigned long pa, unsigned long size, u8 xwr, bool locked) +{ + int rc; + bool first_in_range; + + if (!size) { + sbi_printf("sbi_mpt: invalid size for range 0x%lx\n", pa); + return SBI_EINVAL; + } + + if ((pa & SBI_MPT_PAGE_MASK) || (size & SBI_MPT_PAGE_MASK)) { + sbi_printf("sbi_mpt: unaligned range 0x%lx+0x%lx\n", pa, size); + return SBI_EINVAL; + } + + if (!sch->pa_in_range(pa, size)) { + /* + * A region that pa_in_range() returns false falls into one of two + * different cases - PA which is out of range from the mappable + * range of that mode and second is PA is in range but the end + * (pa + size) is crossing the mappable range. + * + * Distinguish the two by probing pa_in_range() for just + * the first page in that big range and if even the first page + * is out of range it is the first case and if the base is in + * range but the whole region is not, it is case later. + */ + first_in_range = sch->pa_in_range(pa, SBI_MPT_PAGE_SIZE); + + if (locked) { + sbi_printf("sbi_mpt: locked region 0x%lx+0x%lx out of mode range\n", + pa, size); + return SBI_EINVAL; + } + + /* Region with PA which partially crossover the mappable space */ + if (first_in_range) { + sbi_printf("sbi_mpt: fatal region 0x%lx+0x%lx crossover" + " mode '%s' ceiling; high part may alias" + " with low MPTE entries — not mapping this region\n", + pa, size, sch->name); + return SBI_EINVAL; + } + + /* Region which is entirely above the ceiling — unreachable */ + sbi_printf("sbi_mpt: skip out-of-range 0x%lx+0x%lx" + " (entirely above '%s' ceiling; no MPT entry,\n", + pa, size, sch->name); + return 0; + } + + rc = sch->map_range(dom, pa, size, xwr); + if (rc) + return rc; + + if (dom->nregions >= SBI_MPT_MAX_REGIONS_DOMAIN) + return SBI_ENOMEM; + + dom->regions[dom->nregions].pa = pa; + dom->regions[dom->nregions].size = size; + dom->regions[dom->nregions].xwr = xwr; + dom->regions[dom->nregions].locked = locked; + dom->regions[dom->nregions].shared = false; + dom->nregions++; + + return 0; +} + +/* + * mpt_xwr_from_memregion(): derive an MPTE XWR encoding from a SBI domain + * memregion's S/U access flags, clamped by cap. + * + * cap is an upper bound the caller may impose. A region from SBI domain + * may have more permissive access permissions which an supervisor domain + * may want to restict and cap them. + */ +static u8 mpt_xwr_from_memregion(const struct sbi_domain_memregion *mr, + u8 cap) +{ + u8 xwr = SBI_MPT_PERM_NONE; + + if (mr->flags & SBI_DOMAIN_MEMREGION_SU_READABLE) + xwr |= SBI_MPT_PERM_R; + if (mr->flags & SBI_DOMAIN_MEMREGION_SU_WRITABLE) + xwr |= SBI_MPT_PERM_W; + if (mr->flags & SBI_DOMAIN_MEMREGION_SU_EXECUTABLE) + xwr |= SBI_MPT_PERM_X; + + xwr &= cap; + + /* XWR encodings 0b010 (W) and 0b110 (WX) are reserved */ + if ((xwr & SBI_MPT_PERM_W) && !(xwr & SBI_MPT_PERM_R)) { + sbi_printf("sbi_mpt: region 0x%lx: W without R is a reserved" + " XWR encoding — denying region\n", mr->base); + return SBI_MPT_PERM_NONE; + } + + return xwr; +} + +static int smmpt_hart_configure(struct sbi_scratch *scratch, + struct sbi_domain *dom) +{ + return sbi_mpt_hart_activate_for_domain(dom); +} + +static void smmpt_hart_unconfigure(struct sbi_scratch *scratch, + struct sbi_domain *dom) +{ + sbi_mpt_hart_deactivate(); +} + +static struct sbi_hart_protection smmpt_protection = { + .name = "smmpt", + .type = SBI_HART_PROTECTION_TYPE_ID, + .rating = 100, + .configure = smmpt_hart_configure, + .unconfigure = smmpt_hart_unconfigure, +}; + +/* + * sbi_mpt_domain_create(): Create a Supervisor Domain + */ +int sbi_mpt_domain_create(const struct sbi_mpt_domain_config *cfg, + u32 *out_sdid) +{ + int rc; + u32 sdid, i; + unsigned long root_pa; + struct sbi_mpt_mode *sch; + struct sbi_mpt_domain *dom; + const struct sbi_domain_memregion *mr; + struct sbi_mpt_ctrl *ctrl = &mpt_ctrl; + u8 base_xwr = SBI_MPT_PERM_NONE; + + if (!ctrl->ready) + return SBI_ENODEV; + + if (cfg->sbi_dom && cfg->nregions) { + sbi_printf("sbi_mpt: sbi_dom and explicit regions are mutually exclusive\n"); + return SBI_EINVAL; + } + + /* + * sdid_bitmap empty means every available SDID is + * currently in use and no domain can be created. + */ + if (bitmap_empty(ctrl->sdid_bitmap, ctrl->max_domains)) + return SBI_ENOMEM; + + sch = ctrl->mode; + + root_pa = sbi_mpt_pool_alloc(sch->root_table_size(), + sch->root_table_align()); + if (!root_pa) { + sbi_printf("sbi_mpt: OOM root table\n"); + return SBI_ENOMEM; + } + + /* + * Root MPT memory allocated — take SDID from bitmap for root supervisor + * domain + */ + for (sdid = 0; sdid < ctrl->max_domains; sdid++) + if (bitmap_test(ctrl->sdid_bitmap, sdid)) + break; + bitmap_clear(ctrl->sdid_bitmap, sdid, 1); + + dom = &ctrl->domains[sdid]; + SPIN_LOCK_INIT(dom->lock); + + dom->sdid = sdid; + dom->valid = true; + dom->root_pa = root_pa; + dom->mode = sch; + dom->sbi_dom = cfg->sbi_dom; + dom->nregions = 0; + + if (cfg->sbi_dom) { + /* + * Similar to SBI domain, create a baseline which + * maps whole mappable address space by a mode and mark it + * with XWR permissions. Later selectively restrict the + * permissions in another pass. + */ + sbi_domain_for_each_memregion(cfg->sbi_dom, mr) { + if (mr->order < sizeof(unsigned long) * 8) + continue; + + if (!(mr->flags & SBI_DOMAIN_MEMREGION_SU_ACCESS_MASK)) + continue; + + base_xwr = mpt_xwr_from_memregion(mr, cfg->xwr); + rc = sch->map_full_range(dom, base_xwr); + if (rc) + goto fail; + + if (dom->nregions < SBI_MPT_MAX_REGIONS_DOMAIN) { + struct sbi_mpt_region *r = &dom->regions[dom->nregions++]; + r->pa = 0; + r->size = ~0UL & ~SBI_MPT_PAGE_MASK; + r->xwr = base_xwr; + r->locked = false; + r->shared = false; + } + + break; + } + + /* + * Another pass to map every other region which has its own + * permissions derived from the memregion's S/U flags. + */ + sbi_domain_for_each_memregion(cfg->sbi_dom, mr) { + if (mr->order >= sizeof(unsigned long) * 8) + continue; + + rc = mpt_map_range_perm(dom, sch, mr->base, BIT(mr->order), + mpt_xwr_from_memregion(mr, cfg->xwr), false); + if (rc) + goto fail; + } + } + + /* Map explicit regions which are not part of any SBI domain */ + for (i = 0; i < cfg->nregions; i++) { + rc = mpt_map_range_perm(dom, sch, cfg->regions[i].pa, cfg->regions[i].size, + cfg->regions[i].xwr, false); + if (rc) + goto fail; + } + + /* + * Map firmware region with no permission with and mark as locked + */ + if (cfg->fw_protect) { + rc = mpt_map_range_perm(dom, sch, ctrl->fw_pa, ctrl->fw_size, + SBI_MPT_PERM_NONE, true); + if (rc) { + sbi_printf("sbi_mpt: cannot protect firmware" + " rc=%d\n", rc); + goto fail; + } + } + + ctrl->ndomain++; + if (out_sdid) + *out_sdid = sdid; + + sbi_mpt_fence_sdid(sdid); + + /* To be removed */ + sbi_printf("sbi_mpt: SDID %u root=0x%lx mode=%s dom=%s\n", + sdid, root_pa, sch->name, cfg->sbi_dom ? cfg->sbi_dom->name : "(none)"); + return 0; + +fail: + bitmap_set(ctrl->sdid_bitmap, (int)sdid, 1); + dom->valid = false; + return rc; +} + +/* + * sbi_mpt_domain_add_region(): Map a region in a MPT table + */ +int sbi_mpt_domain_add_region(u32 sdid, unsigned long pa, + unsigned long size, u8 xwr) +{ + struct sbi_mpt_domain *dom; + struct sbi_mpt_region *r; + int rc; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + if (!size || (pa & SBI_MPT_PAGE_MASK) || (size & SBI_MPT_PAGE_MASK)) + return SBI_EINVAL; + + dom = sbi_mpt_domain_get(sdid); + if (!dom) + return SBI_EINVAL; + + if (region_overlaps_locked(dom, pa, size)) { + sbi_printf("sbi_mpt: 0x%lx+0x%lx overlaps locked" + " (SDID %u)\n", pa, size, sdid); + return SBI_EINVAL; + } + + if (!dom->mode->pa_in_range(pa, size)) { + /* + * A region that pa_in_range() rejects falls into one of two + * different cases - PA which is out of range from the mappable + * range of that mode and second is PA is in range but the end + * (PA + SIZE) is crossing the mappable range. + * + * Distinguish the two by probing pa_in_range() for just + * the base page and if even the base is out of range it is + * the first case and if the base is in range but the whole + * region is not, it is case later. + */ + if (dom->mode->pa_in_range(pa, SBI_MPT_PAGE_SIZE)) + sbi_printf("sbi_mpt: 0x%lx+0x%lx crossover mode '%s'" + " ceiling (SDID %u) — high part would alias;" + " rejected\n", + pa, size, dom->mode->name, sdid); + else + sbi_printf("sbi_mpt: 0x%lx+0x%lx out of range" + " (SDID %u)\n", pa, size, sdid); + return SBI_EINVAL; + } + + spin_lock(&dom->lock); + rc = dom->mode->map_range(dom, pa, size, xwr); + spin_unlock(&dom->lock); + + if (!rc) { + if (dom->nregions < SBI_MPT_MAX_REGIONS_DOMAIN) { + r = &dom->regions[dom->nregions++]; + r->pa = pa; + r->size = size; + r->xwr = xwr; + r->locked = false; + r->shared = false; + } + + sbi_mpt_fence_sdid(sdid); + } + + return rc; +} + +/* + * sbi_mpt_domain_remove_region(): Unmap a region from the MPT table + */ +int sbi_mpt_domain_remove_region(u32 sdid, unsigned long pa, + unsigned long size) +{ + int rc; + unsigned int i; + struct sbi_mpt_domain *dom; + struct sbi_mpt_region *r; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + + if (!size || (pa & SBI_MPT_PAGE_MASK) || (size & SBI_MPT_PAGE_MASK)) + return SBI_EINVAL; + + dom = sbi_mpt_domain_get(sdid); + if (!dom) + return SBI_EINVAL; + + for (i = 0; i < dom->nregions; i++) { + r = &dom->regions[i]; + + if (r->locked && region_contained(pa, size, r->pa, r->size)) + return SBI_EDENIED; + /* + * Shared regions must be removed via sbi_mpt_unshare_region() + * because that function prevents one domain revoking a region + * that another domain still has mapped. + */ + if (r->shared && r->pa == pa && r->size == size) + return SBI_EDENIED; + } + + spin_lock(&dom->lock); + rc = dom->mode->map_range(dom, pa, size, SBI_MPT_PERM_NONE); + spin_unlock(&dom->lock); + + if (!rc) + sbi_mpt_fence_sdid(sdid); + + return rc; +} + +/* + * Shared memory regions + * + * sbi_mpt_share_region() — map a region into a second domain. + * + * Both domains MPT tables are updated independently — each has its + * own root/inner/leaf table tree. The same PA range can have different + * XWR permissions in different domains. + * + */ + +int sbi_mpt_share_region(u32 src_sdid, u32 dst_sdid, + unsigned long pa, unsigned long size, + u8 dst_xwr) +{ + unsigned int i; + int rc; + struct sbi_mpt_domain *src_dom, *dst_dom; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + + if (!size || (pa & SBI_MPT_PAGE_MASK) || (size & SBI_MPT_PAGE_MASK)) + return SBI_EINVAL; + + if (src_sdid == dst_sdid) + return SBI_EINVAL; + + src_dom = sbi_mpt_domain_get(src_sdid); + dst_dom = sbi_mpt_domain_get(dst_sdid); + if (!src_dom || !dst_dom) + return SBI_EINVAL; + + if (region_overlaps_locked(dst_dom, pa, size)) { + sbi_printf("sbi_mpt: share 0x%lx+0x%lx overlaps locked region in SDID %u\n", + pa, size, dst_sdid); + return SBI_EINVAL; + } + + /* Reject reserved xwr permissions: 0b010 and 0b110 (W without R) */ + if ((dst_xwr & SBI_MPT_PERM_W) && !(dst_xwr & SBI_MPT_PERM_R)) { + sbi_printf("sbi_mpt: share 0x%lx+0x%lx: W without R is a reserved XWR encoding\n", + pa, size); + return SBI_EINVAL; + } + + if (dst_dom->nregions >= SBI_MPT_MAX_REGIONS_DOMAIN) { + sbi_printf("sbi_mpt: share: SDID %u region table full\n", dst_sdid); + return SBI_ENOMEM; + } + + spin_lock(&dst_dom->lock); + rc = dst_dom->mode->map_range(dst_dom, pa, size, dst_xwr); + spin_unlock(&dst_dom->lock); + if (rc) + return rc; + + dst_dom->regions[dst_dom->nregions].pa = pa; + dst_dom->regions[dst_dom->nregions].size = size; + dst_dom->regions[dst_dom->nregions].xwr = dst_xwr; + dst_dom->regions[dst_dom->nregions].locked = false; + dst_dom->regions[dst_dom->nregions].shared = true; + dst_dom->nregions++; + + for (i = 0; i < src_dom->nregions; i++) { + if (src_dom->regions[i].pa == pa && + src_dom->regions[i].size == size) { + src_dom->regions[i].shared = true; + break; + } + } + + sbi_mpt_fence_sdid(src_sdid); + sbi_mpt_fence_sdid(dst_sdid); + + sbi_printf("sbi_mpt: shared 0x%lx+0x%lx SDID %u → SDID %u xwr=%u\n", + pa, size, src_sdid, dst_sdid, dst_xwr); + + return 0; +} + +/* + * sbi_mpt_unshare_region() — Unmap a shared region from a domain. + * + * The other domain remains unaffected just the shared flag gets removed + * for that region from both the domains. + */ +int sbi_mpt_unshare_region(u32 sdid, unsigned long pa, unsigned long size) +{ + u32 s; + int rc; + unsigned int i, j; + struct sbi_mpt_domain *dom, *peer; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + + if (!size || (pa & SBI_MPT_PAGE_MASK) || (size & SBI_MPT_PAGE_MASK)) + return SBI_EINVAL; + + dom = sbi_mpt_domain_get(sdid); + if (!dom) + return SBI_EINVAL; + + for (i = 0; i < dom->nregions; i++) { + if (dom->regions[i].pa == pa && + dom->regions[i].size == size && + dom->regions[i].shared) + break; + } + if (i == dom->nregions) { + sbi_printf("sbi_mpt: unshare 0x%lx+0x%lx not found in SDID %u\n", + pa, size, sdid); + return SBI_EINVAL; + } + + spin_lock(&dom->lock); + rc = dom->mode->map_range(dom, pa, size, SBI_MPT_PERM_NONE); + spin_unlock(&dom->lock); + + if (rc) + return rc; + + dom->nregions -= 1; + dom->regions[i] = dom->regions[dom->nregions]; + + /* Clear the shared flag on any other domain that has this PA region shared.*/ + for (s = 0; s < mpt_ctrl.max_domains; s++) { + peer = sbi_mpt_domain_get(s); + if (!peer || peer == dom) + continue; + + for (j = 0; j < peer->nregions; j++) + if (peer->regions[j].pa == pa && peer->regions[j].size == size) + peer->regions[j].shared = false; + } + + sbi_mpt_fence_sdid(sdid); + + return 0; +} + +/* + * Disable SMMPT on hart + */ +void sbi_mpt_hart_deactivate(void) +{ + /* + * mmpt is inactive in M-mode per spec, so no CSR write + * is needed. + */ +} + +/* + * Enable SMMPT on hart on a supervisor domain(sdid) + */ +int sbi_mpt_hart_activate(u32 sdid) +{ + u32 *p; + unsigned long ppn; + struct sbi_mpt_domain *dom; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + + dom = sbi_mpt_domain_get(sdid); + if (!dom) + return SBI_EINVAL; + + ppn = dom->root_pa >> SBI_MPT_PAGE_SHIFT; + csr_write(SBI_CSR_MMPT, dom->mode->encode_mmpt(ppn, sdid)); + sbi_mpt_fence_sdid(sdid); + + if (mpt_ctrl.sdid_offset) { + p = sbi_scratch_offset_ptr(sbi_scratch_thishart_ptr(), mpt_ctrl.sdid_offset); + *p = sdid; + } + + return 0; +} + +/* + * Activate SMMPT for all harts which are associated with an linked + * SBI domain in an supervisor domain. + */ +int sbi_mpt_hart_activate_for_domain(struct sbi_domain *sbi_dom) +{ + u32 s; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + if (!sbi_dom) + return SBI_EINVAL; + + /* + * Search all supervisor domains for associated sbi_dom and + * activate harts for that supervisor domain + */ + for (s = 0; s < mpt_ctrl.max_domains; s++) { + if (mpt_ctrl.domains[s].valid && mpt_ctrl.domains[s].sbi_dom == sbi_dom) + return sbi_mpt_hart_activate(s); + } + + return 0; +} + +/* + * sbi_mpt_sdid_free(): Returns SDID to the free pool after domain destroy. + */ +void sbi_mpt_sdid_free(u32 sdid) +{ + if (!mpt_ctrl.ready || sdid >= mpt_ctrl.max_domains) + return; + bitmap_set(mpt_ctrl.sdid_bitmap, (int)sdid, 1); + mpt_ctrl.domains[sdid].valid = false; +} + +/* + * sbi_mpt_query_access(): Returns the XWR for a PA in supervisor domain(sdid). + */ +int sbi_mpt_query_access(u32 sdid, unsigned long pa, u8 *out_xwr) +{ + struct sbi_mpt_domain *dom; + + if (!mpt_ctrl.ready) + return SBI_ENODEV; + + if (!out_xwr) + return SBI_EINVAL; + + dom = sbi_mpt_domain_get(sdid); + if (!dom) + return SBI_EINVAL; + + if (!dom->mode->pa_in_range(pa, SBI_MPT_PAGE_SIZE)) { + *out_xwr = SBI_MPT_PERM_NONE; + return 0; + } + + spin_lock(&dom->lock); + *out_xwr = dom->mode->get_xwr(dom, pa); + spin_unlock(&dom->lock); + + return 0; +} + +/* + * sbi_mpt_init(): Initialize MPT structures + */ +int sbi_mpt_init(void) +{ + int rc; + u32 *p; + u32 mode_val; + unsigned long fw_end; + struct sbi_scratch *s; + struct sbi_scratch *scratch = sbi_scratch_thishart_ptr(); + unsigned long fw_pa = scratch->fw_start; + unsigned long fw_size = scratch->fw_size; + + if (mpt_ctrl.ready) + return 0; + + if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMSDID)) { + sbi_printf("sbi_mpt: Smsdid extension absent\n"); + return SBI_ENODEV; + } + + if (!sbi_hart_has_extension(scratch, SBI_HART_EXT_SMMPT)) { + sbi_printf("sbi_mpt: Smmpt extension absent\n"); + return SBI_ENODEV; + } + + mpt_ctrl.mode = init_mpt_mode(&mode_val); + if (!mpt_ctrl.mode) { + sbi_printf("sbi_mpt: WARL probe found no mode\n"); + return SBI_ENODEV; + } + + /* + * SDIDLEN must be probed with a legal non-Bare MODE programmed, + * probe sdidlen must be done only after the probing the mode. + */ + mpt_ctrl.sdid_len = probe_sdidlen(mode_val); + if (!mpt_ctrl.sdid_len) { + sbi_printf("sbi_mpt: SDIDLEN=0 — SDID field not implemented\n"); + return SBI_ENODEV; + } + + mpt_ctrl.max_domains = 1UL << mpt_ctrl.sdid_len; + + /* + * Initialise SDID bitmap + * set bits 0..(max_domains-1) as 1 marking those SDIDs free. + */ + bitmap_fill(mpt_ctrl.sdid_bitmap, mpt_ctrl.max_domains); + + /* Round fw_pa and fw_size to page boundaries before storing. */ + fw_end = (fw_pa + fw_size + SBI_MPT_PAGE_SIZE - 1UL) & ~SBI_MPT_PAGE_MASK; + mpt_ctrl.fw_pa = fw_pa & ~SBI_MPT_PAGE_MASK; + mpt_ctrl.fw_size = fw_end - mpt_ctrl.fw_pa; + + + /* Per-hart SDID scratch slot — initialise all harts to INVALID */ + mpt_ctrl.sdid_offset = sbi_scratch_alloc_offset(sizeof(u32)); + if (!mpt_ctrl.sdid_offset) { + sbi_printf("sbi_mpt: scratch alloc for SDID failed\n"); + } + else { + sbi_for_each_hartindex(i) { + s = sbi_hartindex_to_scratch(i); + if (s) { + p = sbi_scratch_offset_ptr(s, mpt_ctrl.sdid_offset); + *p = SBI_MPT_SDID_INVALID; + } + } + } + + mpt_ctrl.ready = true; + + rc = sbi_hart_protection_register(&smmpt_protection); + if (rc) { + sbi_printf("sbi_mpt: hart protection register failed rc=%d\n", rc); + mpt_ctrl.ready = false; + return rc; + } + + return 0; +} + +/* + * Debug dump + */ +void sbi_mpt_dump(void) +{ + u32 s; + unsigned int r; + const struct sbi_mpt_domain *d; + const unsigned long catchall_size = ~0UL & ~SBI_MPT_PAGE_MASK; + + if (!mpt_ctrl.ready) { + sbi_printf("sbi_mpt: not initialised\n"); + return; + } + + sbi_printf("sbi_mpt: mode=%-10s SDIDLEN=%u max_domains=%u\n", mpt_ctrl.mode->name, + mpt_ctrl.sdid_len, mpt_ctrl.max_domains); + sbi_printf("root_size=0x%lx root_align=0x%lx\n", + mpt_ctrl.mode->root_table_size(), mpt_ctrl.mode->root_table_align()); + sbi_printf("fw 0x%016lx+0x%lx pool=global heap\n", + mpt_ctrl.fw_pa, mpt_ctrl.fw_size); + for (s = 0; s < mpt_ctrl.max_domains; s++) { + bool has_baseline = false; + u8 baseline_xwr = SBI_MPT_PERM_NONE; + + d = &mpt_ctrl.domains[s]; + if (!d->valid) + continue; + sbi_printf("SDID %2u root=0x%016lx regions=%u dom=%s\n", + d->sdid, d->root_pa, d->nregions, d->sbi_dom ? d->sbi_dom->name : "(none)"); + for (r = 0; r < d->nregions; r++) { + + if (d->regions[r].pa == 0 && + d->regions[r].size == catchall_size) { + has_baseline = true; + baseline_xwr = d->regions[r].xwr; + continue; + } + + sbi_printf("S-Domain%u Region%02u : 0x%016lx+0x%-16lx: xwr=%u (%c%c%c)%s%s\n", + d->sdid, r, d->regions[r].pa, + d->regions[r].size, + d->regions[r].xwr, + (d->regions[r].xwr & SBI_MPT_PERM_R) ? 'R' : '-', + (d->regions[r].xwr & SBI_MPT_PERM_W) ? 'W' : '-', + (d->regions[r].xwr & SBI_MPT_PERM_X) ? 'X' : '-', + d->regions[r].locked ? " [LOCKED]" : "", + d->regions[r].shared ? " [SHARED]" : ""); + } + + if (has_baseline) { + sbi_printf("S-Domain%u Default : %-37s: xwr=%u (%c%c%c) [baseline, overlay by regions above]\n", + d->sdid, "Rest of the address space", + baseline_xwr, + (baseline_xwr & SBI_MPT_PERM_R) ? 'R' : '-', + (baseline_xwr & SBI_MPT_PERM_W) ? 'W' : '-', + (baseline_xwr & SBI_MPT_PERM_X) ? 'X' : '-'); + } + else { + sbi_printf("S-Domain%u : xwr=0 (---) [no baseline — unmapped PAs denied]\n", + d->sdid); + } + } +} diff --git a/lib/sbi/sbi_mpt_mode.c b/lib/sbi/sbi_mpt_mode.c new file mode 100644 index 00000000..6fed2ef5 --- /dev/null +++ b/lib/sbi/sbi_mpt_mode.c @@ -0,0 +1,997 @@ +/* SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Authors: Rahul Pathak <[email protected]> + */ + +#include <sbi/sbi_mpt.h> +#include <sbi/sbi_error.h> +#include <sbi/sbi_string.h> +#include <sbi/sbi_console.h> + +#if __riscv_xlen == 64 + +/* + * Smmpt43 — 43-bit supervisor physical address (SPA) + * +---------+---------+---------+---------+--------+ + * | pn[2] | pn[1] | pn[0] | pi | offset | + * | [42:34] | [33:25] | [24:16] | [15:12] | [11:0] | + * | 9b | 9b | 9b | 4b | 12b | + * +---------+---------+---------+---------+--------+ + * + * Smmpt52 — 52-bit supervisor physical address (SPA) + * +---------+---------+---------+---------+---------+--------+ + * | pn[3] | pn[2] | pn[1] | pn[0] | pi | offset | + * | [51:43] | [42:34] | [33:25] | [24:16] | [15:12] | [11:0] | + * | 9b | 9b | 9b | 9b | 4b | 12b | + * +---------+---------+---------+---------+---------+--------+ + * + * Smmpt64 — 64-bit supervisor physical address (SPA) + * +---------+---------+---------+---------+---------+---------+--------+ + * | pn[4] | pn[3] | pn[2] | pn[1] | pn[0] | pi | offset | + * | [63:52] | [51:43] | [42:34] | [33:25] | [24:16] | [15:12] | [11:0] | + * | 12b | 9b | 9b | 9b | 9b | 4b | 12b | + * +---------+---------+---------+---------+---------+---------+--------+ + * + */ + +/* Innter entries are MPTEs not in root MPT page */ +#define RV64_INNER_ENTRIES 512U +#define RV64_MPTE_SIZE 8UL +#define RV64_TABLE_SIZE (RV64_INNER_ENTRIES * RV64_MPTE_SIZE) + +/* Smmpt64 root: 4096 entries × 8B = 32KB, 32KB aligned */ +#define RV64_ROOT_SMMPT64_ENTRIES 4096U +#define RV64_ROOT_SMMPT64_SIZE (RV64_ROOT_SMMPT64_ENTRIES * RV64_MPTE_SIZE) + +#define RV64_NUMPGINRANGE 4U +#define RV64_PAGES_PER_MPTE (1U << RV64_NUMPGINRANGE) +#define RV64_PAGES_PER_MPTE_MASK (RV64_PAGES_PER_MPTE - 1U) +#define RV64_INNER_PN_BITS 9U + +/* Bytes covered by one complete leaf table = pages_per_leaf × PAGE_SIZE */ +#define RV64_LEAF_RANGE (RV64_PAGES_PER_MPTE * SBI_MPT_PAGE_SIZE) + +/* + * NAPOT G=4 for Smmpt43, Smmpt52 and Smmpt64 + * + * G=4 means 2^(G+1)=32 contiguous leaf mpte which form a NAPOT group. + * Each RV64 leaf MPTE covers 64KiB so one NAPOT group covers 2MiB. + * + * All other G values apart from 4 are reserved. + */ +#define RV64_NAPOT_G 4U +#define RV64_NAPOT_G_SHIFT 12U +#define RV64_NAPOT_MPTE_COUNT (1U << (RV64_NAPOT_G + 1)) +#define RV64_NAPOT_SIZE ((unsigned long)RV64_NAPOT_MPTE_COUNT * RV64_LEAF_RANGE) +#define RV64_NAPOT_PN0_ALIGN RV64_NAPOT_MPTE_COUNT + +/* + * Swith to disable NAPOT compaction of MPTE entries while writing. + */ +#define RV64_NAPOT_DISABLE 0 + +/* + * Supervisor Physucal address fields extraction + */ + +/* + * rv64_table_idx() — extract pn[level] from a SPA. + */ +static inline u32 rv64_table_idx(unsigned long pa, u32 level, + const struct sbi_mpt_mode *sch) +{ + u32 shift = 16 + level * 9; + u32 mask = (sch->mode_val == SBI_MMPT_MODE_SMMPT64 && level == 4) ? 0xFFF : 0x1FF; + + return ((pa >> shift) & mask); +} + +/* + * rv64_tuple_idx_shift() — XWR tuple index shift at level i. + */ +static inline u32 rv64_tuple_idx_shift(u32 level) +{ + return (SBI_MPT_PAGE_SHIFT + level * RV64_INNER_PN_BITS); +} + +static inline u32 rv64_tuple_idx(unsigned long pa, u32 level) +{ + return ((pa >> rv64_tuple_idx_shift(level)) & RV64_PAGES_PER_MPTE_MASK); +} + +/* + * rv64_mpte_range() — bytes covered by one MPTE at level L. + */ +static inline unsigned long rv64_mpte_range(u32 level) +{ + return RV64_LEAF_RANGE << (level * RV64_INNER_PN_BITS); +} + +/* + * RV64 MPTE Read/Write functions + */ + +static inline u64 rv64_read_mpte(unsigned long pa) +{ + return *(volatile u64 *)pa; +} + +static inline void rv64_write_mpte(unsigned long pa, u64 v) +{ + *(volatile u64 *)pa = v; +} + +static inline unsigned long rv64_mpte_pa(unsigned long table_pa, u32 idx) +{ + return (table_pa + idx * RV64_MPTE_SIZE); +} + +static inline unsigned long rv64_next_table_pa(u64 mpte) +{ + return (unsigned long)((u64)(mpte >> SBI_MPTE_PPN_SHIFT) << SBI_MPT_PAGE_SHIFT); +} + +/* + * Generate Napot Leaf MPTE + * + * Constructs one MPTE value used for all + * RV64_NAPOT_MPTE_COUNT entries in a NAPOT group. + * + * All MPTEs in the group are identical + */ +static inline u64 rv64_napot_leaf_mpte(u8 xwr) +{ + u64 mpte = SBI_MPTE_V | SBI_MPTE_L | SBI_MPTE_N; + + mpte = sbi_mpte_leaf_set_xwr(mpte, 0, xwr); + mpte |= ((u64)RV64_NAPOT_G << RV64_NAPOT_G_SHIFT); + + return mpte; +} + +/* + * rv64_napot_leaf_xwr() — recover the uniform XWR from a NAPOT leaf. + */ +static inline u8 rv64_napot_leaf_xwr(u64 mpte) +{ + return ((mpte >> SBI_MPTE_XWR_BASE) & SBI_MPTE_XWR_MASK); +} + +/* + * rv64_napot_demote() — Convert a Napot MPTE group into RV64_NAPOT_MPTE_COUNT + * normal(Non-Napot) MPTE carrying the SAME permission. + * + * leaf_mpte_pa is any leaf mpte pa from the Napot group + */ +static void rv64_napot_demote(struct sbi_mpt_domain *dom, + unsigned long leaf_mpte_pa, unsigned long pa) +{ + u8 xwr; + u32 pn0, pg, i; + u64 leaf; + unsigned long grp_base; + + xwr = rv64_napot_leaf_xwr(rv64_read_mpte(leaf_mpte_pa)); + pn0 = rv64_table_idx(pa, 0, dom->mode); + grp_base = leaf_mpte_pa - (pn0 & (RV64_NAPOT_MPTE_COUNT - 1)) * RV64_MPTE_SIZE; + leaf = SBI_MPTE_V | SBI_MPTE_L; + + for (pg = 0; pg < RV64_PAGES_PER_MPTE; pg++) + leaf = sbi_mpte_leaf_set_xwr(leaf, pg, xwr); + + for (i = 0; i < RV64_NAPOT_MPTE_COUNT; i++) + rv64_write_mpte(grp_base + i * RV64_MPTE_SIZE, leaf); +} + +/* + * MPT table best-level selection + * + * Returns the highest level at which the range [pa, pa+size] can be + * covered by a single leaf MPTE. Level 0 is always valid because thats + * the last resort. + */ +static u32 rv64_best_level(unsigned long pa, unsigned long size, + u32 top_level) +{ + u32 lvl; + unsigned long range; + + for (lvl = top_level; lvl >= 1; lvl--) { + range = rv64_mpte_range(lvl); + + if (size >= range && (pa & (range - 1)) == 0) + return lvl; + } + return 0; +} + +/* + * Generic N-level walk with lazy table allocation + */ +static unsigned long rv64_split_leaf(unsigned long parent_ep) +{ + u64 parent = rv64_read_mpte(parent_ep); + unsigned long sub; + u32 j, pg, sh; + u8 xwr; + u64 child; + + sub = sbi_mpt_pool_alloc(RV64_TABLE_SIZE, SBI_MPT_PAGE_SIZE); + if (!sub) + return 0; + + for (j = 0; j < RV64_INNER_ENTRIES; j++) { + sh = sbi_mpte_xwr_shift(j >> 5); + xwr = (((unsigned long)parent >> sh) & SBI_MPTE_XWR_MASK); + child = SBI_MPTE_V | SBI_MPTE_L; /* N=0 uniform leaf */ + + for (pg = 0; pg < RV64_PAGES_PER_MPTE; pg++) + child = sbi_mpte_leaf_set_xwr(child, pg, xwr); + + rv64_write_mpte(sub + j * RV64_MPTE_SIZE, child); + } + + /* parent leaf -> non-leaf pointer to MPT sub table. */ + rv64_write_mpte(parent_ep, sbi_mpte_nonleaf(sub)); + + return sub; +} + +/* + * Walk a MPT table and return MPTE PA and its suitable level + */ +static unsigned long rv64_walk_alloc(struct sbi_mpt_domain *dom, + unsigned long pa, + unsigned long size, + u32 *out_level, + u32 top_level) +{ + u32 lvl, idx; + u64 mpte; + unsigned long ep, sub, new_pa; + const struct sbi_mpt_mode *sch = dom->mode; + unsigned long table_pa = dom->root_pa; + u32 best_lvl = rv64_best_level(pa, size, top_level); + + + for (lvl = top_level; lvl >= 1; lvl--) { + idx = rv64_table_idx(pa, lvl, sch); + ep = rv64_mpte_pa(table_pa, idx); + mpte = rv64_read_mpte(ep); + + if (mpte & SBI_MPTE_L) { + if (lvl > best_lvl) { + sub = rv64_split_leaf(ep); + if (!sub) + return 0; + + table_pa = sub; + continue; + } + + *out_level = lvl; + return ep; + } + + if (!(mpte & SBI_MPTE_V)) { + if (lvl <= best_lvl) { + *out_level = lvl; + return ep; + } + /* Allocate inner table (always 4KiB, all levels, all modes) */ + new_pa = sbi_mpt_pool_alloc(RV64_TABLE_SIZE, + SBI_MPT_PAGE_SIZE); + if (!new_pa) + return 0; + + rv64_write_mpte(ep, (u64)sbi_mpte_nonleaf(new_pa)); + table_pa = new_pa; + } + else { + table_pa = rv64_next_table_pa(mpte); + } + } + + *out_level = 0; + + return rv64_mpte_pa(table_pa, rv64_table_idx(pa, 0, sch)); +} + +/* + * Generic map_range — shared by Smmpt43, Smmpt52, Smmpt64 + * + * Maps a range [pa, pa+size] in the MPT table + */ +static int __rv64_map_range(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, + u8 xwr, u32 top_level) +{ + unsigned long cur = pa; + unsigned long end = pa + size; + unsigned long leaf_mpte_pa, mpte_range, mpte_base, mpte_end, batch_end; + u32 level, pn0, i, pg_first, pg_last, pg; + u64 napot, mpte; + + while (cur < end) { + /* + * Check if the region qualifies for NAPOT range and the + * region is contained + */ + if ((cur & (RV64_NAPOT_SIZE - 1)) == 0 && + cur + RV64_NAPOT_SIZE > cur && + cur + RV64_NAPOT_SIZE <= end) { + leaf_mpte_pa = rv64_walk_alloc(dom, cur, end - cur, &level, top_level); + if (!leaf_mpte_pa) + return SBI_ENOMEM; + + pn0 = rv64_table_idx(cur, 0, dom->mode); + + if (!RV64_NAPOT_DISABLE && level == 0 && (pn0 & (RV64_NAPOT_PN0_ALIGN - 1)) == 0) { + napot = rv64_napot_leaf_mpte(xwr); + + for (i = 0; i < RV64_NAPOT_MPTE_COUNT; i++) + rv64_write_mpte(leaf_mpte_pa + i * RV64_MPTE_SIZE, napot); + + cur += RV64_NAPOT_SIZE; + continue; + } + } + else { + leaf_mpte_pa = rv64_walk_alloc(dom, cur, end - cur, &level, top_level); + if (!leaf_mpte_pa) + return SBI_ENOMEM; + } + + /* Normal path */ + mpte_range = rv64_mpte_range(level); + pg_first = rv64_tuple_idx(cur, level); + mpte_base = cur & ~(mpte_range - 1); + + mpte_end = mpte_base + mpte_range; + if (mpte_end < mpte_base) + mpte_end = ~0UL; + + batch_end = (end < mpte_end) ? end : mpte_end; + pg_last = rv64_tuple_idx(batch_end - SBI_MPT_PAGE_SIZE, level); + + /* + * If this level-0 MPTE is a Napot member, expand the + * whole naturally-aligned group to plain leaves and write same + * permissions xwr + */ + mpte = rv64_read_mpte(leaf_mpte_pa); + if (level == 0 && (mpte & (u64)SBI_MPTE_N)) { + rv64_napot_demote(dom, leaf_mpte_pa, cur); + sbi_mpt_fence_sdid(dom->sdid); + mpte = rv64_read_mpte(leaf_mpte_pa); + } + + if (!(mpte & (u64)(SBI_MPTE_V | SBI_MPTE_L))) + mpte = (u64)(SBI_MPTE_V | SBI_MPTE_L); + + for (pg = pg_first; pg <= pg_last; pg++) + mpte = sbi_mpte_leaf_set_xwr(mpte, pg, xwr); + + rv64_write_mpte(leaf_mpte_pa, mpte); + + cur = batch_end; + } + + return 0; +} + +/* + * Per-mode mmpt encoding + */ + +static unsigned long smmpt43_encode_mmpt(unsigned long ppn, u32 sdid) +{ + return sbi_mmpt_encode(SBI_MMPT_MODE_SMMPT43, sdid, ppn); +} + +static unsigned long smmpt52_encode_mmpt(unsigned long ppn, u32 sdid) +{ + return sbi_mmpt_encode(SBI_MMPT_MODE_SMMPT52, sdid, ppn); +} + +static unsigned long smmpt64_encode_mmpt(unsigned long ppn, u32 sdid) +{ + return sbi_mmpt_encode(SBI_MMPT_MODE_SMMPT64, sdid, ppn); +} + +static unsigned long smmpt43_root_table_size(void) +{ + return RV64_TABLE_SIZE; +} + +static unsigned long smmpt43_root_table_align(void) +{ + return SBI_MPT_PAGE_SIZE; +} + +static bool smmpt43_pa_in_range(unsigned long pa, unsigned long size) +{ + unsigned long pa_max = 1UL << 43; + + return (pa < pa_max && size <= pa_max - pa); +} + +static unsigned long smmpt52_root_table_size(void) +{ + return RV64_TABLE_SIZE; +} + +static unsigned long smmpt52_root_table_align(void) +{ + return SBI_MPT_PAGE_SIZE; +} + +static bool smmpt52_pa_in_range(unsigned long pa, unsigned long size) +{ + unsigned long pa_max = 1UL << 52; + + return (pa < pa_max && size <= pa_max - pa); +} + +static unsigned long smmpt64_root_table_size(void) +{ + return RV64_ROOT_SMMPT64_SIZE; +} + +static unsigned long smmpt64_root_table_align(void) +{ + return RV64_ROOT_SMMPT64_SIZE; +} + +static bool smmpt64_pa_in_range(unsigned long pa, + unsigned long size) +{ + /* + * Smmpt64 covers the full 64-bit PA space. + */ + return true; +} + +/* + * Per-mode map_range wrappers + * + * Smmpt43: + * Root level 2, 9-bit, 512 entries, 4KiB root MPT size + * + * Smmpt52: + * Root level 3, 9-bit, 512 entries, 4KiB root MPT size + * + * Smmpt64: + * Root level 4, 12-bit, 4096 entries, 32KiB root MPT size + */ + +static int smmpt43_map_range(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, u8 xwr) +{ + return __rv64_map_range(dom, pa, size, xwr, 2); +} + +static int smmpt52_map_range(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, u8 xwr) +{ + return __rv64_map_range(dom, pa, size, xwr, 3); +} + +static int smmpt64_map_range(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, u8 xwr) +{ + return __rv64_map_range(dom, pa, size, xwr, 4); +} + +/* + * __rv64_map_full_range() — Write every MPTE at Root MPT + * + * Writes xwr into every root table MPTE directly as a leaf + * superpage covering the mode entire addressable PA range. + * No intermidiate MPT tables allocated just all leaf MPTEs at + * root level. + */ + +static int __rv64_map_full_range(struct sbi_mpt_domain *dom, u8 xwr, + unsigned long root_entries) +{ + u32 pg; + unsigned long i; + u64 mpte = SBI_MPTE_V | SBI_MPTE_L; + + for (pg = 0; pg < RV64_PAGES_PER_MPTE; pg++) + mpte = sbi_mpte_leaf_set_xwr(mpte, pg, xwr); + + for (i = 0; i < root_entries; i++) + rv64_write_mpte(rv64_mpte_pa(dom->root_pa, i), mpte); + + return 0; +} + +static int smmpt43_map_full_range(struct sbi_mpt_domain *dom, u8 xwr) +{ + return __rv64_map_full_range(dom, xwr, RV64_INNER_ENTRIES); +} + +static int smmpt52_map_full_range(struct sbi_mpt_domain *dom, u8 xwr) +{ + return __rv64_map_full_range(dom, xwr, RV64_INNER_ENTRIES); +} + +static int smmpt64_map_full_range(struct sbi_mpt_domain *dom, u8 xwr) +{ + return __rv64_map_full_range(dom, xwr, RV64_ROOT_SMMPT64_ENTRIES); +} + +/* + * Get the access permissions(xwr) of a PA + */ +static u8 __rv64_get_xwr(struct sbi_mpt_domain *dom, unsigned long pa, + u32 top_level) +{ + const struct sbi_mpt_mode *sch = dom->mode; + unsigned long table_pa = dom->root_pa; + u32 level, idx, pi; + u64 mpte; + + for (level = top_level; ; level--) { + idx = rv64_table_idx(pa, level, sch); + mpte = rv64_read_mpte(rv64_mpte_pa(table_pa, idx)); + + if (!(mpte & SBI_MPTE_V)) + return SBI_MPT_PERM_NONE; + + if (!(mpte & SBI_MPTE_L) && (mpte & SBI_MPTE_N)) + return SBI_MPT_PERM_NONE; + + if (mpte & SBI_MPTE_L) { + if (mpte & SBI_MPTE_N) + return rv64_napot_leaf_xwr(mpte); + + pi = rv64_tuple_idx(pa, level); + + return (mpte >> sbi_mpte_xwr_shift(pi)) & SBI_MPTE_XWR_MASK; + } + + if (level == 0) + return SBI_MPT_PERM_NONE; + + table_pa = rv64_next_table_pa(mpte); + } + + return SBI_MPT_PERM_NONE; +} + +static u8 smmpt43_get_xwr(struct sbi_mpt_domain *dom, unsigned long pa) +{ + return __rv64_get_xwr(dom, pa, 2); +} + +static u8 smmpt52_get_xwr(struct sbi_mpt_domain *dom, unsigned long pa) +{ + return __rv64_get_xwr(dom, pa, 3); +} + +static u8 smmpt64_get_xwr(struct sbi_mpt_domain *dom, unsigned long pa) +{ + return __rv64_get_xwr(dom, pa, 4); +} + +struct sbi_mpt_mode smmpt43_mode = { + .name = "Smmpt43", + .mode_val = SBI_MMPT_MODE_SMMPT43, + .encode_mmpt = smmpt43_encode_mmpt, + .map_range = smmpt43_map_range, + .root_table_size = smmpt43_root_table_size, + .root_table_align = smmpt43_root_table_align, + .pa_in_range = smmpt43_pa_in_range, + .map_full_range = smmpt43_map_full_range, + .get_xwr = smmpt43_get_xwr, +}; + +struct sbi_mpt_mode smmpt52_mode = { + .name = "Smmpt52", + .mode_val = SBI_MMPT_MODE_SMMPT52, + .encode_mmpt = smmpt52_encode_mmpt, + .map_range = smmpt52_map_range, + .root_table_size = smmpt52_root_table_size, + .root_table_align = smmpt52_root_table_align, + .pa_in_range = smmpt52_pa_in_range, + .map_full_range = smmpt52_map_full_range, + .get_xwr = smmpt52_get_xwr, +}; + +struct sbi_mpt_mode smmpt64_mode = { + .name = "Smmpt64", + .mode_val = SBI_MMPT_MODE_SMMPT64, + .encode_mmpt = smmpt64_encode_mmpt, + .map_range = smmpt64_map_range, + .root_table_size = smmpt64_root_table_size, + .root_table_align = smmpt64_root_table_align, + .pa_in_range = smmpt64_pa_in_range, + .map_full_range = smmpt64_map_full_range, + .get_xwr = smmpt64_get_xwr, +}; + +#else /* __riscv_xlen == 64 */ + +/* + * Smmpt34 (RV32) — 34-bit supervisor physical address (SPA) + * + * +---------+---------+---------+--------+ + * | pn[1] | pn[0] | pi | offset | + * | [33:25] | [24:15] | [14:12] | [11:0] | + * | 9b | 10b | 3b | 12b | + * +---------+---------+---------+--------+ + * + */ + +#define RV32_MPTE_SIZE 4UL +#define RV32_ROOT_ENTRIES 512U +#define RV32_LEAF_ENTRIES 1024U +#define RV32_ROOT_SIZE (RV32_ROOT_ENTRIES * RV32_MPTE_SIZE) +#define RV32_LEAF_SIZE (RV32_LEAF_ENTRIES * RV32_MPTE_SIZE) + +/* pn field widths */ +#define RV32_PN1_BITS 9U +#define RV32_PN0_BITS 10U + +#define RV32_NUMPGINRANGE 3U +#define RV32_PAGES_PER_MPTE (1U << RV32_NUMPGINRANGE) +#define RV32_PAGES_PER_MPTE_MASK (RV32_PAGES_PER_MPTE - 1) + +/* Range covered by one MPTE at each level */ +#define RV32_LEAF_RANGE (RV32_PAGES_PER_MPTE * SBI_MPT_PAGE_SIZE) +#define RV32_ROOT_RANGE (RV32_LEAF_RANGE << RV32_PN0_BITS) + +/* NAPOT constants (G=6) */ +#define RV32_NAPOT_G 6U +#define RV32_NAPOT_G_SHIFT 12U +#define RV32_NAPOT_COUNT (1U << (RV32_NAPOT_G + 1)) +#define RV32_NAPOT_SIZE (RV32_NAPOT_COUNT * RV32_LEAF_RANGE) +#define RV32_NAPOT_PN0_ALIGN RV32_NAPOT_COUNT + +/* + * Swith to disable NAPOT compaction of MPTE entries while writing. + */ +#define RV32_NAPOT_DISABLE 0 + +/* + * rv32_pn1_table_idx() — extract pn[1] from a SPA. + */ +static inline u32 rv32_pn1_table_idx(unsigned long pa) +{ + return ((pa >> 25) & 0x1FFU); +} + +/* + * rv32_pn0_table_idx() — extract pn[0] from a SPA. + */ +static inline u32 rv32_pn0_table_idx(unsigned long pa) +{ + return ((pa >> 15) & 0x3FFU); +} + +static inline u32 rv32_tuple_idx_shift(u32 level) +{ + return (level == 0) ? SBI_MPT_PAGE_SHIFT : SBI_MPT_PAGE_SHIFT + RV32_PN0_BITS; +} + +static inline u32 rv32_tuple_idx(unsigned long pa, u32 level) +{ + return ((pa >> rv32_tuple_idx_shift(level)) & RV32_PAGES_PER_MPTE_MASK); +} + +/* + * Bytes covered by one MPTE at the given level + */ +static inline unsigned long rv32_mpte_range(u32 level) +{ + return (level == 0) ? RV32_LEAF_RANGE : RV32_ROOT_RANGE; +} + +/* + * RV32 MPTE Read/Write functions + */ +static inline u32 rv32_read_mpte(unsigned long pa) +{ + return *(volatile u32 *)pa; +} + +static inline void rv32_write_mpte(unsigned long pa, u32 v) +{ + *(volatile u32 *)pa = v; +} + +static inline unsigned long rv32_mpte_pa(unsigned long table_pa, u32 idx) +{ + return (table_pa + idx * RV32_MPTE_SIZE); +} + +static inline unsigned long rv32_next_table_pa(u32 mpte) +{ + return ((mpte >> SBI_MPTE_PPN_SHIFT) << SBI_MPT_PAGE_SHIFT); +} + +/* + * Generate Napot Leaf MPTE + * + * Constructs one MPTE value used for all RV32_NAPOT_COUNT entries + * in a NAPOT group. + * + * All MPTEs in the group are identical + */ +static inline u32 rv32_napot_leaf(u8 xwr) +{ + u32 mpte = SBI_MPTE_V | SBI_MPTE_L | SBI_MPTE_N; + + mpte = sbi_mpte_leaf_set_xwr(mpte, 0, xwr); + mpte |= ((u32)RV32_NAPOT_G << RV32_NAPOT_G_SHIFT); + + return mpte; +} + +/* + * Get the xwr from a Napot leaf. + */ +static inline u8 rv32_napot_xwr(u32 mpte) +{ + return ((mpte >> SBI_MPTE_XWR_BASE) & SBI_MPTE_XWR_MASK); +} + +/* + * rv32_napot_demote() — Convert a Napot MPTE group into RV64_NAPOT_MPTE_COUNT + * normal(Non-Napot) MPTE carrying the SAME permission. + * + * leaf_mpte_pa is any of the group MPTE PA + */ +static void rv32_napot_demote(struct sbi_mpt_domain *dom, + unsigned long leaf_mpte_pa, unsigned long pa) +{ + u32 pg, leaf, i; + unsigned long grp_base; + u8 xwr = rv32_napot_xwr(rv32_read_mpte(leaf_mpte_pa)); + u32 pn0 = rv32_pn0_table_idx(pa); + + grp_base = leaf_mpte_pa - (pn0 & (RV32_NAPOT_COUNT - 1)) * RV32_MPTE_SIZE; + + leaf = (SBI_MPTE_V | SBI_MPTE_L); /* N=0 */ + + for (pg = 0; pg < RV32_PAGES_PER_MPTE; pg++) + leaf = sbi_mpte_leaf_set_xwr(leaf, pg, xwr); + + for (i = 0; i < RV32_NAPOT_COUNT; i++) + rv32_write_mpte(grp_base + i * RV32_MPTE_SIZE, leaf); +} + +static inline u32 rv32_best_level(unsigned long pa, unsigned long size) +{ + if (size >= RV32_ROOT_RANGE && (pa & (RV32_ROOT_RANGE - 1)) == 0) + return 1; + + return 0; +} + +/* + * MPT Walk for RV32 Smmpt43 + * + * Returns the PA of the MPTE covering pa + */ +static unsigned long rv32_walk_alloc(struct sbi_mpt_domain *dom, + unsigned long pa, + unsigned long size, + u32 *out_level) +{ + unsigned long root_ep = rv32_mpte_pa(dom->root_pa, rv32_pn1_table_idx(pa)); + u32 root_mpte = rv32_read_mpte(root_ep); + u32 best = rv32_best_level(pa, size); + unsigned long leaf_pa; + + if (root_mpte & (u32)SBI_MPTE_L) { + /* Existing root-level leaf */ + *out_level = 1; + return root_ep; + } + + if (!(root_mpte & (u32)SBI_MPTE_V) && best == 1) { + /* new entry, range fits at root level so skip leaf allocation */ + *out_level = 1; + return root_ep; + } + + if (!(root_mpte & (u32)SBI_MPTE_V)) { + /* Allocate leaf table from global heap */ + leaf_pa = sbi_mpt_pool_alloc(RV32_LEAF_SIZE, + SBI_MPT_PAGE_SIZE); + if (!leaf_pa) + return 0; + + rv32_write_mpte(root_ep, sbi_mpte_nonleaf(leaf_pa)); + *out_level = 0; + + return rv32_mpte_pa(leaf_pa, rv32_pn0_table_idx(pa)); + } + + /* Valid non-leaf: follow PPN to existing leaf table */ + *out_level = 0; + + return rv32_mpte_pa(rv32_next_table_pa(rv32_read_mpte(root_ep)), rv32_pn0_table_idx(pa)); +} + +/* + * smmpt34_map_range(): Maps a range [pa, pa+size] in the MPT table + */ + +static int smmpt34_map_range(struct sbi_mpt_domain *dom, + unsigned long pa, unsigned long size, u8 xwr) +{ + unsigned long cur = pa; + unsigned long end = pa + size; + unsigned long leaf_mpte_pa; + unsigned long mpte_range, mpte_base, mpte_end, batch_end; + u32 level, pg_first, pg_last, pg, i; + u32 mpte, napot; + + while (cur < end) { + /* + * Check if the region qualifies for NAPOT range and the + * region is contained + */ + if ((cur & (RV32_NAPOT_SIZE - 1)) == 0 && + cur + RV32_NAPOT_SIZE > cur && + cur + RV32_NAPOT_SIZE <= end) { + leaf_mpte_pa = rv32_walk_alloc(dom, cur, end - cur, &level); + if (!leaf_mpte_pa) + return SBI_ENOMEM; + + if (!RV32_NAPOT_DISABLE && level == 0 && (rv32_pn0_table_idx(cur) & (RV32_NAPOT_PN0_ALIGN - 1)) == 0) { + napot = rv32_napot_leaf(xwr); + + for (i = 0; i < RV32_NAPOT_COUNT; i++) + rv32_write_mpte(leaf_mpte_pa + i * RV32_MPTE_SIZE, napot); + + cur += RV32_NAPOT_SIZE; + + continue; + } + } else { + leaf_mpte_pa = rv32_walk_alloc(dom, cur, end - cur, &level); + if (!leaf_mpte_pa) + return SBI_ENOMEM; + } + + /* Normal path */ + mpte_range = rv32_mpte_range(level); + pg_first = rv32_tuple_idx(cur, level); + mpte_base = cur & ~(mpte_range - 1UL); + mpte_end = mpte_base + mpte_range; + batch_end = (end < mpte_end) ? end : mpte_end; + pg_last = rv32_tuple_idx(batch_end - SBI_MPT_PAGE_SIZE, level); + + /* + * If this level-0 MPTE is a Napot member, expand the + * whole naturally-aligned group to plain leaves and write same + * permissions xwr + */ + mpte = rv32_read_mpte(leaf_mpte_pa); + if (level == 0 && (mpte & (u32)SBI_MPTE_N)) { + rv32_napot_demote(dom, leaf_mpte_pa, cur); + sbi_mpt_fence_sdid(dom->sdid); + mpte = rv32_read_mpte(leaf_mpte_pa); + } + if (!(mpte & (u32)(SBI_MPTE_V | SBI_MPTE_L))) + mpte = SBI_MPTE_V | SBI_MPTE_L; + + for (pg = pg_first; pg <= pg_last; pg++) + mpte = sbi_mpte_leaf_set_xwr(mpte, pg, xwr); + + rv32_write_mpte(leaf_mpte_pa, mpte); + cur = batch_end; + } + + return 0; +} + +/* + * mmpt encoding + */ +static unsigned long smmpt34_encode_mmpt(unsigned long ppn, u32 sdid) +{ + return sbi_mmpt_encode(SBI_MMPT_MODE_SMMPT34, sdid, ppn); +} + +static unsigned long smmpt34_root_table_size(void) +{ + return RV32_ROOT_SIZE; +} + +static unsigned long smmpt34_root_table_align(void) +{ + return SBI_MPT_PAGE_SIZE; +} + +static bool smmpt34_pa_in_range(unsigned long pa, unsigned long size) +{ + /* + * Return true since 1 << 34 wil be UB + * in RV32 case and anyways variables are limited by RV32 + * architectural width + */ + + return true; +} + +/* + * smmpt34_map_full_range() — Write every MPTE at Root MPT + * + * Writes xwr into every root table MPTE directly as a leaf + * superpage covering the mode entire addressable PA range. + * No intermidiate MPT tables allocated just all leaf MPTEs at + * root level. + */ + +static int smmpt34_map_full_range(struct sbi_mpt_domain *dom, u8 xwr) +{ + u32 mpte = SBI_MPTE_V | SBI_MPTE_L; + u32 pg, i; + + for (pg = 0; pg < RV32_PAGES_PER_MPTE; pg++) + mpte = sbi_mpte_leaf_set_xwr(mpte, pg, xwr); + + for (i = 0; i < RV32_ROOT_ENTRIES; i++) + rv32_write_mpte(rv32_mpte_pa(dom->root_pa, i), mpte); + + return 0; +} + +/* Get the access permissions(xwr) of a PA */ +static u8 smmpt34_get_xwr(struct sbi_mpt_domain *dom, unsigned long pa) +{ + u32 mpte, pi; + unsigned long table_pa = dom->root_pa; + + mpte = rv32_read_mpte(rv32_mpte_pa(table_pa, rv32_pn1_table_idx(pa))); + + if (!(mpte & SBI_MPTE_V)) + return SBI_MPT_PERM_NONE; + + if (!(mpte & SBI_MPTE_L) && (mpte & SBI_MPTE_N)) + return SBI_MPT_PERM_NONE; + + if (mpte & SBI_MPTE_L) { + if (mpte & SBI_MPTE_N) + return rv32_napot_xwr(mpte); + + pi = rv32_tuple_idx(pa, 1); + return (mpte >> sbi_mpte_xwr_shift(pi)) & SBI_MPTE_XWR_MASK; + } + + table_pa = rv32_next_table_pa(mpte); + mpte = rv32_read_mpte(rv32_mpte_pa(table_pa, rv32_pn0_table_idx(pa))); + + if (!(mpte & SBI_MPTE_V) || !(mpte & SBI_MPTE_L)) + return SBI_MPT_PERM_NONE; + + if (mpte & SBI_MPTE_N) + return rv32_napot_xwr(mpte); + + pi = rv32_tuple_idx(pa, 0); + + return ((mpte >> sbi_mpte_xwr_shift(pi)) & SBI_MPTE_XWR_MASK); +} + +struct sbi_mpt_mode smmpt34_mode = { + .name = "Smmpt34", + .mode_val = SBI_MMPT_MODE_SMMPT34, + .encode_mmpt = smmpt34_encode_mmpt, + .map_range = smmpt34_map_range, + .root_table_size = smmpt34_root_table_size, + .root_table_align = smmpt34_root_table_align, + .pa_in_range = smmpt34_pa_in_range, + .map_full_range = smmpt34_map_full_range, + .get_xwr = smmpt34_get_xwr, +}; + +#endif /* __riscv_xlen == 32 */ -- 2.51.0 -- opensbi mailing list [email protected] http://lists.infradead.org/mailman/listinfo/opensbi