git: b83204edf101 - main - powerpc/pmap: Support booke64 kernel pmap growing

The branch main has been updated by jhibbits:

URL: https://cgit.FreeBSD.org/src/commit/?id=b83204edf101d2c8ad40005ca2fe9a76796cd4f1

commit b83204edf101d2c8ad40005ca2fe9a76796cd4f1
Author:     Justin Hibbits <[email protected]>
AuthorDate: 2026-08-05 03:47:27 +0000
Commit:     Justin Hibbits <[email protected]>
CommitDate: 2026-08-07 02:52:28 +0000

    powerpc/pmap: Support booke64 kernel pmap growing
    
    In preparation of increasing the KVA on powerpc64 to 2TB to mirror
    amd64's, rework the 64-bit Book-E pmap to not allocate all page table
    pages at boot time, since that would be a waste of a lot of memory.
    Instead, allocate all page table pages for the higher levels, leaving
    the leaves (page directories) for dynamic allocation.  This cuts the
    boot-time page table size down from ~64MB to ~8MB with the current 32GB
    KVA size, and bumping to 2TB KVA the boot-time page table is still ~8MB
    instead of ballooning to ~4GB of mostly wasted space.
---
 sys/powerpc/booke/pmap.c    | 18 +++++++--
 sys/powerpc/booke/pmap_64.c | 95 ++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 101 insertions(+), 12 deletions(-)

diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c
index 545416921d09..ce16566a6f13 100644
--- a/sys/powerpc/booke/pmap.c
+++ b/sys/powerpc/booke/pmap.c
@@ -359,6 +359,9 @@ static int		mmu_booke_decode_kernel_ptr(vm_offset_t addr,
 static void		mmu_booke_page_array_startup(long);
 static bool mmu_booke_page_is_mapped(vm_page_t m);
 static bool mmu_booke_ps_enabled(pmap_t pmap);
+#ifdef __powerpc64__
+static int		mmu_booke_growkernel(vm_offset_t);
+#endif
 
 static struct pmap_funcs mmu_booke_methods = {
 	/* pmap dispatcher interface */
@@ -402,6 +405,9 @@ static struct pmap_funcs mmu_booke_methods = {
 	.page_array_startup = mmu_booke_page_array_startup,
 	.page_is_mapped = mmu_booke_page_is_mapped,
 	.ps_enabled = mmu_booke_ps_enabled,
+#ifdef __powerpc64__
+	.growkernel_nopanic = mmu_booke_growkernel,
+#endif
 
 	/* Internal interfaces */
 	.bootstrap = mmu_booke_bootstrap,
@@ -700,7 +706,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
 	    (uintptr_t)msgbufp, data_end);
 
 	data_end = round_page(data_end);
-	data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
 
 	/* Retrieve phys/avail mem regions */
 	mem_regions(&physmem_regions, &physmem_regions_sz,
@@ -709,7 +714,6 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
 	if (PHYS_AVAIL_ENTRIES < availmem_regions_sz)
 		panic("mmu_booke_bootstrap: phys_avail too small");
 
-	data_end = round_page(data_end);
 	vm_page_array = (vm_page_t)data_end;
 	/*
 	 * Get a rough idea (upper bound) on the size of the page array.  The
@@ -723,6 +727,14 @@ mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)
 	sz = (round_page(sz) / (PAGE_SIZE + sizeof(struct vm_page)));
 	data_end += round_page(sz * sizeof(struct vm_page));
 
+	/*
+	 * Reserve kernel page-table pages last, so their reservation size can
+	 * be computed from the final bootstrap data_end (on 64-bit, only leaf
+	 * ptbls covering [VM_MIN_KERNEL_ADDRESS, data_end + slack] are
+	 * pre-allocated; the rest are added on demand by pmap_growkernel()).
+	 */
+	data_end = round_page(mmu_booke_alloc_kernel_pgtables(data_end));
+
 	/* Pre-round up to 1MB.  This wastes some space, but saves TLB entries */
 	data_end = roundup2(data_end, 1 << 20);
 
@@ -1216,7 +1228,7 @@ mmu_booke_kremove(vm_offset_t va)
 
 	pte = pte_find(kernel_pmap, va);
 
-	if (!PTE_ISVALID(pte)) {
+	if (pte == NULL || !PTE_ISVALID(pte)) {
 		CTR1(KTR_PMAP, "%s: invalid pte", __func__);
 
 		return;
diff --git a/sys/powerpc/booke/pmap_64.c b/sys/powerpc/booke/pmap_64.c
index 97f9a5967167..3f2db07ce931 100644
--- a/sys/powerpc/booke/pmap_64.c
+++ b/sys/powerpc/booke/pmap_64.c
@@ -113,6 +113,13 @@ unsigned int kernel_pdirs;
 static uma_zone_t ptbl_root_zone;
 static pte_t ****kernel_ptbl_root;
 
+/*
+ * Slack space beyond the bootstrap data_end for which leaf ptbls are
+ * pre-allocated.  Must cover post-alloc growth (1MB roundup + TLB1 grow +
+ * kstack + guard pages).  Rounded up to PDIR_SIZE by the reservation code.
+ */
+#define	BOOTSTRAP_LEAF_SLACK	(2 * PDIR_SIZE)
+
 /*
  * Base of the pmap_mapdev() region.  On 32-bit it immediately follows the
  * userspace address range.  On On 64-bit it's far above, at (1 << 63), and
@@ -262,14 +269,16 @@ get_pgtbl_page(pmap_t pmap, void **ptr_tbl, uint32_t index,
 	vm_page_t	m;
 
 	page = ptr_tbl[index];
-	KASSERT(page != 0 || pmap != kernel_pmap,
-	    ("NULL page table page found in kernel pmap!"));
 	if (page == NULL) {
 		page = mmu_booke_alloc_page(pmap, index, nosleep);
 		if (ptr_tbl[index] == NULL) {
 			*isnew = true;
 			ptr_tbl[index] = page;
-			if (hold_parent) {
+			/*
+			 * Kernel page-table pages are never freed, so we do
+			 * not maintain refcounts on their parents.
+			 */
+			if (hold_parent && pmap != kernel_pmap) {
 				m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)ptr_tbl));
 				m->ref_count++;
 			}
@@ -518,7 +527,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
 {
 	pte_t		*pte;
 	vm_size_t	kva_size;
-	int		kernel_pdirs, kernel_pgtbls, pdir_l1s;
+	int		kernel_pdirs, pdir_l1s;
+	unsigned int	kernel_pgtbls;
 	vm_offset_t	va, l1_va, pdir_va, ptbl_va;
 	int		i, j, k;
 
@@ -526,7 +536,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
 	kernel_pmap->pm_root = kernel_ptbl_root;
 	pdir_l1s = howmany(kva_size, PG_ROOT_SIZE);
 	kernel_pdirs = howmany(kva_size, PDIR_L1_SIZE);
-	kernel_pgtbls = howmany(kva_size, PDIR_SIZE);
+	kernel_pgtbls = howmany(kernel_vm_end - VM_MIN_KERNEL_ADDRESS,
+	    PDIR_SIZE);
 
 	/* Initialize kernel pdir */
 	l1_va = (vm_offset_t)kernel_ptbl_root +
@@ -537,7 +548,8 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
 		printf("ptbl_root_va: %#lx\n", (vm_offset_t)kernel_ptbl_root);
 		printf("l1_va: %#lx (%d entries)\n", l1_va, pdir_l1s);
 		printf("pdir_va: %#lx(%d entries)\n", pdir_va, kernel_pdirs);
-		printf("ptbl_va: %#lx(%d entries)\n", ptbl_va, kernel_pgtbls);
+		printf("ptbl_va: %#lx(%u entries)\n", ptbl_va, kernel_pgtbls);
+		printf("kernel_vm_end: %#lx\n", kernel_vm_end);
 	}
 
 	va = VM_MIN_KERNEL_ADDRESS;
@@ -550,8 +562,18 @@ kernel_pte_alloc(vm_offset_t data_end, vm_offset_t addr)
 			kernel_pmap->pm_root[i][j] = (pte_t **)pdir_va;
 			for (k = 0;
 			    k < PDIR_NENTRIES && va < VM_MAX_KERNEL_ADDRESS;
-			    k++, va += PDIR_SIZE, ptbl_va += PAGE_SIZE)
-				kernel_pmap->pm_root[i][j][k] = (pte_t *)ptbl_va;
+			    k++, va += PDIR_SIZE) {
+				/*
+				 * Only wire up leaf ptbl pages for the
+				 * pre-allocated bootstrap KVA range; the rest
+				 * are populated lazily by mmu_booke_growkernel().
+				 */
+				if (va < kernel_vm_end) {
+					kernel_pmap->pm_root[i][j][k] =
+					    (pte_t *)ptbl_va;
+					ptbl_va += PAGE_SIZE;
+				}
+			}
 		}
 	}
 	/*
@@ -572,16 +594,71 @@ static vm_offset_t
 mmu_booke_alloc_kernel_pgtables(vm_offset_t data_end)
 {
 	vm_size_t kva_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS;
+	unsigned int leaves;
+
 	kernel_ptbl_root = (pte_t ****)data_end;
 
 	data_end += round_page(PG_ROOT_NENTRIES * sizeof(pte_t ***));
 	data_end += howmany(kva_size, PG_ROOT_SIZE) * PAGE_SIZE;
 	data_end += howmany(kva_size, PDIR_L1_SIZE) * PAGE_SIZE;
-	data_end += howmany(kva_size, PDIR_SIZE) * PAGE_SIZE;
+	/*
+	 * Reserve leaf page-table pages only for the current bootstrap KVA
+	 * range plus a small slack for post-alloc growth (1MB TLB roundup,
+	 * kstack + guard).  Leaves for the rest of KVA are allocated on
+	 * demand by mmu_booke_growkernel().  The two-step howmany() converges
+	 * because leaves themselves add at most one extra leaf per 512 leaves.
+	 */
+	leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS +
+	    BOOTSTRAP_LEAF_SLACK, PDIR_SIZE);
+	leaves = howmany(data_end - VM_MIN_KERNEL_ADDRESS +
+	    BOOTSTRAP_LEAF_SLACK + leaves * PAGE_SIZE, PDIR_SIZE);
+	data_end += leaves * PAGE_SIZE;
+
+	kernel_vm_end = VM_MIN_KERNEL_ADDRESS + leaves * PDIR_SIZE;
 
 	return (data_end);
 }
 
+/*
+ * Grow the kernel page-table by allocating additional leaf ptbl pages so
+ * that pte_find(kernel_pmap, va) can succeed for [kernel_vm_end, addr].
+ *
+ * All upper levels (root, pdir_l1, pdir) are pre-populated for the whole
+ * KVA at bootstrap, so only the leaf level needs to be allocated here.
+ */
+static int
+mmu_booke_growkernel(vm_offset_t addr)
+{
+	pte_t		**pdir;
+	vm_page_t	m;
+	vm_offset_t	va;
+
+	if (addr <= kernel_vm_end)
+		return (KERN_SUCCESS);
+
+	addr = roundup2(addr, PDIR_SIZE);
+	if (addr - 1 >= vm_map_max(kernel_map))
+		addr = vm_map_max(kernel_map);
+
+	for (va = kernel_vm_end; va < addr; va += PDIR_SIZE) {
+		pdir = kernel_pmap->pm_root[PG_ROOT_IDX(va)][PDIR_L1_IDX(va)];
+		KASSERT(pdir != NULL,
+		    ("mmu_booke_growkernel: NULL pdir at va %#lx", va));
+		if (pdir[PDIR_IDX(va)] != NULL)
+			continue;
+		m = vm_page_alloc_noobj(VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED |
+		    VM_ALLOC_ZERO);
+		if (m == NULL) {
+			kernel_vm_end = va;
+			return (KERN_RESOURCE_SHORTAGE);
+		}
+		m->pindex = va >> PDIR_SHIFT;
+		pdir[PDIR_IDX(va)] = (pte_t *)VM_PAGE_TO_DMAP(m);
+	}
+	kernel_vm_end = addr;
+	return (KERN_SUCCESS);
+}
+
 /*
  * Initialize a preallocated and zeroed pmap structure,
  * such as one in a vmspace structure.