[PATCH 2/9] mm/mglru: suppress cross-node empty page table walks

In the current MGLRU, lru_gen_use_mm() will mark one process's mm used
on all nodes at each context switch. So each nodes's aging walks into
each mm's page tables. For an mm with memory on one or only a subset of
nodes, the other nodes' walks find no pages for one lruvec. While these
empty walks are pure waste.

Track per-mm, per-node empty-walk marks: bit N on mm->lru_gen.empty_map is
set when node N's walk of the mm found no page for this lruvec, and
get_next_mm() will skip the mm on node N between re-scan passes. The re-scan
is driven by each node's own pass count (mm_state->seq), so every
mglru_empty_skip_gens-th (default 4) pass re-walks all empty-marked mms to
close migration/NUMA-balancing windows; keeping it on the node's own clock
avoids a shared "oldest marking" sequence latching at the slowest node.

A walk is "empty" when it traversed the page tables and found no folio for
this lruvec.

A page that appears on the node during the skip (fault or migration) is not
aged until the re-scan; a later patch invalidates the skip on those
paths. mm_struct grows by 8 bytes per process.

Signed-off-by: Baoquan He <[email protected]>
---
 include/linux/mm_types.h |  3 +++
 include/linux/mmzone.h   |  2 ++
 mm/vmscan.c              | 52 ++++++++++++++++++++++++++++++++++------
 3 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 6d815f6440c9..3738e8877b73 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1410,6 +1410,8 @@ struct mm_struct {
 			 * page table walkers cleared the corresponding bits.
 			 */
 			unsigned long bitmap;
+			/* bit N: node N's last walk found no folio; skip until re-scan */
+			unsigned long empty_map;
 #ifdef CONFIG_MEMCG
 			/* points to the memcg of "owner" above */
 			struct mem_cgroup *memcg;
@@ -1503,6 +1505,7 @@ static inline void lru_gen_init_mm(struct mm_struct *mm)
 {
 	INIT_LIST_HEAD(&mm->lru_gen.list);
 	mm->lru_gen.bitmap = 0;
+	mm->lru_gen.empty_map = 0;
 #ifdef CONFIG_MEMCG
 	mm->lru_gen.memcg = NULL;
 #endif
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 229d27fbfb54..30213a880db0 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -633,6 +633,8 @@ struct lru_gen_mm_walk {
 	int batched;
 	int swappiness;
 	bool force_scan;
+	/* this aging pass is an empty-walk re-scan pass (every K-th) */
+	bool rescan_pass;
 };
 
 /*
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 92cb83a78971..e8ba49683b28 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2710,6 +2710,14 @@ static bool should_clear_pmd_young(void)
 	return arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG);
 }
 
+/*
+ * Cross-node empty walk suppression. lru_gen_use_mm() marks an mm used on all
+ * nodes, so aging on a node where the mm has no memory wastes a full page table
+ * walk. Skip such an mm for up to MGLRU_EMPTY_SKIP_GENS generations after an
+ * empty walk, then force-rescan to close migration/mlock/NUMA-balancing windows.
+ */
+#define MGLRU_EMPTY_SKIP_GENS 4
+
 /******************************************************************************
  *                          shorthand helpers
  ******************************************************************************/
@@ -2930,9 +2938,17 @@ static struct mm_struct *get_next_mm(struct lru_gen_mm_walk *walk)
 	mm = list_entry(mm_state->head, struct mm_struct, lru_gen.list);
 	key = pgdat->node_id % BITS_PER_TYPE(mm->lru_gen.bitmap);
 
+	/* skip if not used on this node since the last walk */
 	if (!walk->force_scan && !test_bit(key, &mm->lru_gen.bitmap))
 		return NULL;
 
+	/* Skip empty-marked mms except on a re-scan pass; off on node-id alias */
+	if (!walk->force_scan &&
+	    nr_node_ids <= BITS_PER_TYPE(mm->lru_gen.bitmap) &&
+	    !walk->rescan_pass &&
+	    test_bit(key, &mm->lru_gen.empty_map))
+		return NULL;
+
 	clear_bit(key, &mm->lru_gen.bitmap);
 	mmgrab(mm);
 
@@ -3833,7 +3849,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end,
 	return -EAGAIN;
 }
 
-static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
+static bool walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
 {
 	static const struct mm_walk_ops mm_walk_ops = {
 		.test_walk = should_skip_vma,
@@ -3841,6 +3857,7 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
 		.walk_lock = PGWALK_RDLOCK,
 	};
 	int err;
+	bool walked = false;
 	struct lruvec *lruvec = walk->lruvec;
 
 	walk->next_addr = FIRST_USER_ADDRESS;
@@ -3859,6 +3876,7 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
 			err = walk_page_range(mm, walk->next_addr, ULONG_MAX, &mm_walk_ops, walk);
 
 			mmap_read_unlock(mm);
+			walked = true;
 		}
 
 		if (walk->batched)
@@ -3866,6 +3884,9 @@ static void walk_mm(struct mm_struct *mm, struct lru_gen_mm_walk *walk)
 
 		cond_resched();
 	} while (err == -EAGAIN);
+
+	/* true only if the page tables were traversed - a failed trylock/stale seq is not */
+	return walked;
 }
 
 static struct lru_gen_mm_walk *set_mm_walk(struct pglist_data *pgdat, bool force_alloc)
@@ -4110,22 +4131,39 @@ static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
 	walk->seq = seq;
 	walk->swappiness = swappiness;
 	walk->force_scan = force_scan;
+	/* every skip_gens-th pass re-scans empty-marked mms; 0 disables */
+	walk->rescan_pass = mglru_empty_skip_gens == 0 ||
+			    mm_state->seq % READ_ONCE(mglru_empty_skip_gens) == 0;
 
 	do {
 		success = iterate_mm_list(walk, &mm);
 		if (mm) {
+			int nid = lruvec_pgdat(lruvec)->node_id;
+			int key = nid % BITS_PER_TYPE(mm->lru_gen.bitmap);
+			bool walked;
 			bool empty = false;
 
-			walk_mm(mm, walk);
-			/* A walk that traversed page tables but found no folio
-			 * belonging to this lruvec (node+memcg) is pure waste. */
-			if (walk->mm_stats[MM_LEAF_TOTAL]) {
+			walked = walk_mm(mm, walk);
+
+			/* Empty = walked, no eligible folio; failed trylock/stale seq is not */
+			empty = walked && walk->mm_stats[MM_LEAF_ELIGIBLE] == 0;
+			/* skip_gens == 0: stop maintaining marks */
+			if (mglru_empty_skip_gens &&
+			    nr_node_ids <= BITS_PER_TYPE(mm->lru_gen.bitmap)) {
+				if (empty)
+					set_bit(key, &mm->lru_gen.empty_map);
+				else
+					/* found eligible folios: clear the marking */
+					clear_bit(key, &mm->lru_gen.empty_map);
+			}
+
+			/* Count completed walks with the same "empty" definition */
+			if (walked) {
 				walk->mm_stats[MM_WALK_TOTAL]++;
-				if (walk->mm_stats[MM_LEAF_ELIGIBLE] == 0) {
+				if (empty) {
 					walk->mm_stats[MM_WALK_EMPTY]++;
 					walk->mm_stats[MM_LEAF_TOTAL_EMPTY] +=
 						walk->mm_stats[MM_LEAF_TOTAL];
-					empty = true;
 				}
 			}
 			trace_mm_vmscan_lru_gen_walk(
-- 
2.54.0