[PATCH 5/9] mm/mglru: add PUD-level Bloom filter state

Add a coarser double-buffered Bloom filters to struct lru_gen_mm_state,
one level up from the PMD-level filters. They operate at 1GB (PUD)
granularity, whose 512 PMDs would all fail the PMD-level filter - a general
suppression of unnecessary page table walks. Cross-node empty walks are the
most visible case: a foreign mm's PUDs have no young entries for this lruvec,
so the entire PMD iteration is pure waste.

No behavior change yet; the filters are populated and used by later
patches.

Signed-off-by: Baoquan He <[email protected]>
---
 include/linux/mmzone.h |  6 ++++--
 mm/vmscan.c            | 12 ++++++------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 30213a880db0..64eb010624b2 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -612,8 +612,10 @@ struct lru_gen_mm_state {
 	struct list_head *head;
 	/* where the last iteration ended before */
 	struct list_head *tail;
-	/* Bloom filters flip after each iteration */
-	unsigned long *filters[NR_BLOOM_FILTERS];
+	/* PMD-level Bloom filters flip after each iteration */
+	unsigned long *pmd_filters[NR_BLOOM_FILTERS];
+	/* PUD-level Bloom filters flip after each iteration */
+	unsigned long *pud_filters[NR_BLOOM_FILTERS];
 	/* the mm stats for debugging */
 	unsigned long stats[NR_HIST_GENS][NR_MM_STATS];
 };
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f592f04fe1bd..912cbf86c7ff 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2849,7 +2849,7 @@ static bool test_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long s
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = READ_ONCE(mm_state->filters[gen]);
+	filter = READ_ONCE(mm_state->pmd_filters[gen]);
 	if (!filter)
 		return true;
 
@@ -2865,7 +2865,7 @@ static void update_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = READ_ONCE(mm_state->filters[gen]);
+	filter = READ_ONCE(mm_state->pmd_filters[gen]);
 	if (!filter)
 		return;
 
@@ -2882,7 +2882,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 	unsigned long *filter;
 	int gen = filter_gen_from_seq(seq);
 
-	filter = mm_state->filters[gen];
+	filter = mm_state->pmd_filters[gen];
 	if (filter) {
 		bitmap_clear(filter, 0, BIT(BLOOM_FILTER_SHIFT));
 		return;
@@ -2890,7 +2890,7 @@ static void reset_bloom_filter(struct lru_gen_mm_state *mm_state, unsigned long
 
 	filter = bitmap_zalloc(BIT(BLOOM_FILTER_SHIFT),
 			       __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
-	WRITE_ONCE(mm_state->filters[gen], filter);
+	WRITE_ONCE(mm_state->pmd_filters[gen], filter);
 }
 
 /******************************************************************************
@@ -6010,8 +6010,8 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg)
 			continue;
 
 		for (i = 0; i < NR_BLOOM_FILTERS; i++) {
-			bitmap_free(mm_state->filters[i]);
-			mm_state->filters[i] = NULL;
+			bitmap_free(mm_state->pmd_filters[i]);
+			mm_state->pmd_filters[i] = NULL;
 		}
 	}
 }
-- 
2.54.0