[RFC PATCH v3 13/14] mm/memcontrol, sched/numa: Gate NUMA promotions into memcg tiers

Memory promotions that go through should_numa_migrate_memory determine
if a promotion should be ratelimited / throttled by checking how much
headroom there is in the destination node. If there is enough headroom,
there is no reason to be throttling promotions.

On tiered systems, however, a promotion may trigger reclaim on a node
that has plenty of promotion headroom since the memcg tier may be at
the limit. For these allocations, we should make sure that memcg tier
fullness is also considered when determining whether a promotion should
be able to go through without getting limited.

Add an additional condition to check before letting a promotion
candidate go through un-ratelimited, by checking if the memcg tier is
already at its limit.

No-op unless the system has tiered memcg limits enabled.

Signed-off-by: Joshua Hahn <[email protected]>
---
 include/linux/memcontrol.h |  7 +++++++
 kernel/sched/fair.c        |  3 ++-
 mm/memcontrol.c            | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 9c2f11191a499..a7c366b431a0e 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -654,6 +654,8 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
 		page_counter_read(&memcg->memory);
 }
 
+bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid);
+
 int __mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, gfp_t gfp);
 
 /**
@@ -1172,6 +1174,11 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
 	return false;
 }
 
+static inline bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid)
+{
+	return false;
+}
+
 static inline int mem_cgroup_charge(struct folio *folio,
 		struct mm_struct *mm, gfp_t gfp)
 {
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d78467ec6ee13..397b0f3e67f5f 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2697,7 +2697,8 @@ bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
 		long nr = folio_nr_pages(folio);
 
 		pgdat = NODE_DATA(dst_nid);
-		if (pgdat_free_space_enough(pgdat)) {
+		if (pgdat_free_space_enough(pgdat) &&
+		    !mem_cgroup_tier_over_limit(folio, dst_nid)) {
 			/* workload changed, reset hot threshold */
 			pgdat->nbp_threshold = 0;
 			mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE_NRL, nr);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 4dce7c6fefd98..05611a01aa082 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2590,6 +2590,41 @@ static u64 swap_find_max_overage(struct mem_cgroup *memcg)
 	return max_overage;
 }
 
+bool mem_cgroup_tier_over_limit(struct folio *folio, int dst_nid)
+{
+	struct mem_cgroup *memcg;
+	int dst_slot;
+
+	if (!mem_cgroup_tiered_limits())
+		return false;
+
+	dst_slot = nid_tier_slot(dst_nid);
+	if (nid_tier_slot(folio_nid(folio)) == dst_slot)
+		return false;
+
+	guard(rcu)();
+	memcg = folio_memcg(folio);
+	if (!memcg || mem_cgroup_is_root(memcg))
+		return false;
+
+	do {
+		struct page_counter *tier_counter;
+		unsigned long limit;
+
+		tier_counter = mem_cgroup_tier_counter(memcg, dst_slot);
+		if (!tier_counter)
+			continue;
+
+		limit = min(READ_ONCE(tier_counter->max),
+			    READ_ONCE(tier_counter->high));
+		if (page_counter_read(tier_counter) > limit)
+			return true;
+	} while ((memcg = parent_mem_cgroup(memcg)) &&
+		  !mem_cgroup_is_root(memcg));
+
+	return false;
+}
+
 /*
  * Get the number of jiffies that we should penalise a mischievous cgroup which
  * is exceeding its memory.high by checking both it and its ancestors.
-- 
2.53.0-Meta