[RFC PATCH v3 08/14] mm/memcontrol: Make memory.low and memory.min tier-aware

On machines serving multiple workloads whose memory is isolated via
the memory cgroup controller, it is currently impossible to enforce a
fair distribution of tiered memory among the workloads, as the only
enforceable limits have to do with total memory footprint, but not where
that memory resides.

This makes ensuring a consistent and baseline performance difficult, as
each workload's performance is heavily impacted by workload-external
factors such as which other workloads are co-located in the same host,
and the order at which different workloads are started.

Extend the existing memory.{low, min} protection to be tier-aware in
order to enforce proportional best-effort and guaranteed memory
protection of higher-tier memory.

No-op unless the system has tiered memcg limits enabled.

Signed-off-by: Joshua Hahn <[email protected]>
---
 include/linux/memcontrol.h | 25 +++++++++++++++++++++----
 mm/memcontrol.c            | 11 ++++++++++-
 mm/vmscan.c                | 15 +++++++++------
 3 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index f7a92b66330ec..ceba0fd6de184 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -18,6 +18,7 @@
 #include <linux/page_counter.h>
 #include <linux/vmpressure.h>
 #include <linux/eventfd.h>
+#include <linux/memory-tiers.h>
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/writeback.h>
@@ -618,21 +619,37 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
 }
 
 static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+					struct mem_cgroup *memcg, int nid)
 {
 	if (mem_cgroup_unprotected(target, memcg))
 		return false;
 
+	if (mem_cgroup_tiered_limits()) {
+		int slot = nid_tier_slot(nid);
+
+		if (slot >= 0)
+			return READ_ONCE(memcg->tier[slot].elow) >=
+				page_counter_read(&memcg->tier[slot]);
+	}
+
 	return READ_ONCE(memcg->memory.elow) >=
 		page_counter_read(&memcg->memory);
 }
 
 static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+					struct mem_cgroup *memcg, int nid)
 {
 	if (mem_cgroup_unprotected(target, memcg))
 		return false;
 
+	if (mem_cgroup_tiered_limits()) {
+		int slot = nid_tier_slot(nid);
+
+		if (slot >= 0)
+			return READ_ONCE(memcg->tier[slot].emin) >=
+				page_counter_read(&memcg->tier[slot]);
+	}
+
 	return READ_ONCE(memcg->memory.emin) >=
 		page_counter_read(&memcg->memory);
 }
@@ -1142,13 +1159,13 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target,
 	return true;
 }
 static inline bool mem_cgroup_below_low(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+					struct mem_cgroup *memcg, int nid)
 {
 	return false;
 }
 
 static inline bool mem_cgroup_below_min(struct mem_cgroup *target,
-					struct mem_cgroup *memcg)
+					struct mem_cgroup *memcg, int nid)
 {
 	return false;
 }
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index f3714dfd85aa0..025496794cb91 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5310,7 +5310,16 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 	if (!root)
 		root = root_mem_cgroup;
 
-	page_counter_calculate_protection(&root->memory, &memcg->memory, recursive_protection);
+	page_counter_calculate_protection(&root->memory, &memcg->memory,
+					  recursive_protection);
+
+	if (mem_cgroup_tiered_limits()) {
+		int nr_tier_slots = mt_nr_tier_slots();
+
+		for (int slot = 0; slot < nr_tier_slots; slot++)
+			page_counter_calculate_protection(&root->tier[slot],
+				&memcg->tier[slot], recursive_protection);
+	}
 }
 
 static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
diff --git a/mm/vmscan.c b/mm/vmscan.c
index ffe7ea3c5aff6..29f3f12042650 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4190,7 +4190,7 @@ static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	DEFINE_MIN_SEQ(lruvec);
 
-	if (mem_cgroup_below_min(NULL, memcg))
+	if (mem_cgroup_below_min(NULL, memcg, lruvec_pgdat(lruvec)->node_id))
 		return false;
 
 	if (!lruvec_is_sizable(lruvec, sc))
@@ -5057,6 +5057,7 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
 	bool need_rotate = false, should_age = false;
 	long nr_batch, nr_to_scan;
 	int swappiness = get_swappiness(lruvec, sc);
+	int nid = lruvec_pgdat(lruvec)->node_id;
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 
 	nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
@@ -5064,7 +5065,7 @@ static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
 		int delta;
 		DEFINE_MAX_SEQ(lruvec);
 
-		if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) {
+		if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg, nid)) {
 			need_rotate = true;
 			break;
 		}
@@ -5104,12 +5105,13 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
 	unsigned long reclaimed = sc->nr_reclaimed;
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
+	int nid = pgdat->node_id;
 
 	/* lru_gen_age_node() called mem_cgroup_calculate_protection() */
-	if (mem_cgroup_below_min(NULL, memcg))
+	if (mem_cgroup_below_min(NULL, memcg, nid))
 		return MEMCG_LRU_YOUNG;
 
-	if (mem_cgroup_below_low(NULL, memcg)) {
+	if (mem_cgroup_below_low(NULL, memcg, nid)) {
 		/* see the comment on MEMCG_NR_GENS */
 		if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL)
 			return MEMCG_LRU_TAIL;
@@ -6168,6 +6170,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
 	};
 	struct mem_cgroup_reclaim_cookie *partial = &reclaim;
 	struct mem_cgroup *memcg;
+	int nid = pgdat->node_id;
 
 	/*
 	 * In most cases, direct reclaimers can do partial walks
@@ -6197,13 +6200,13 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
 
 		mem_cgroup_calculate_protection(target_memcg, memcg);
 
-		if (mem_cgroup_below_min(target_memcg, memcg)) {
+		if (mem_cgroup_below_min(target_memcg, memcg, nid)) {
 			/*
 			 * Hard protection.
 			 * If there is no reclaimable memory, OOM.
 			 */
 			continue;
-		} else if (mem_cgroup_below_low(target_memcg, memcg)) {
+		} else if (mem_cgroup_below_low(target_memcg, memcg, nid)) {
 			/*
 			 * Soft protection.
 			 * Respect the protection only as long as
-- 
2.53.0-Meta