[RFC PATCH v3 04/14] mm/memcontrol: Allocate per-tier page_counters

Tier-aware limits need one page_counter per memory tier.  Add a
page_counter array (memcg->tier) to struct mem_cgroup, sized to
nr_node_ids (an upper bound on the number of tiers) and allocated only
when tiered limits are enabled, so memcgs pay nothing when the feature is
off.  Initialise and parent-link every slot in mem_cgroup_css_alloc(),
and free the array in __mem_cgroup_free().

Signed-off-by: Joshua Hahn <[email protected]>
---
 include/linux/memcontrol.h |  1 +
 mm/memcontrol.c            | 23 +++++++++++++++++++++++
 mm/memory-tiers.c          | 17 ++++++++++++++---
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index dce03df7eae05..bb5bde87ac85a 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -207,6 +207,7 @@ struct mem_cgroup {
 
 	/* Accounted resources */
 	struct page_counter memory;		/* Both v1 & v2 */
+	struct page_counter *tier;		/* v2 only */
 
 	union {
 		struct page_counter swap;	/* v2 only */
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index ec28512de6a23..d096010366515 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -53,6 +53,7 @@
 #include <linux/seq_file.h>
 #include <linux/vmpressure.h>
 #include <linux/memremap.h>
+#include <linux/memory-tiers.h>
 #include <linux/mm_inline.h>
 #include <linux/cpu.h>
 #include <linux/oom.h>
@@ -4126,6 +4127,7 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 	memcg1_free_events(memcg);
 	kfree(memcg->vmstats);
 	free_percpu(memcg->vmstats_percpu);
+	kfree(memcg->tier);
 	kfree(memcg);
 }
 
@@ -4178,6 +4180,13 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
 		if (!alloc_mem_cgroup_per_node_info(memcg, node))
 			goto fail;
 
+	if (mem_cgroup_tiered_limits()) {
+		memcg->tier = kcalloc(nr_node_ids, sizeof(*memcg->tier),
+				      GFP_KERNEL);
+		if (!memcg->tier)
+			goto fail;
+	}
+
 	if (memcg_wb_domain_init(memcg, GFP_KERNEL))
 		goto fail;
 
@@ -4206,6 +4215,16 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
 	return ERR_PTR(error);
 }
 
+static void memcg_init_tier_counters(struct mem_cgroup *memcg,
+				     struct mem_cgroup *parent, bool protection)
+{
+	for (int i = 0; i < nr_node_ids; i++) {
+		page_counter_init(&memcg->tier[i],
+				  parent ? &parent->tier[i] : NULL, protection);
+		page_counter_set_high(&memcg->tier[i], PAGE_COUNTER_MAX);
+	}
+}
+
 static struct cgroup_subsys_state * __ref
 mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 {
@@ -4231,6 +4250,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
 	if (parent) {
 		page_counter_init(&memcg->memory, &parent->memory, memcg_on_dfl);
+		if (mem_cgroup_tiered_limits())
+			memcg_init_tier_counters(memcg, parent, memcg_on_dfl);
 		page_counter_init(&memcg->swap, &parent->swap, false);
 #ifdef CONFIG_MEMCG_V1
 		WRITE_ONCE(memcg->swappiness, mem_cgroup_swappiness(parent));
@@ -4243,6 +4264,8 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
 		init_memcg_stats();
 		init_memcg_events();
 		page_counter_init(&memcg->memory, NULL, true);
+		if (mem_cgroup_tiered_limits())
+			memcg_init_tier_counters(memcg, NULL, true);
 		page_counter_init(&memcg->swap, NULL, false);
 #ifdef CONFIG_MEMCG_V1
 		page_counter_init(&memcg->kmem, NULL, false);
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 36187c0ea9ded..bd5c78cd26ec4 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -52,6 +52,7 @@ static int nr_tier_slots;
 static int tier_slot_ids[MAX_NUMNODES]   = {[0 ... MAX_NUMNODES - 1] = -1,};
 static int node_tier_slots[MAX_NUMNODES] = {[0 ... MAX_NUMNODES - 1] = -1,};
 static nodemask_t tier_nodemasks[MAX_NUMNODES];
+static unsigned long tier_capacity[MAX_NUMNODES];
 
 struct memory_dev_type *default_dram_type;
 nodemask_t default_dram_nodes __initdata = NODE_MASK_NONE;
@@ -307,8 +308,10 @@ static void establish_tier_slots(void)
 
 	lockdep_assert_held_once(&memory_tier_lock);
 
-	for (int slot = 0; slot < old_nr_tier_slots; slot++)
+	for (int slot = 0; slot < old_nr_tier_slots; slot++) {
 		nodes_clear(tier_nodemasks[slot]);
+		WRITE_ONCE(tier_capacity[slot], 0);
+	}
 
 	for (int nid = 0; nid < nr_node_ids; nid++) {
 		struct memory_tier *memtier = NULL;
@@ -323,8 +326,16 @@ static void establish_tier_slots(void)
 
 		WRITE_ONCE(node_tier_slots[nid], slot);
 
-		if (slot != -1)
-			node_set(nid, tier_nodemasks[slot]);
+		if (slot < 0)
+			continue;
+
+		node_set(nid, tier_nodemasks[slot]);
+		for (int i = 0; i < MAX_NR_ZONES; i++) {
+			struct zone *zone = &NODE_DATA(nid)->node_zones[i];
+
+			WRITE_ONCE(tier_capacity[slot],
+				tier_capacity[slot] + zone_managed_pages(zone));
+		}
 	}
 	WRITE_ONCE(nr_tier_slots, highest_slot);
 }
-- 
2.53.0-Meta