[RFC PATCH v3 05/14] mm/memcontrol: Set tier limits proportional to memory limits

Compute proportional per-tier limits based on memory limits when
users write to memory limit sysfs files, or when memory hotplug causes
tier proportions to be shifted.

No-op unless the system has tiered memcg limits enabled.

Signed-off-by: Joshua Hahn <[email protected]>
---
 include/linux/memcontrol.h   | 10 +++++++
 include/linux/memory-tiers.h |  6 ++++
 mm/memcontrol.c              | 54 ++++++++++++++++++++++++++++++++++++
 mm/memory-tiers.c            | 13 +++++++++
 4 files changed, 83 insertions(+)

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index bb5bde87ac85a..f7a92b66330ec 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -537,11 +537,17 @@ static inline bool mem_cgroup_tiered_limits(void)
 {
 	return static_branch_unlikely(&memcg_tiered_limits_key);
 }
+
+void establish_memcg_tier_limits(void);
 #else
 static inline bool mem_cgroup_tiered_limits(void)
 {
 	return false;
 }
+
+static inline void establish_memcg_tier_limits(void)
+{
+}
 #endif
 
 static inline void mem_cgroup_protection(struct mem_cgroup *root,
@@ -1102,6 +1108,10 @@ static inline bool mem_cgroup_tiered_limits(void)
 	return false;
 }
 
+static inline void establish_memcg_tier_limits(void)
+{
+}
+
 static inline void memcg_memory_event(struct mem_cgroup *memcg,
 				      enum memcg_memory_event event)
 {
diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
index 0e49645cdd1a9..04b396f60b457 100644
--- a/include/linux/memory-tiers.h
+++ b/include/linux/memory-tiers.h
@@ -55,6 +55,7 @@ struct memory_dev_type *mt_find_alloc_memory_type(int adist,
 						  struct list_head *memory_types);
 void mt_put_memory_types(struct list_head *memory_types);
 const nodemask_t *mt_tier_nodes(int slot);
+unsigned long mt_scale_by_tier(unsigned long val, int slot);
 #ifdef CONFIG_NUMA_MIGRATION
 int next_demotion_node(int node, const nodemask_t *allowed_mask);
 void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets);
@@ -169,5 +170,10 @@ static inline const nodemask_t *mt_tier_nodes(int slot)
 {
 	return NULL;
 }
+
+static inline unsigned long mt_scale_by_tier(unsigned long val, int slot)
+{
+	return val;
+}
 #endif	/* CONFIG_NUMA */
 #endif  /* _LINUX_MEMORY_TIERS_H */
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index d096010366515..defd04acfb3fd 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4421,6 +4421,35 @@ static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
 	mem_cgroup_free(memcg);
 }
 
+static inline unsigned long page_counter_max_or_scale(unsigned long val,
+						      int slot)
+{
+	return val == PAGE_COUNTER_MAX ? PAGE_COUNTER_MAX :
+					 mt_scale_by_tier(val, slot);
+}
+
+static void memcg_scale_tier_limits(struct mem_cgroup *memcg)
+{
+	unsigned long min = READ_ONCE(memcg->memory.min);
+	unsigned long low = READ_ONCE(memcg->memory.low);
+	unsigned long high = READ_ONCE(memcg->memory.high);
+	unsigned long max = READ_ONCE(memcg->memory.max);
+	int nr_tier_slots = mt_nr_tier_slots();
+
+	for (int slot = 0; slot < nr_tier_slots; slot++) {
+		unsigned long new_min = page_counter_max_or_scale(min, slot);
+		unsigned long new_low = page_counter_max_or_scale(low, slot);
+		unsigned long new_high = page_counter_max_or_scale(high, slot);
+		unsigned long new_max = page_counter_max_or_scale(max, slot);
+		struct page_counter *tier = &memcg->tier[slot];
+
+		page_counter_set_min(tier, new_min);
+		page_counter_set_low(tier, new_low);
+		page_counter_set_high(tier, new_high);
+		xchg(&tier->max, new_max);
+	}
+}
+
 /**
  * mem_cgroup_css_reset - reset the states of a mem_cgroup
  * @css: the target css
@@ -4454,6 +4483,8 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
 	page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
 	memcg1_soft_limit_reset(memcg);
 	page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
+	if (mem_cgroup_tiered_limits())
+		memcg_scale_tier_limits(memcg);
 	memcg_wb_domain_size_changed(memcg);
 }
 
@@ -4797,6 +4828,21 @@ static ssize_t memory_peak_write(struct kernfs_open_file *of, char *buf,
 			  &memcg->memory_peaks);
 }
 
+#ifdef CONFIG_NUMA
+void establish_memcg_tier_limits(void)
+{
+	struct mem_cgroup *memcg;
+
+	if (!mem_cgroup_tiered_limits())
+		return;
+
+	for_each_mem_cgroup_tree(memcg, NULL) {
+		if (memcg != root_mem_cgroup)
+			memcg_scale_tier_limits(memcg);
+	}
+}
+#endif
+
 #undef OFP_PEAK_UNSET
 
 static int memory_min_show(struct seq_file *m, void *v)
@@ -4818,6 +4864,8 @@ static ssize_t memory_min_write(struct kernfs_open_file *of,
 		return err;
 
 	page_counter_set_min(&memcg->memory, min);
+	if (mem_cgroup_tiered_limits())
+		memcg_scale_tier_limits(memcg);
 
 	return nbytes;
 }
@@ -4841,6 +4889,8 @@ static ssize_t memory_low_write(struct kernfs_open_file *of,
 		return err;
 
 	page_counter_set_low(&memcg->memory, low);
+	if (mem_cgroup_tiered_limits())
+		memcg_scale_tier_limits(memcg);
 
 	return nbytes;
 }
@@ -4866,6 +4916,8 @@ static ssize_t memory_high_write(struct kernfs_open_file *of,
 		return err;
 
 	page_counter_set_high(&memcg->memory, high);
+	if (mem_cgroup_tiered_limits())
+		memcg_scale_tier_limits(memcg);
 
 	if (of->file->f_flags & O_NONBLOCK)
 		goto out;
@@ -4925,6 +4977,8 @@ static ssize_t memory_max_write(struct kernfs_open_file *of,
 		return err;
 
 	xchg(&memcg->memory.max, max);
+	if (mem_cgroup_tiered_limits())
+		memcg_scale_tier_limits(memcg);
 
 	if (of->file->f_flags & O_NONBLOCK)
 		goto out;
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index bd5c78cd26ec4..e2c99f51c36d1 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -810,6 +810,7 @@ static int __init memory_tier_late_init(void)
 
 	establish_demotion_targets();
 	establish_tier_slots();
+	establish_memcg_tier_limits();
 	put_online_mems();
 
 	return 0;
@@ -967,6 +968,16 @@ const nodemask_t *mt_tier_nodes(int slot)
 	return &tier_nodemasks[slot];
 }
 
+unsigned long mt_scale_by_tier(unsigned long val, int slot)
+{
+	unsigned long total_capacity = totalram_pages();
+
+	if (slot < 0 || !total_capacity)
+		return 0;
+
+	return mult_frac(val, READ_ONCE(tier_capacity[slot]), total_capacity);
+}
+
 static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 					      unsigned long action, void *_arg)
 {
@@ -979,6 +990,7 @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 		if (clear_node_memory_tier(nn->nid)) {
 			establish_demotion_targets();
 			establish_tier_slots();
+			establish_memcg_tier_limits();
 		}
 		mutex_unlock(&memory_tier_lock);
 		break;
@@ -988,6 +1000,7 @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 		if (!IS_ERR(memtier)) {
 			establish_demotion_targets();
 			establish_tier_slots();
+			establish_memcg_tier_limits();
 		}
 		mutex_unlock(&memory_tier_lock);
 		break;
-- 
2.53.0-Meta