[RFC PATCH v3 03/14] mm/memory-tiers: Introduce a mapping from nid to tier_slot

Establishing tiered memcg limits will require an ordering of tiers, as
well as a way to account how much memory is present in each tier.
This will need to be done starting at boot, so that all memory becomes
properly accounted.

However, tiers can come online and offline at runtime due to DAX memory
whose nodes can be hotplugged / hot-unplugged, and these nodes' tiers
are not available at boot.

Therefore, to establish a fixed mapping from nid to tier that isn't
sparse like the tier_ids, introduce a new "tier_slot" which is a dense
index that does not change once a tier comes online.

Also introduce a helper to retrieve the nodemask associated with a tier.

Signed-off-by: Joshua Hahn <[email protected]>
---
 include/linux/memory-tiers.h | 18 ++++++++
 mm/memory-tiers.c            | 86 +++++++++++++++++++++++++++++++++++-
 2 files changed, 102 insertions(+), 2 deletions(-)

diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h
index 7999c58629eeb..0e49645cdd1a9 100644
--- a/include/linux/memory-tiers.h
+++ b/include/linux/memory-tiers.h
@@ -41,6 +41,8 @@ extern struct memory_dev_type *default_dram_type;
 extern nodemask_t default_dram_nodes;
 struct memory_dev_type *alloc_memory_type(int adistance);
 void put_memory_type(struct memory_dev_type *memtype);
+int mt_nr_tier_slots(void);
+int nid_tier_slot(int nid);
 void init_node_memory_type(int node, struct memory_dev_type *default_type);
 void clear_node_memory_type(int node, struct memory_dev_type *memtype);
 int register_mt_adistance_algorithm(struct notifier_block *nb);
@@ -52,6 +54,7 @@ int mt_perf_to_adistance(struct access_coordinate *perf, int *adist);
 struct memory_dev_type *mt_find_alloc_memory_type(int adist,
 						  struct list_head *memory_types);
 void mt_put_memory_types(struct list_head *memory_types);
+const nodemask_t *mt_tier_nodes(int slot);
 #ifdef CONFIG_NUMA_MIGRATION
 int next_demotion_node(int node, const nodemask_t *allowed_mask);
 void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets);
@@ -151,5 +154,20 @@ static inline struct memory_dev_type *mt_find_alloc_memory_type(int adist,
 static inline void mt_put_memory_types(struct list_head *memory_types)
 {
 }
+
+static inline int mt_nr_tier_slots(void)
+{
+	return 0;
+}
+
+static inline int nid_tier_slot(int nid)
+{
+	return -1;
+}
+
+static inline const nodemask_t *mt_tier_nodes(int slot)
+{
+	return NULL;
+}
 #endif	/* CONFIG_NUMA */
 #endif  /* _LINUX_MEMORY_TIERS_H */
diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c
index 54851d8a195b0..36187c0ea9ded 100644
--- a/mm/memory-tiers.c
+++ b/mm/memory-tiers.c
@@ -43,6 +43,16 @@ static LIST_HEAD(memory_tiers);
  */
 static LIST_HEAD(default_memory_types);
 static struct node_memory_type_map node_memory_types[MAX_NUMNODES];
+
+/*
+ * nr_tier_slots and tier_slot_ids are written with memory_tier_lock and
+ * read locklessly. nr_tier_slots is monotonically increasing.
+ */
+static int nr_tier_slots;
+static int tier_slot_ids[MAX_NUMNODES]   = {[0 ... MAX_NUMNODES - 1] = -1,};
+static int node_tier_slots[MAX_NUMNODES] = {[0 ... MAX_NUMNODES - 1] = -1,};
+static nodemask_t tier_nodemasks[MAX_NUMNODES];
+
 struct memory_dev_type *default_dram_type;
 nodemask_t default_dram_nodes __initdata = NODE_MASK_NONE;
 
@@ -273,6 +283,65 @@ static struct memory_tier *__node_get_memory_tier(int node)
 				     lockdep_is_held(&memory_tier_lock));
 }
 
+/* Caller must hold memory_tier_lock */
+static int tier_id_slot(int tier_id)
+{
+	int slot, free_slot = -1;
+
+	for (slot = 0; slot < nr_node_ids; slot++) {
+		if (tier_slot_ids[slot] == tier_id)
+			return slot;
+		if (tier_slot_ids[slot] == -1 && free_slot == -1) {
+			free_slot = slot;
+			tier_slot_ids[slot] = tier_id;
+		}
+	}
+
+	return free_slot;
+}
+
+static void establish_tier_slots(void)
+{
+	int old_nr_tier_slots = mt_nr_tier_slots();
+	int highest_slot = old_nr_tier_slots;
+
+	lockdep_assert_held_once(&memory_tier_lock);
+
+	for (int slot = 0; slot < old_nr_tier_slots; slot++)
+		nodes_clear(tier_nodemasks[slot]);
+
+	for (int nid = 0; nid < nr_node_ids; nid++) {
+		struct memory_tier *memtier = NULL;
+		int slot = -1;
+
+		if (node_state(nid, N_MEMORY))
+			memtier = __node_get_memory_tier(nid);
+		if (memtier) {
+			slot = tier_id_slot(memtier->dev.id);
+			highest_slot = max(highest_slot, slot + 1);
+		}
+
+		WRITE_ONCE(node_tier_slots[nid], slot);
+
+		if (slot != -1)
+			node_set(nid, tier_nodemasks[slot]);
+	}
+	WRITE_ONCE(nr_tier_slots, highest_slot);
+}
+
+int mt_nr_tier_slots(void)
+{
+	return READ_ONCE(nr_tier_slots);
+}
+
+int nid_tier_slot(int nid)
+{
+	if (nid < 0 || nid >= MAX_NUMNODES)
+		return -1;
+
+	return READ_ONCE(node_tier_slots[nid]);
+}
+
 #ifdef CONFIG_NUMA_MIGRATION
 bool node_is_toptier(int node)
 {
@@ -729,6 +798,7 @@ static int __init memory_tier_late_init(void)
 	}
 
 	establish_demotion_targets();
+	establish_tier_slots();
 	put_online_mems();
 
 	return 0;
@@ -878,6 +948,14 @@ int mt_calc_adistance(int node, int *adist)
 }
 EXPORT_SYMBOL_GPL(mt_calc_adistance);
 
+const nodemask_t *mt_tier_nodes(int slot)
+{
+	if (slot < 0)
+		return NULL;
+
+	return &tier_nodemasks[slot];
+}
+
 static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 					      unsigned long action, void *_arg)
 {
@@ -887,15 +965,19 @@ static int __meminit memtier_hotplug_callback(struct notifier_block *self,
 	switch (action) {
 	case NODE_REMOVED_LAST_MEMORY:
 		mutex_lock(&memory_tier_lock);
-		if (clear_node_memory_tier(nn->nid))
+		if (clear_node_memory_tier(nn->nid)) {
 			establish_demotion_targets();
+			establish_tier_slots();
+		}
 		mutex_unlock(&memory_tier_lock);
 		break;
 	case NODE_ADDED_FIRST_MEMORY:
 		mutex_lock(&memory_tier_lock);
 		memtier = set_node_memory_tier(nn->nid);
-		if (!IS_ERR(memtier))
+		if (!IS_ERR(memtier)) {
 			establish_demotion_targets();
+			establish_tier_slots();
+		}
 		mutex_unlock(&memory_tier_lock);
 		break;
 	}
-- 
2.53.0-Meta