Re: [PATCH RFC 09/15] mm/mglru: frequency guided workingset promotion (MGLRU-FG)

On 08/04/26 at 03:47am, Kairui Song via B4 Relay wrote:
...snip...
> +int folio_inc_lru_refs(struct folio *folio, bool is_fault, bool is_exec)
> +{
> +	int max_gen, min_gen;
> +	int type, refs, gen, new_gen;
> +	unsigned long new_flags, old_flags, max_seq;
> +	struct lru_gen_folio *lrugen;
> +	struct lruvec *lruvec;
> +
> +	type = folio_is_file_lru(folio);
> +	lruvec = folio_lruvec_live_get(folio);
> +	lrugen = &lruvec->lrugen;
> +
> +	old_flags = READ_ONCE(*folio_flags(folio, 0));
> +	do {
> +		new_flags = old_flags;
> +		gen = lru_gen_from_flags(old_flags);
> +		refs = lru_refs_from_flags(old_flags) + 1;
> +		new_gen = gen;
> +		if (!(old_flags & BIT(PG_lru)) || gen < 0)
> +			goto out;
> +
> +		max_seq = READ_ONCE(lrugen->max_seq);
> +		max_gen = lru_gen_from_seq(max_seq);
> +		min_gen = lru_gen_from_seq(READ_ONCE(lrugen->min_seq[type]));
> +		if (gen == max_gen)
> +			goto out;
> +

I am a little confused about the new mechanism. In the current mglru, it
does have the issue both mm walk and fd read set PG_referenced at the
1st access, this is a obvious drawback. Now with the change, the ref
count is clearer, while the mm walk and fd read accessing is still mixed.
Imagine the cases below:
 - one fd read; then mm walk; directly move to max_gen;
 - one mm walk; then several times fd read; promote to next gen;

Can I understand the final effect as:
1) explicti ref count;
2) more drastically promote mm walk based on the mixing ref counting;
   - compared with the old behaviour: move to next gen when 2nd mm walk

I can only see one benefit and one significant change. Do I understand
it correctly, and is it worth?

Thanks
Baoquan

> +		if (is_fault || is_exec) {
> +			/* Promote second page table access or executable */
> +			if (refs > LRU_REFS_REFERENCED || is_exec)
> +				new_gen = max_gen;
> +			else
> +				new_gen = (gen + 1UL) % MAX_NR_GENS;
> +			refs = min(refs, LRU_REFS_PROTECTED);
> +		} else if (refs > LRU_REFS_MAX) {
> +			/* LRU refs counting overflow, bump the gen */
> +			new_gen = (gen + 1UL) % MAX_NR_GENS;
> +			refs = LRU_REFS_PROTECTED;
> +		} else if (gen == min_gen && refs >= LRU_REFS_WORKINGSET) {
> +			/* Defer eviction of just accessed workingset */
> +			new_gen = (gen + 1UL) % MAX_NR_GENS;
> +			refs = min(refs, LRU_REFS_PROTECTED);
>  		}
> +out:
> +		refs = min(refs, LRU_REFS_MAX);
> +		lru_refs_set_flags(&new_flags, refs);
> +		if (new_gen >= 0)
> +			lru_gen_set_flags(&new_flags, new_gen);
> +	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
>  
> -		folio_set_lru_refs(folio, 1);
> -		return false;
> +	if (new_gen != gen) {
> +		/*
> +		 * Gen can only go forward, so concurrent aging is
> +		 * usually fine, except when multiple aging increase
> +		 * max_seq multiple times, new_gen may have go beyond
> +		 * the new max_seq's current gen border and causes
> +		 * hotness inversion. In that very unlikely case,
> +		 * just activate the folio.
> +		 */
> +		lru_gen_update_size(lruvec, folio, gen, new_gen);
> +		if (unlikely(READ_ONCE(lrugen->max_seq) - max_seq > MIN_NR_GENS))
> +			folio_activate(folio);
>  	}
>  
> -	/* Promote on second access */
> -	if (folio_lru_refs(folio) > 1) {
> -		folio_set_lru_refs(folio, 0);
> -		folio_set_workingset(folio);
> -	} else {
> -		folio_mark_accessed(folio);
> -	}
> -	return true;
> +	folio_lruvec_live_put(lruvec);
> +	return refs;
> +}
> +
> +/*
> + * Update the folio's lru refs indicator during a page table walk.
> + * max_seq is stable since this runs inside the aging process.
> + *
> + * Returns the old generation and stores the new generation in @new_gen when
> + * the folio is promoted (to max_gen) or advanced by one generation.
> + * Returns -1 if no gen change occurred.
> + */
> +static int folio_inc_lru_refs_walk(struct folio *folio, struct lruvec *lruvec,
> +				   const vma_flags_t *vma_flags, int *new_gen)
> +{
> +	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> +	unsigned long max_seq = READ_ONCE(lruvec->lrugen.max_seq);
> +	int refs, gen, max_gen, ret;
> +
> +	max_gen = lru_gen_from_seq(max_seq);
> +
> +	do {
> +		gen = lru_gen_from_flags(old_flags);
> +		refs = lru_refs_from_flags(old_flags) + 1;
> +		new_flags = old_flags;
> +
> +		if (gen >= 0 && gen != max_gen) {
> +			ret = gen;
> +			/* Promote second page table access or executable */
> +			if (refs > LRU_REFS_REFERENCED || is_exec_file_folio(folio, vma_flags))
> +				*new_gen = max_gen;
> +			else
> +				*new_gen = (gen + 1) % MAX_NR_GENS;
> +			lru_gen_set_flags(&new_flags, *new_gen);
> +			lru_refs_set_flags(&new_flags, min(refs, LRU_REFS_PROTECTED));
> +		} else {
> +			ret = -1;
> +			lru_refs_set_flags(&new_flags, min(refs, LRU_REFS_MAX));
> +		}
> +	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
> +
> +	return ret;
> +}
> +
> +/*
> + * Update the folio's lru refs indicator while the folio is isolated.
> + * Only used on mapped folios upon the final eviction, when the folio is
> + * off the LRU list (isolated).
> + *
> + * Increments the refs count (capped at LRU_REFS_PROTECTED).  Returns true
> + * if the caller should activate the folio (second access or
> + * executable), false to keep it in the eviction list.
> + */
> +static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags)
> +{
> +	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> +	int refs;
> +
> +	do {
> +		new_flags = old_flags;
> +		refs = lru_refs_from_flags(old_flags) + 1;
> +		lru_refs_set_flags(&new_flags, min(refs, LRU_REFS_PROTECTED));
> +	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
> +
> +	/* Promote second page table access or executable */
> +	return refs > LRU_REFS_REFERENCED || is_exec_file_folio(folio, vma_flags);
>  }
>  #else
> -static bool lru_gen_set_refs(struct folio *folio, const vma_flags_t *vma_flags)
> +static bool folio_inc_lru_refs_isolated(struct folio *folio, const vma_flags_t *vma_flags)
>  {
>  	return false;
>  }
> @@ -896,7 +1035,8 @@ static enum folio_references folio_check_references(struct folio *folio,
>  		if (!referenced_ptes)
>  			return FOLIOREF_RECLAIM;
>  
> -		return lru_gen_set_refs(folio, &vma_flags) ? FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
> +		return folio_inc_lru_refs_isolated(folio, &vma_flags) ?
> +		       FOLIOREF_ACTIVATE : FOLIOREF_KEEP;
>  	}
>  
>  	referenced_folio = folio_test_clear_referenced(folio);
> @@ -3262,59 +3402,31 @@ static bool positive_ctrl_err(struct ctrl_pos *sp, struct ctrl_pos *pv)
>   *                          the aging
>   ******************************************************************************/
>  
> -/* promote pages accessed through page tables */
> -static int folio_update_gen(struct folio *folio, int new_gen, const vma_flags_t *vma_flags)
> -{
> -	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
> -	int old_gen;
> -
> -	/*
> -	 * See the comment on LRU_REFS_FLAGS, and activate file-backed
> -	 * executable folios after first usage to avoid typical IO
> -	 * thrashing from reclaiming.
> -	 */
> -	if (!folio_test_referenced(folio) && !folio_test_workingset(folio) &&
> -	    !is_exec_file_folio(folio, vma_flags)) {
> -		folio_set_lru_refs(folio, 1);
> -		return -1;
> -	}
> -
> -	do {
> -		old_gen = lru_gen_from_flags(old_flags);
> -		new_flags = old_flags;
> -
> -		/* lru_gen_del_folio() has isolated this page? */
> -		if (old_gen < 0)
> -			break;
> -
> -		lru_gen_set_flags(&new_flags, new_gen);
> -		lru_refs_set_flags(&new_flags, 0);
> -		new_flags |= BIT(PG_workingset);
> -	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
> -
> -	return old_gen;
> -}
> -
> -/* protect pages accessed multiple times through file descriptors */
> +/*
> + * Force bump a folio's generation. Used for PID protection or defer the
> + * eviction of temporarily unevictable folio.
> + */
>  static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
>  {
> +	int refs;
>  	int type = folio_is_file_lru(folio);
>  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
>  	int old_gen, new_gen, min_gen = lru_gen_from_seq(lrugen->min_seq[type]);
>  	unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0));
>  
>  	do {
> +		new_flags = old_flags;
> +		refs = lru_refs_from_flags(old_flags);
>  		old_gen = lru_gen_from_flags(old_flags);
>  		VM_WARN_ON_ONCE_FOLIO(old_gen < 0, folio);
>  
> -		/* folio_update_gen() has promoted this page? */
> +		/* folio has been promoted? */
>  		if (old_gen >= 0 && old_gen != min_gen)
>  			return old_gen;
>  
> -		new_flags = old_flags;
>  		new_gen = (old_gen + 1) % MAX_NR_GENS;
>  		lru_gen_set_flags(&new_flags, new_gen);
> -		lru_refs_set_flags(&new_flags, 0);
> +		lru_refs_set_flags(&new_flags, min(refs, LRU_REFS_WORKINGSET));
>  	} while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags));
>  
>  	lru_gen_update_size(lruvec, folio, old_gen, new_gen);
> @@ -3518,21 +3630,17 @@ static void walk_update_folio(struct lru_gen_mm_walk *walk, struct vm_area_struc
>  	if (!folio)
>  		return;
>  
> -	new_gen = lru_gen_from_seq(READ_ONCE(lruvec->lrugen.max_seq));
> -
>  	if (dirty && !folio_test_dirty(folio) &&
>  	    !(folio_test_anon(folio) && folio_test_swapbacked(folio) &&
>  	      !folio_test_swapcache(folio)))
>  		folio_mark_dirty(folio);
>  
>  	if (walk) {
> -		old_gen = folio_update_gen(folio, new_gen, &vma->flags);
> -		if (old_gen >= 0 && old_gen != new_gen)
> +		old_gen = folio_inc_lru_refs_walk(folio, lruvec, &vma->flags, &new_gen);
> +		if (old_gen >= 0)
>  			update_batch_size(walk, folio, old_gen, new_gen);
> -	} else if (lru_gen_set_refs(folio, &vma->flags)) {
> -		old_gen = folio_lru_gen(folio);
> -		if (old_gen >= 0 && old_gen != new_gen)
> -			folio_activate(folio);
> +	} else {
> +		folio_inc_lru_refs(folio, true, is_exec_file_folio(folio, &vma->flags));
>  	}
>  }
>  
> @@ -3917,7 +4025,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
>  		while (!list_empty(head)) {
>  			struct folio *folio = lru_to_folio(head);
>  			int refs = folio_lru_refs(folio);
> -			bool workingset = folio_test_workingset(folio);
> +			int delta = folio_nr_pages(folio);
> +			int tier = lru_tier_from_refs(refs);
>  
>  			VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio);
>  			VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio);
> @@ -3927,14 +4036,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
>  			new_gen = folio_inc_gen(lruvec, folio);
>  			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
>  
> -			/* don't count the workingset being lazily promoted */
> -			if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
> -				int tier = lru_tier_from_refs(refs, workingset);
> -				int delta = folio_nr_pages(folio);
> -
> -				WRITE_ONCE(lrugen->protected[hist][type][tier],
> -					   lrugen->protected[hist][type][tier] + delta);
> -			}
> +			WRITE_ONCE(lrugen->protected[hist][type][tier],
> +				   lrugen->protected[hist][type][tier] + delta);
>  
>  			if (!--remaining)
>  				return false;
> @@ -4649,8 +4752,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
>  	int zone = folio_zonenum(folio);
>  	int delta = folio_nr_pages(folio);
>  	int refs = folio_lru_refs(folio);
> -	bool workingset = folio_test_workingset(folio);
> -	int tier = lru_tier_from_refs(refs, workingset);
> +	int tier = lru_tier_from_refs(refs);
>  	struct lru_gen_folio *lrugen = &lruvec->lrugen;
>  
>  	VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio);
> @@ -4672,17 +4774,15 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
>  	}
>  
>  	/* protected */
> -	if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
> +	if (tier > tier_idx) {
> +		int hist = lru_hist_from_seq(lrugen->min_seq[type]);
> +
>  		gen = folio_inc_gen(lruvec, folio);
>  		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
>  
> -		/* don't count the workingset being lazily promoted */
> -		if (refs + workingset != BIT(LRU_REFS_WIDTH) + 1) {
> -			int hist = lru_hist_from_seq(lrugen->min_seq[type]);
> +		WRITE_ONCE(lrugen->protected[hist][type][tier],
> +			   lrugen->protected[hist][type][tier] + delta);
>  
> -			WRITE_ONCE(lrugen->protected[hist][type][tier],
> -				   lrugen->protected[hist][type][tier] + delta);
> -		}
>  		return true;
>  	}
>  
> @@ -4710,10 +4810,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
>  		return false;
>  	}
>  
> -	/* see the comment on LRU_REFS_FLAGS */
> -	if (!folio_test_referenced(folio))
> -		folio_set_lru_refs(folio, 0);
> -
>  	success = lru_gen_del_folio(lruvec, folio, true);
>  	VM_WARN_ON_ONCE_FOLIO(!success, folio);
>  
> @@ -4801,13 +4897,13 @@ static int get_tier_idx(struct lruvec *lruvec, int type)
>  	struct ctrl_pos sp, pv = {};
>  
>  	/*
> -	 * To leave a margin for fluctuations, use a larger gain factor (2:3).
> +	 * To leave a margin for fluctuations, use a larger gain factor (1:2).
>  	 * This value is chosen because any other tier would have at least twice
>  	 * as many refaults as the first tier.
>  	 */
> -	read_ctrl_pos(lruvec, type, 0, 1, 2, &sp);
>  	for (tier = 1; tier < MAX_NR_TIERS; tier++) {
> -		read_ctrl_pos(lruvec, type, tier, tier + 1, 3, &pv);
> +		read_ctrl_pos(lruvec, type, 0, tier, 1, &sp);
> +		read_ctrl_pos(lruvec, type, tier, tier + 1, 2, &pv);
>  		if (!positive_ctrl_err(&sp, &pv))
>  			break;
>  	}
> @@ -4930,10 +5026,8 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
>  		}
>  
>  		/* don't add rejected folios to the oldest generation */
> -		if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type]) {
> -			folio_set_lru_refs(folio, 0);
> +		if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
>  			folio_set_active(folio);
> -		}
>  	}
>  
>  	move_folios_to_lru(&list);
> diff --git a/mm/workingset.c b/mm/workingset.c
> index 5438e9390011..452fe8554990 100644
> --- a/mm/workingset.c
> +++ b/mm/workingset.c
> @@ -189,6 +189,13 @@
>  #define EVICTION_MASK	(~0UL >> EVICTION_SHIFT)
>  #define EVICTION_MASK_ANON	(~0UL >> EVICTION_SHIFT_ANON)
>  
> +/*
> + * LRU refs uses LRU_REFS_WIDTH + 2 bits, the 2 bits are PG_workingset and
> + * PG_referenced. But here we record PG_workingset separately (to reuse
> + * pack_shadow).
> + */
> +#define LRU_REFS_BITS ((LRU_REFS_WIDTH + 2) - 1)
> +
>  /*
>   * Eviction timestamps need to be able to cover the full range of
>   * actionable refaults. However, bits are tight in the xarray
> @@ -242,13 +249,12 @@ static void *lru_gen_eviction(struct folio *folio)
>  	int type = folio_is_file_lru(folio);
>  	int delta = folio_nr_pages(folio);
>  	int refs = folio_lru_refs(folio);
> -	bool workingset = folio_test_workingset(folio);
> -	int tier = lru_tier_from_refs(refs, workingset);
> +	int tier = lru_tier_from_refs(refs);
>  	struct mem_cgroup *memcg;
>  	struct pglist_data *pgdat = folio_pgdat(folio);
>  	unsigned short memcg_id;
>  
> -	BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_WIDTH >
> +	BUILD_BUG_ON(LRU_GEN_WIDTH + LRU_REFS_BITS >
>  		     BITS_PER_LONG - max(EVICTION_SHIFT, EVICTION_SHIFT_ANON));
>  
>  	rcu_read_lock();
> @@ -256,14 +262,14 @@ static void *lru_gen_eviction(struct folio *folio)
>  	lruvec = mem_cgroup_lruvec(memcg, pgdat);
>  	lrugen = &lruvec->lrugen;
>  	min_seq = READ_ONCE(lrugen->min_seq[type]);
> -	token = (min_seq << LRU_REFS_WIDTH) | max(refs - 1, 0);
> +	token = (min_seq << LRU_REFS_BITS) | refs >> 1;
>  
>  	hist = lru_hist_from_seq(min_seq);
>  	atomic_long_add(delta, &lrugen->evicted[hist][type][tier]);
>  	memcg_id = mem_cgroup_private_id(memcg);
>  	rcu_read_unlock();
>  
> -	return pack_shadow(memcg_id, pgdat, token, workingset, type);
> +	return pack_shadow(memcg_id, pgdat, token, refs & 1, type);
>  }
>  
>  /*
> @@ -284,11 +290,24 @@ static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
>  	*lruvec = mem_cgroup_lruvec(memcg, pgdat);
>  
>  	max_seq = READ_ONCE((*lruvec)->lrugen.max_seq);
> -	max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_WIDTH;
> +	max_seq &= (file ? EVICTION_MASK : EVICTION_MASK_ANON) >> LRU_REFS_BITS;
>  
> -	return abs_diff(max_seq, *token >> LRU_REFS_WIDTH) < MAX_NR_GENS;
> +	return abs_diff(max_seq, *token >> LRU_REFS_BITS) < MAX_NR_GENS;
>  }
>  
> +/*
> + * Restore the refs of a refaulted folio from its shadow entry.
> + *
> + * Any folio that was accessed at least once before eviction (refs >=
> + * LRU_REFS_REFERENCED) is activated on a fault-driven refault, giving it a
> + * strong gen placement. Non-fault refaults (e.g. readahead) are not
> + * activated regardless of refs.
> + *
> + * The restored refs is capped at LRU_REFS_PROTECTED to prevent stale
> + * high-tier history from carrying over across eviction cycles. The
> + * WORKINGSET_RESTORE stat is bumped only for refs >= LRU_REFS_WORKINGSET
> + * to track genuine workingset restoration.
> + */
>  static void lru_gen_refault(struct folio *folio, void *shadow)
>  {
>  	bool recent;
> @@ -314,21 +333,29 @@ static void lru_gen_refault(struct folio *folio, void *shadow)
>  	lrugen = &lruvec->lrugen;
>  
>  	hist = lru_hist_from_seq(READ_ONCE(lrugen->min_seq[type]));
> -	refs = (token & (BIT(LRU_REFS_WIDTH) - 1)) + 1;
> -	tier = lru_tier_from_refs(refs, workingset);
> +	refs = ((token & (BIT(LRU_REFS_BITS) - 1)) << 1) + workingset;
> +	tier = lru_tier_from_refs(refs);
>  
>  	atomic_long_add(delta, &lrugen->refaulted[hist][type][tier]);
>  
> -	if (workingset) {
> -		/* Send refaulted workingset folios to active generations. */
> +	/*
> +	 * Activate a fault-driven refault: the folio was accessed at
> +	 * least once before eviction and would have been promoted had
> +	 * it stayed in memory.
> +	 */
> +	if (refs >= LRU_REFS_REFERENCED) {
>  		if (lru_gen_in_fault()) {
>  			folio_set_active(folio);
>  			mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
>  		}
> -		folio_set_workingset(folio);
> +		/* Cap restored refs to prevent stale high-tier carry-over */
> +		folio_set_lru_refs(folio, min(refs, LRU_REFS_PROTECTED));
> +	}
> +
> +	/* WORKINGSET_RESTORE tracks genuine workingset-level refaults */
> +	if (refs >= LRU_REFS_WORKINGSET)
>  		mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
> -	} else
> -		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, (refs - 1UL) << LRU_REFS_PGOFF);
> +
>  unlock:
>  	rcu_read_unlock();
>  }
> 
> -- 
> 2.55.0
> 
>