Re: [PATCH] mm/huge_memory: let special huge VMAs bypass the THP policy check

> From d6537260722c8741586e6295c8eea68d06087efa Mon Sep 17 00:00:00 2001
> From: "Lorenzo Stoakes (ARM)" <[email protected]>
> Date: Wed, 5 Aug 2026 11:35:13 +0100
> Subject: [PATCH] ideas
> 
> ---
>  mm/huge_memory.c | 32 +++++++++++++++++++++++++++++---
>  1 file changed, 29 insertions(+), 3 deletions(-)
> 
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index abc65d608c23..5fa01364f089 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -111,6 +111,34 @@ static bool vma_is_special_huge(const struct vm_area_struct *vma)
>  	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
>  }
> 
> +static bool should_obey_thp_file_tunables(const struct vm_area_struct *vma,
> +					  bool forced_collapse)
> +{> +	if (forced_collapse)
> +		return false;
> +	VM_WARN_ON_ONCE(vma_is_anonymous(vma));
> +	/* Huge PFN mappings allocate no folios so the policy doesn't apply. */
> +	if (vma_test(vma, VMA_PFNMAP_BIT) && vma->vm_ops->huge_fault)
> +		return false;
> +	return true;
> +}

If that's sufficient, then this is is the better direction.

> +> +static bool can_thp_collapse_file(const struct vm_area_struct *vma,
> +		vm_flags_t vm_flags, bool forced_collapse)
> +{
> +	/* Override THP tunables? */
> +	if (!should_obey_thp_file_tunables(vma, forced_collapse))
> +		return true;
> +	/* THP=always? */
> +	if (hugepage_global_always())
> +		return true;
> +	/* THP=madvise? */
> +	if (!hugepage_global_enabled())
> +		return false;
> +	/* Has VMA had madvise(..., MADV_HUGEPAGE) applied to it? */
> +	return vm_flags & VM_HUGEPAGE;
> +}
> +
>  unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
>  					 vm_flags_t vm_flags,
>  					 enum tva_type type,
> @@ -188,9 +216,7 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
>  		 * Enforce THP collapse requirements as necessary. Anonymous vmas
>  		 * were already handled in thp_vma_allowable_orders().
>  		 */
> -		if (!forced_collapse &&
> -		    (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
> -						    !hugepage_global_always())))
> +		if (!can_thp_collapse_file(vma, vm_flags, forced_collapse))
>  			return 0;

I played a bit with that and came up with the following. Not quite happy about
it, just for your inspiration on naming and what to split out.

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index a00df56a68b57..8ba77608959d4 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -111,6 +111,54 @@ static bool vma_is_special_huge(const struct vm_area_struct
*vma)
 	return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT);
 }

+static bool file_vma_honors_thp_toggles(struct vm_area_struct *vma,
+		enum tva_type type)
+{
+	const bool forced_collapse = type == TVA_FORCED_COLLAPSE;
+
+	if (forced_collapse)
+		return false;
+	/* Huge PFN mappings allocate no folios so the policy doesn't apply. */
+	return !(vma_test(vma, VMA_PFNMAP_BIT) && vma->vm_ops->huge_fault);
+}
+
+static bool vma_thp_toggles_enabled(struct vm_area_struct *vma,
+		vm_flags_t vm_flags)
+{
+	/* THP=always? */
+	if (hugepage_global_always())
+		return true;
+	/* THP=madvise and actually advised? */
+	return hugepage_global_enabled() && vm_flags & VM_HUGEPAGE;
+}
+
+static bool file_vma_forces_order_0(struct vm_area_struct *vma,
+		vm_flags_t vm_flags, enum tva_type type)
+{
+	const bool in_pf = type == TVA_PAGEFAULT;
+	const bool smaps = type == TVA_SMAPS;
+
+	/*
+	 * Enforce THP collapse requirements as necessary. Anonymous vmas
+	 * were already handled in thp_vma_allowable_orders().
+	 */
+
+	if (file_vma_honors_thp_toggles(vma, type) &&
+	    !vma_thp_toggles_enabled(vma, vm_flags))
+		return true;
+
+	/*
+	 * Trust that ->huge_fault() handlers know what they are doing
+	 * in fault path.
+	 */
+	if (((in_pf || smaps)) && vma->vm_ops->huge_fault)
+		return false;
+	/* Only regular file is valid in collapse path */
+	if (((!in_pf || smaps)) && file_thp_enabled(vma))
+		return false;
+	return true;
+}
+
 unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma,
 					 vm_flags_t vm_flags,
 					 enum tva_type type,
@@ -183,27 +231,8 @@ unsigned long __thp_vma_allowable_orders(struct
vm_area_struct *vma,
 						   vma, vma_start_pgoff(vma), 0,
 						   forced_collapse);

-	if (!vma_is_anonymous(vma)) {
-		/*
-		 * Enforce THP collapse requirements as necessary. Anonymous vmas
-		 * were already handled in thp_vma_allowable_orders().
-		 */
-		if (!forced_collapse &&
-		    (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) &&
-						    !hugepage_global_always())))
-			return 0;
-
-		/*
-		 * Trust that ->huge_fault() handlers know what they are doing
-		 * in fault path.
-		 */
-		if (((in_pf || smaps)) && vma->vm_ops->huge_fault)
-			return orders;
-		/* Only regular file is valid in collapse path */
-		if (((!in_pf || smaps)) && file_thp_enabled(vma))
-			return orders;
-		return 0;
-	}
+	if (!vma_is_anonymous(vma))
+		return __file_vma_forces_order_0(vma, vm_flags, type) ? 0 : orders;

 	if (vma_is_temporary_stack(vma))
 		return 0;

-- 
Cheers,

David