Re: [PATCH v8 08/15] hugetlb: Use the has_hwpoisoned flag

On 7/31/2026 1:07 PM, Matthew Wilcox (Oracle) wrote:
> Other large folios use the has_hwpoisoned flag. Convert hugetlb to=20
> match. This will help us use the per-page hwpoison flag in the future.=20
> Also introduce a folio_test_huge_poison(). This has exactly the same=20
> meaning as folio_test_has_hwpoisoned()
>=20
>=20
> Other large folios use the has_hwpoisoned flag.  Convert hugetlb to match.
> This will help us use the per-page hwpoison flag in the future.
>=20
> Also introduce a folio_test_huge_poison().  This has exactly the same
> meaning as folio_test_has_hwpoisoned() but can be used when we don't have
> a reference to the folio containing the page.  folio_test_has_hwpoisoned()
> can race with folio splitting / reallocation and trip the assertions
> in const_folio_flags().
>=20
> This closes a gap where a page in a previously-poisoned hugetlb folio
> could be observed to not have the hwpoison bit set.
>=20
> Signed-off-by: Matthew Wilcox (Oracle) <[email protected]>
> ---
>   fs/Kconfig                 |  2 +-
>   fs/hugetlbfs/inode.c       |  2 +-
>   include/linux/page-flags.h | 46 ++++++++++++++++++++++++------
>   mm/Kconfig                 |  6 +++-
>   mm/hugetlb.c               | 10 +++----
>   mm/memory-failure.c        | 58 +++++++++++++++++++++++++++-----------
>   mm/rmap.c                  | 43 +++++++++++++++++-----------
>   7 files changed, 119 insertions(+), 48 deletions(-)
>=20
> diff --git a/fs/Kconfig b/fs/Kconfig
> index cf6ae64776e6..eddac4ed214b 100644
> --- a/fs/Kconfig
> +++ b/fs/Kconfig
> @@ -272,7 +272,7 @@ endif # HUGETLBFS
>  =20
>   config HUGETLB_PAGE
>   	def_bool HUGETLBFS
> -	select XARRAY_MULTI
> +	select LARGE_FOLIO
>  =20
>   config HUGETLB_PAGE_OPTIMIZE_VMEMMAP
>   	def_bool HUGETLB_PAGE
> diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> index 216e1a0dd0b2..fbac554886c3 100644
> --- a/fs/hugetlbfs/inode.c
> +++ b/fs/hugetlbfs/inode.c
> @@ -258,7 +258,7 @@ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb=
, struct iov_iter *to)
>   		} else {
>   			folio_unlock(folio);
>  =20
> -			if (!folio_test_hwpoison(folio))
> +			if (!folio_test_has_hwpoisoned(folio))
>   				want =3D nr;
>   			else {
>   				/*
> diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
> index 4185a03a45cf..4fab3fbfd430 100644
> --- a/include/linux/page-flags.h
> +++ b/include/linux/page-flags.h
> @@ -893,14 +893,19 @@ static inline int PageTransCompound(const struct pa=
ge *page)
>   TESTPAGEFLAG_FALSE(TransCompound, transcompound)
>   #endif
>  =20
> -#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAG=
E)
> +#if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_LARGE_FOLIO)
>   /*
> - * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned i=
n the
> - * compound page.
> + * folio_has_hwpoisoned indicates that at least one page is hwpoisoned i=
n the
> + * folio.  That page will usually also have the HWPoison flag set, but t=
his
> + * is not possible for folios which have HVO (see memory-failure for the
> + * scheme used in that case).  You probably don't want to call this dire=
ctly;
> + * use folio_has_hwpoisoned_page() instead.
>    *
>    * This flag is set by hwpoison handler.  Cleared by THP split or free =
page.
>    */
>   FOLIO_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE)
> +FOLIO_TEST_SET_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE)
> +FOLIO_TEST_CLEAR_FLAG(has_hwpoisoned, FOLIO_SECOND_PAGE)
>   #else
>   FOLIO_FLAG_FALSE(has_hwpoisoned)
>   #endif
> @@ -1041,8 +1046,29 @@ PAGE_TYPE_OPS(Slab, slab, slab)
>  =20
>   #ifdef CONFIG_HUGETLB_PAGE
>   FOLIO_TYPE_OPS(hugetlb, hugetlb)
> +
> +#ifdef CONFIG_MEMORY_FAILURE
> +static inline bool folio_test_huge_poison(const struct folio *folio)
> +{
> +	return (READ_ONCE(folio->page.page_type) >> 23) =3D=3D
> +		((PGTY_hugetlb << 1) | 1);
> +}
> +
> +static inline void folio_set_huge_poison(struct folio *folio)
> +{
> +	folio->page.page_type |=3D (1 << 23);
> +}
> +
> +static inline void folio_clear_huge_poison(struct folio *folio)
> +{
> +	folio->page.page_type &=3D ~(1 << 23);
> +}
> +#else
> +FOLIO_TEST_FLAG_FALSE(huge_poison)
> +#endif
>   #else
>   FOLIO_TEST_FLAG_FALSE(hugetlb)
> +FOLIO_TEST_FLAG_FALSE(huge_poison)
>   #endif
>  =20
>   PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
> @@ -1069,9 +1095,10 @@ static inline bool PageHuge(const struct page *pag=
e)
>   }
>  =20
>   /*
> - * Check if a page is currently marked HWPoisoned. Note that this check =
is
> - * best effort only and inherently racy: there is no way to synchronize =
with
> - * failing hardware.
> + * Check if a page is currently marked HWPoisoned.  This check is best
> + * effort only and inherently racy: there is no way to synchronize with
> + * failing hardware.  The caller may not have a refcount on the folio
> + * containing the page, so we must be careful to not trip any assertions.
>    */
>   static inline bool is_page_hwpoison(const struct page *page)
>   {
> @@ -1080,12 +1107,15 @@ static inline bool is_page_hwpoison(const struct =
page *page)
>   	if (PageHWPoison(page))
>   		return true;
>   	folio =3D page_folio(page);
> -	return folio_test_hugetlb(folio) && PageHWPoison(&folio->page);
> +	if (folio_test_huge_poison(folio))
> +		return true;
> +	/* In case we raced with hugetlb transferring flags */
> +	return PageHWPoison(page);
>   }
>  =20
>   static inline bool folio_has_hwpoisoned_page(const struct folio *folio)
>   {
> -	return folio_test_hwpoison(folio) ||
> +	return PageHWPoison(&folio->page) ||
>   	       (folio_test_large(folio) && folio_test_has_hwpoisoned(folio));
>   }
>  =20
> diff --git a/mm/Kconfig b/mm/Kconfig
> index 9e0ca4824905..e666dd14ca0c 100644
> --- a/mm/Kconfig
> +++ b/mm/Kconfig
> @@ -843,11 +843,15 @@ config PERSISTENT_HUGE_ZERO_FOLIO
>   config MM_ID
>   	def_bool n
>  =20
> +config LARGE_FOLIO
> +	def_bool n
> +	select XARRAY_MULTI
> +
>   menuconfig TRANSPARENT_HUGEPAGE
>   	bool "Transparent Hugepage Support"
>   	depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT
>   	select COMPACTION
> -	select XARRAY_MULTI
> +	select LARGE_FOLIO
>   	select MM_ID
>   	help
>   	  Transparent Hugepages allows the kernel to use huge pages and
> diff --git a/mm/hugetlb.c b/mm/hugetlb.c
> index 47403d02be88..40ae967b9ecc 100644
> --- a/mm/hugetlb.c
> +++ b/mm/hugetlb.c
> @@ -1255,7 +1255,7 @@ static struct folio *dequeue_hugetlb_folio_node_exa=
ct(struct hstate *h,
>   		if (pin && !folio_is_longterm_pinnable(folio))
>   			continue;
>  =20
> -		if (folio_test_hwpoison(folio))
> +		if (folio_test_has_hwpoisoned(folio))
>   			continue;
>  =20
>   		if (is_migrate_isolate_page(&folio->page))
> @@ -1387,7 +1387,7 @@ static void folio_clear_hugetlb(struct folio *folio)
>   	 * Move HWPoison flag to each error page
>   	 * which makes any healthy pages reusable.
>   	 */
> -	if (unlikely(folio_test_hwpoison(folio)))
> +	if (unlikely(folio_test_has_hwpoisoned(folio)))
>   		folio_clear_hugetlb_hwpoison(folio);
>  =20
>   	__folio_clear_hugetlb(folio);
> @@ -4003,7 +4003,7 @@ long demote_pool_huge_page(struct hstate *src, node=
mask_t *nodes_allowed,
>   		struct folio *folio, *next;
>  =20
>   		list_for_each_entry_safe(folio, next, &src->hugepage_freelists[node],=
 lru) {
> -			if (folio_test_hwpoison(folio))
> +			if (folio_test_has_hwpoisoned(folio))
>   				continue;
>  =20
>   			remove_hugetlb_folio(src, folio, false);
> @@ -5814,7 +5814,7 @@ static vm_fault_t hugetlb_no_page(struct address_sp=
ace *mapping,
>   		 * don't have hwpoisoned swap entry for errored virtual address.
>   		 * So we need to block hugepage fault by PG_hwpoison bit check.
>   		 */
> -		if (unlikely(folio_test_hwpoison(folio))) {
> +		if (unlikely(folio_test_has_hwpoisoned(folio))) {
>   			ret =3D VM_FAULT_HWPOISON_LARGE |
>   				VM_FAULT_SET_HINDEX(hstate_index(h));
>   			goto backout_unlocked;
> @@ -6323,7 +6323,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
>   	ptl =3D huge_pte_lock(h, dst_mm, dst_pte);
>  =20
>   	ret =3D -EIO;
> -	if (folio_test_hwpoison(folio))
> +	if (folio_test_has_hwpoisoned(folio))
>   		goto out_release_unlock;
>  =20
>   	ret =3D -EEXIST;
> diff --git a/mm/memory-failure.c b/mm/memory-failure.c
> index 1dd0e7b99bb1..714e1b398f2c 100644
> --- a/mm/memory-failure.c
> +++ b/mm/memory-failure.c
> @@ -1829,7 +1829,7 @@ bool is_raw_hwpoison_page_in_hugepage(struct page *=
page)
>   	struct folio *folio =3D page_folio(page);
>   	bool ret =3D false;
>  =20
> -	if (!folio_test_hwpoison(folio))
> +	if (!folio_test_has_hwpoisoned(folio))
>   		return false;
>  =20
>   	if (!folio_test_hugetlb(folio))
> @@ -1881,6 +1881,24 @@ static unsigned long __folio_free_raw_hwp(struct f=
olio *folio, bool move_flag)
>   #define	MF_HUGETLB_FOLIO_PRE_POISONED	3	/* folio already poisoned */
>   #define	MF_HUGETLB_PAGE_PRE_POISONED	4	/* exact page already poisoned */
>   #define	MF_HUGETLB_RETRY		5	/* hugepage is busy, retry */
> +
> +static inline int hugetlb_set_poison(struct folio *folio)
> +{
> +	if (folio_test_set_has_hwpoisoned(folio))
> +		return MF_HUGETLB_FOLIO_PRE_POISONED;
> +	folio_set_huge_poison(folio);
> +	return 0;
> +}
> +
> +static inline int hugetlb_clear_poison(struct folio *folio)
> +{
> +	if (!folio_test_has_hwpoisoned(folio))
> +		return -EBUSY;
> +	folio_clear_huge_poison(folio);
> +	folio_clear_has_hwpoisoned(folio);
> +	return 0;
> +}
> +
>   /*
>    * Set hugetlb folio as hwpoisoned, update folio private raw hwpoison l=
ist
>    * to keep track of the poisoned pages.
> @@ -1890,12 +1908,12 @@ static int hugetlb_update_hwpoison(struct folio *=
folio, struct page *page)
>   	struct llist_head *head;
>   	struct raw_hwp_page *raw_hwp;
>   	struct raw_hwp_page *p;
> -	int ret =3D folio_test_set_hwpoison(folio) ? MF_HUGETLB_FOLIO_PRE_POISO=
NED : 0;
> +	int ret =3D hugetlb_set_poison(folio);
>  =20
>   	/*
>   	 * Once the hwpoison hugepage has lost reliable raw error info,
> -	 * there is little meaning to keep additional error info precisely,
> -	 * so skip to add additional raw error info.
> +	 * there is no point in keeping additional error info precisely,
> +	 * so skip adding additional raw error info.
>   	 */
>   	if (folio_test_hugetlb_raw_hwp_unreliable(folio))
>   		return MF_HUGETLB_FOLIO_PRE_POISONED;
> @@ -1950,8 +1968,8 @@ void folio_clear_hugetlb_hwpoison(struct folio *fol=
io)
>   		return;
>   	if (folio_test_hugetlb_vmemmap_optimized(folio))
>   		return;
> -	folio_clear_hwpoison(folio);
>   	folio_free_raw_hwp(folio, true);
> +	folio_clear_has_hwpoisoned(folio);
>   }
>  =20
>   static int get_huge_page_for_hwpoison(unsigned long pfn, int flags,
> @@ -2104,6 +2122,11 @@ static inline unsigned long folio_free_raw_hwp(str=
uct folio *folio, bool flag)
>   {
>   	return 0;
>   }
> +
> +static inline int hugetlb_clear_poison(struct folio *folio)
> +{
> +	return 0;
> +}
>   #endif	/* CONFIG_HUGETLB_PAGE */
>  =20
>   /* Drop the extra refcount in case we come from madvise() */
> @@ -2733,8 +2756,10 @@ int unpoison_memory(unsigned long pfn)
>   				hugetlb_unlock_irq();
>   				goto unlock_mutex;
>   			}
> +			ret =3D hugetlb_clear_poison(folio);
> +		} else {
> +			ret =3D TestClearPageHWPoison(p) ? 0 : -EBUSY;
>   		}
> -		ret =3D folio_test_clear_hwpoison(folio) ? 0 : -EBUSY;
>   		hugetlb_unlock_irq();
>   	} else if (ghp < 0) {
>   		if (ghp =3D=3D -EHWPOISON) {
> @@ -2744,17 +2769,18 @@ int unpoison_memory(unsigned long pfn)
>   			unpoison_pr_info("%#lx: failed to grab page\n",
>   					 pfn, &unpoison_rs);
>   		}
> -	} else {
> -		if (folio_test_hugetlb(folio)) {
> -			huge =3D true;
> -			count =3D folio_free_raw_hwp(folio, false);
> -			if (count =3D=3D 0) {
> -				folio_put(folio);
> -				goto unlock_mutex;
> -			}
> -			p =3D &folio->page;
> +	} else if (folio_test_hugetlb(folio)) {
> +		huge =3D true;
> +		count =3D folio_free_raw_hwp(folio, false);
> +		if (count =3D=3D 0) {
> +			folio_put(folio);
> +			goto unlock_mutex;
>   		}
> -
> +		ret =3D hugetlb_clear_poison(folio);
> +		folio_put(folio);
> +		if (!ret)
> +			folio_put(folio);
> +	} else {
>   		folio_put(folio);
>   		if (TestClearPageHWPoison(p)) {
>   			folio_put(folio);
> diff --git a/mm/rmap.c b/mm/rmap.c
> index 1c77d5dc06e9..fd19a0bfbfe7 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -1978,6 +1978,22 @@ static inline unsigned int folio_unmap_pte_batch(s=
truct folio *folio,
>   				     FPB_RESPECT_WRITE | FPB_RESPECT_SOFT_DIRTY);
>   }
>  =20
> +/*
> + * Since we cannot split a hugetlb folio, we want to insert a poison
> + * entry into the page table for the whole folio even if only one page
> + * is poisoned.  Otherwise, we've split down to the PTE level and we only
> + * want to poison the precise page
> + */
> +static bool ttu_create_hwpoison(const struct folio *folio,
> +		const struct page *page, enum ttu_flags flags)
> +{
> +	if (!(flags & TTU_HWPOISON))
> +		return false;
> +	if (folio_test_hugetlb(folio))
> +		return folio_test_has_hwpoisoned(folio);
> +	return PageHWPoison(page);
> +}
> +
>   /*
>    * @arg: enum ttu_flags will be passed to this argument
>    */
> @@ -1993,7 +2009,6 @@ static bool try_to_unmap_one(struct folio *folio, s=
truct vm_area_struct *vma,
>   	enum ttu_flags flags =3D (enum ttu_flags)(long)arg;
>   	unsigned long nr_pages =3D 1, end_addr;
>   	unsigned long pfn;
> -	unsigned long hsz =3D 0;
>   	int ptes =3D 0;
>  =20
>   	/*
> @@ -2023,9 +2038,6 @@ static bool try_to_unmap_one(struct folio *folio, s=
truct vm_area_struct *vma,
>   		 */
>   		adjust_range_if_pmd_sharing_possible(vma, &range.start,
>   						     &range.end);
> -
> -		/* We need the huge page size for set_huge_pte_at() */
> -		hsz =3D huge_page_size(hstate_vma(vma));
>   	}
>   	mmu_notifier_invalidate_range_start(&range);
>  =20
> @@ -2121,7 +2133,8 @@ static bool try_to_unmap_one(struct folio *folio, s=
truct vm_area_struct *vma,
>   			 * The try_to_unmap() is only passed a hugetlb page
>   			 * in the case where the hugetlb page is poisoned.
>   			 */
> -			VM_BUG_ON_PAGE(!PageHWPoison(subpage), subpage);
> +			VM_BUG_ON_FOLIO(!folio_has_hwpoisoned_page(folio),
> +					folio);
>   			/*
>   			 * huge_pmd_unshare may unmap an entire PMD page.
>   			 * There is no way of knowing exactly which PMDs may
> @@ -2200,12 +2213,12 @@ static bool try_to_unmap_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   		/* Update high watermark before we lower rss */
>   		update_hiwater_rss(mm);
>  =20
> -		if (PageHWPoison(subpage) && (flags & TTU_HWPOISON)) {
> +		if (ttu_create_hwpoison(folio, subpage, flags)) {
>   			pteval =3D swp_entry_to_pte(make_hwpoison_entry(subpage));
>   			if (folio_test_hugetlb(folio)) {
>   				hugetlb_count_sub(folio_nr_pages(folio), mm);
>   				set_huge_pte_at(mm, address, pvmw.pte, pteval,
> -						hsz);
> +						folio_size(folio));
>   			} else {
>   				dec_mm_counter(mm, mm_counter(folio));
>   				set_pte_at(mm, address, pvmw.pte, pteval);
> @@ -2423,7 +2436,6 @@ static bool try_to_migrate_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   	struct mmu_notifier_range range;
>   	enum ttu_flags flags =3D (enum ttu_flags)(long)arg;
>   	unsigned long pfn;
> -	unsigned long hsz =3D 0;
>  =20
>   	/*
>   	 * When racing against e.g. zap_pte_range() on another cpu,
> @@ -2452,9 +2464,6 @@ static bool try_to_migrate_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   		 */
>   		adjust_range_if_pmd_sharing_possible(vma, &range.start,
>   						     &range.end);
> -
> -		/* We need the huge page size for set_huge_pte_at() */
> -		hsz =3D huge_page_size(hstate_vma(vma));
>   	}
>   	mmu_notifier_invalidate_range_start(&range);
>  =20
> @@ -2607,14 +2616,14 @@ static bool try_to_migrate_one(struct folio *foli=
o, struct vm_area_struct *vma,
>   		/* Update high watermark before we lower rss */
>   		update_hiwater_rss(mm);
>  =20
> -		if (PageHWPoison(subpage)) {
> +		if (ttu_create_hwpoison(folio, subpage, TTU_HWPOISON)) {
>   			VM_WARN_ON_FOLIO(folio_is_device_private(folio), folio);
>  =20
>   			pteval =3D swp_entry_to_pte(make_hwpoison_entry(subpage));
>   			if (folio_test_hugetlb(folio)) {
>   				hugetlb_count_sub(folio_nr_pages(folio), mm);
>   				set_huge_pte_at(mm, address, pvmw.pte, pteval,
> -						hsz);
> +						folio_size(folio));
>   			} else {
>   				dec_mm_counter(mm, mm_counter(folio));
>   				set_pte_at(mm, address, pvmw.pte, pteval);
> @@ -2644,7 +2653,8 @@ static bool try_to_migrate_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   			if (arch_unmap_one(mm, vma, address, pteval) < 0) {
>   				if (folio_test_hugetlb(folio))
>   					set_huge_pte_at(mm, address, pvmw.pte,
> -							pteval, hsz);
> +							pteval,
> +							folio_size(folio));
>   				else
>   					set_pte_at(mm, address, pvmw.pte, pteval);
>   				ret =3D false;
> @@ -2657,7 +2667,8 @@ static bool try_to_migrate_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   				if (anon_exclusive &&
>   				    hugetlb_try_share_anon_rmap(folio)) {
>   					set_huge_pte_at(mm, address, pvmw.pte,
> -							pteval, hsz);
> +							pteval,
> +							folio_size(folio));
>   					ret =3D false;
>   					page_vma_mapped_walk_done(&pvmw);
>   					break;
> @@ -2703,7 +2714,7 @@ static bool try_to_migrate_one(struct folio *folio,=
 struct vm_area_struct *vma,
>   			}
>   			if (folio_test_hugetlb(folio))
>   				set_huge_pte_at(mm, address, pvmw.pte, swp_pte,
> -						hsz);
> +						folio_size(folio));
>   			else
>   				set_pte_at(mm, address, pvmw.pte, swp_pte);
>   			trace_set_migration_pte(address, pte_val(swp_pte),
> --=20
> 2.47.3
>=20

So, we now have two sets keeping track of folio level hugetlb hwpoison=20
status: one with PG_has_hwpoison, another with (PGTY_hugetlb + bit23) in=20
the .page_type field.  I don't see a better way though.

Reviewed-by: Jane Chu <[email protected]>

thanks,
-jane