[glibc] malloc: Don't call __get_thp_mode/__get_thp_size twice

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=ccd4cd5324caaaea1bb12bc8cef073170da51a19

commit ccd4cd5324caaaea1bb12bc8cef073170da51a19
Author: H.J. Lu <[email protected]>
Date:   Sat Apr 18 11:36:41 2026 +0800

    malloc: Don't call __get_thp_mode/__get_thp_size twice
    
    Both ld.so and malloc track kernel THP mode and THP page size when THP
    in ld.so is enabled by
    
    GLIBC_TUNABLES=glibc.elf.thp=1
    
    and THP in malloc is enabled by
    
    GLIBC_TUNABLES=glibc.malloc.hugetlb=1
    
    But DL_MAP_DEFAULT_THP_PAGESIZE and MALLOC_DEFAULT_THP_PAGESIZE may be
    different when they are defined in <hugepages.h>.
    
    If THP in ld.so is enabled, change malloc to use kernel THP mode from
    ld.so, instead of calling __get_thp_mode, and use THP page size from
    ld.so if it came from __get_thp_size.  This avoids calling __get_thp_mode
    and __get_thp_size again, which are quite expensive:
    
    1. Initialize mp_.thp_mode and GL(dl_thp_mode) to thp_mode_unknown.
    2. Set mp_.thp_mode to GL(dl_thp_mode) if GL(dl_thp_mode) isn't
    thp_mode_unknown.  Otherwise call __get_thp_mode to set mp_.thp_mode.
    3. GL(dl_elf_thp_pagesize) is set to DL_MAP_DEFAULT_THP_PAGESIZE without
    calling __get_thp_size and THP page size for malloc may be different from
    THP page size for ld.so.  Set mp_.thp_pagesize to GL(dl_elf_thp_pagesize)
    if DL_MAP_DEFAULT_THP_PAGESIZE is defined.  Otherwise call __get_thp_size
    to set mp_.thp_pagesize.
    4. THP page size in malloc is capped to MAX_THP_PAGESIZE.  If THP page
    size is above MAX_THP_PAGESIZE, THP in malloc is disabled.
    
    These result in when glibc.elf.thp is set to 1, malloc uses the actual
    kernel THP mode instead of defaulting to madvise mode and madvise_thp
    will stop issuing MADV_HUGEPAGE if kernel THP mode is always.
    
    This fixes BZ #34083.
    
    Signed-off-by: H.J. Lu <[email protected]>

Diff:
---
 NEWS                                   |  9 ++++++
 malloc/malloc.c                        | 56 +++++++++++++++++++++++++++++++---
 manual/tunables.texi                   |  4 +++
 sysdeps/generic/hugepages.h            | 16 +++++++++-
 sysdeps/unix/sysv/linux/dl-exec-post.h |  9 +-----
 5 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/NEWS b/NEWS
index 41b59da04b..f9d90c5194 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,15 @@ Version 2.44
 
 Major new features:
 
+* A new tunable, glibc.elf.thp, is added to map read-only segments with
+  Transparent Huge Pages (THP) if THP isn't disable in kernel.  When
+  glibc.elf.thp is set to 1, malloc uses the actual kernel THP mode
+  instead of defaulting to madvise mode and madvise_thp will stop issuing
+  MADV_HUGEPAGE if kernel THP mode is always.
+
+* THP page size in malloc is capped to MAX_THP_PAGESIZE.  If THP page
+  size is above MAX_THP_PAGESIZE, THP in malloc is disabled.
+
 * Additional optimized and correctly rounded mathematical functions have
   been imported from the CORE-MATH project, in particular cosh, sinh, and
   tanh.
diff --git a/malloc/malloc.c b/malloc/malloc.c
index a354576aca..c39d60b509 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1529,7 +1529,7 @@ static struct malloc_par mp_ =
   .mmap_threshold = DEFAULT_MMAP_THRESHOLD,
   .trim_threshold = DEFAULT_TRIM_THRESHOLD,
   .arena_test = sizeof (long) == 4 ? 2 : 8,
-  .thp_mode = thp_mode_not_supported
+  .thp_mode = thp_mode_unknown
 #if USE_TCACHE
   ,
   .tcache_count = TCACHE_FILL_COUNT,
@@ -4657,13 +4657,41 @@ do_set_mxfast (size_t value)
   return 1;
 }
 
+#ifdef HAVE_THP
+static __always_inline enum thp_mode_t
+get_dl_thp_mode (void)
+{
+  return GL(dl_thp_mode);
+}
+
+static __always_inline unsigned long int
+get_dl_elf_thp_pagesize (void)
+{
+  return GL(dl_elf_thp_pagesize);
+}
+#else
+# define get_dl_thp_mode()		__get_thp_mode ()
+# define get_dl_elf_thp_pagesize()	__get_thp_size ()
+#endif
+
 static __always_inline int
 do_set_hugetlb (size_t value)
 {
+  /* If __get_thp_mode and __get_thp_size have been called during
+     startup, don't call them again here.  */
+  enum thp_mode_t thp_mode = get_dl_thp_mode ();
+
   /* Enable THP if MALLOC_DEFAULT_THP_PAGESIZE is non-zero.  */
   if (MALLOC_DEFAULT_THP_PAGESIZE > 0)
     {
-      mp_.thp_mode = thp_mode_madvise;
+      /* If thp_mode is unknown, THP segment load is disabled by
+	 GLIBC_TUNABLES=glibc.elf.thp=0.  In this case, set
+	 mp_.thp_mode to madvise.  Otherwise, set it to thp_mode to
+	 keep mp_.thp_mode in sync with GL(dl_thp_mode).  */
+      if (thp_mode == thp_mode_unknown)
+	mp_.thp_mode = thp_mode_madvise;
+      else
+	mp_.thp_mode = thp_mode;
       mp_.thp_pagesize = MALLOC_DEFAULT_THP_PAGESIZE;
     }
 
@@ -4680,9 +4708,27 @@ do_set_hugetlb (size_t value)
       if (MALLOC_DEFAULT_THP_PAGESIZE > 0)
 	return 0;
 
-      mp_.thp_mode = __get_thp_mode ();
-      if (mp_.thp_mode == thp_mode_madvise || mp_.thp_mode == thp_mode_always)
-	mp_.thp_pagesize = __get_thp_size ();
+      if (thp_mode == thp_mode_unknown)
+	{
+	  /* Call __get_thp_mode and __get_thp_size when THP segment load
+	     is disabled.  */
+	  mp_.thp_mode = __get_thp_mode ();
+	  if (mp_.thp_mode == thp_mode_madvise
+	      || mp_.thp_mode == thp_mode_always)
+	    mp_.thp_pagesize = __get_thp_size ();
+	}
+      else
+	{
+	  /* THP segment load is enabled.  GL(dl_elf_thp_pagesize) is
+	     set to DL_MAP_DEFAULT_THP_PAGESIZE if it isn't zero.  In
+	     this case, call get_capped_thp_size () instead of using
+	     DL_MAP_DEFAULT_THP_PAGESIZE for malloc.  */
+	  mp_.thp_mode = thp_mode;
+	  if (DL_MAP_DEFAULT_THP_PAGESIZE != 0)
+	    mp_.thp_pagesize = get_capped_thp_size ();
+	  else
+	    mp_.thp_pagesize = get_dl_elf_thp_pagesize ();
+	}
     }
   else if (value >= 2)
     __get_hugepage_config (value == 2 ? 0 : value, &mp_.hp_pagesize,
diff --git a/manual/tunables.texi b/manual/tunables.texi
index e0f141077b..e1d9fbae9c 100644
--- a/manual/tunables.texi
+++ b/manual/tunables.texi
@@ -255,6 +255,10 @@ Setting its value to @code{1} enables the use of @code{madvise} with
 only if the system supports Transparent Huge Page (currently only on Linux).
 This is the default used for AArch64.
 
+When @code{glibc.elf.thp} set to 1, malloc uses the actual kernel THP mode
+instead of defaulting to madvise mode and madvise_thp will stop issuing
+MADV_HUGEPAGE if kernel THP mode is always.
+
 Setting its value to @code{2} enables the use of Huge Page directly with
 @code{mmap} with the use of @code{MAP_HUGETLB} flag.  The huge page size
 to use will be the default one provided by the system.  A value larger than
diff --git a/sysdeps/generic/hugepages.h b/sysdeps/generic/hugepages.h
index 8e54661527..e8fc70fbf9 100644
--- a/sysdeps/generic/hugepages.h
+++ b/sysdeps/generic/hugepages.h
@@ -37,7 +37,8 @@ unsigned long int __get_thp_size (void) attribute_hidden;
 
 enum thp_mode_t
 {
-  thp_mode_not_supported = 0,
+  thp_mode_unknown = 0,
+  thp_mode_not_supported,
   thp_mode_always,
   thp_mode_madvise,
   thp_mode_never
@@ -64,4 +65,17 @@ void __get_hugepage_config (size_t requested, size_t *pagesize, int *flags)
 # define MAX_THP_PAGESIZE	(32 * 1024 * 1024)
 #endif
 
+/* If THP page size is above MAX_THP_PAGESIZE, return 0 to cap the THP
+   size at MAX_THP_PAGESIZE to avoid over-aligning on systems with very
+   large normal pages (like 64K pages with 512M huge pages).  */
+
+static inline unsigned long int
+get_capped_thp_size (void)
+{
+  unsigned long int size = __get_thp_size ();
+  if (size > MAX_THP_PAGESIZE)
+    size = 0;
+  return size;
+}
+
 #endif /* _HUGEPAGES_H */
diff --git a/sysdeps/unix/sysv/linux/dl-exec-post.h b/sysdeps/unix/sysv/linux/dl-exec-post.h
index dd8a379809..9a49486db2 100644
--- a/sysdeps/unix/sysv/linux/dl-exec-post.h
+++ b/sysdeps/unix/sysv/linux/dl-exec-post.h
@@ -48,14 +48,7 @@ _dl_get_thp_config (void)
       GL(dl_thp_mode) = __get_thp_mode ();
       if (GL(dl_thp_mode) == thp_mode_always
 	  || GL(dl_thp_mode) == thp_mode_madvise)
-	{
-	  GL(dl_elf_thp_pagesize) = __get_thp_size ();
-	  /* We cap the huge page size at MAX_THP_PAGESIZE to avoid
-	     over-aligning on systems with very large normal pages
-	     (like 64K pages with 512M huge pages).  */
-	  if (GL(dl_elf_thp_pagesize) > MAX_THP_PAGESIZE)
-	    GL(dl_elf_thp_pagesize) = 0;
-	}
+	GL(dl_elf_thp_pagesize) = get_capped_thp_size ();
       else
 	GL(dl_elf_thp_pagesize) = 0;