[newlib-cygwin/main] Cygwin: dlfcn: fix dlopen refcounting for DLLs loaded with RTLD_NODELETE

https://sourceware.org/git/gitweb.cgi?p=newlib-cygwin.git;h=1c4fc31c48a0c7cc6de122c50e25e618c169c343

commit 1c4fc31c48a0c7cc6de122c50e25e618c169c343
Author:     Corinna Vinschen <[email protected]>
AuthorDate: Sun Mar 30 20:42:07 2025 +0200
Commit:     Corinna Vinschen <[email protected]>
CommitDate: Sun Mar 30 20:52:34 2025 +0200

    Cygwin: dlfcn: fix dlopen refcounting for DLLs loaded with RTLD_NODELETE
    
    Refcounting is performed even if the DLL is loaded at least once
    with RTLD_NODELETE.  In that case we must not decrement the count
    on dlclose, even if FreeLibrary will never unload the DLL.
    
    Use INT_MAX as dll count if RTLD_NODELETE has been used at least once.
    Skip any further refcounting and make sure the DLL is still pinned
    in memory after fork.
    
    Fixes: 33297d810d90 ("Cygwin: dlfcn: Fix reference counting")
    Signed-off-by: Corinna Vinschen <[email protected]>

Diff:
---
 winsup/cygwin/dlfcn.cc      | 15 +++++++++++----
 winsup/cygwin/dll_init.cc   | 16 +++++++++++++---
 winsup/cygwin/release/3.6.1 |  2 ++
 3 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/winsup/cygwin/dlfcn.cc b/winsup/cygwin/dlfcn.cc
index f98c7db475f7..a1e121ff333c 100644
--- a/winsup/cygwin/dlfcn.cc
+++ b/winsup/cygwin/dlfcn.cc
@@ -278,7 +278,13 @@ dlopen (const char *name, int flags)
 	{
 	  dll *d = dlls.find (ret, true);
 	  if (d)
-	    ++d->count;
+	    {
+	      /* count == INT_MIN is used to specify RTLD_NODELETE */
+	      if (d->count == INT_MIN || gmheflags)
+		d->count = INT_MIN;
+	      else
+		++d->count;
+	    }
 	}
 
       if (ret && gmheflags)
@@ -348,14 +354,15 @@ dlclose (void *handle)
   int ret = 0;
   if (handle != GetModuleHandle (NULL))
     {
-      /* reference counting */
+      /* Reference counting.
+	 count == INT_MIN is used to specify RTLD_NODELETE */
       dll *d = dlls.find (handle, true);
-      if (!d || d->count <= 0)
+      if (!d || (d->count <= 0 && d->count != INT_MIN))
 	{
 	  errno = ENOENT;
 	  ret = -1;
 	}
-      else
+      else if (d->count != INT_MIN)
 	{
 	  --d->count;
 	  if (!FreeLibrary ((HMODULE) handle))
diff --git a/winsup/cygwin/dll_init.cc b/winsup/cygwin/dll_init.cc
index c6fb94a43379..aab11d1e9d3d 100644
--- a/winsup/cygwin/dll_init.cc
+++ b/winsup/cygwin/dll_init.cc
@@ -782,9 +782,19 @@ void dll_list::load_after_fork_impl (HANDLE parent, dll* d, int retries)
       if (h != d->handle)
 	fabort ("unable to map %W (using %W) to same address as parent: %p != %p",
 		d->ntname, buffered_shortname (d->forkedntname ()), d->handle, h);
-      /* Fix OS reference count. */
-      for (int cnt = 1; cnt < d->count; ++cnt)
-	LoadLibraryW (buffered_shortname (d->forkedntname ()));
+      /* Fix OS reference count.
+	 count == INT_MIN is used to specify RTLD_NODELETE.  If so, we don't
+	 have to call LoadLibraryW count times, just mark the DLL as pinned. */
+      if (d->count == INT_MIN) /* RTLD_NODELETE */
+	{
+	  HMODULE hm;
+	  GetModuleHandleExW (GET_MODULE_HANDLE_EX_FLAG_PIN,
+			      buffered_shortname (d->forkedntname ()),
+			      &hm);
+	}
+      else
+	for (int cnt = 1; cnt < d->count; ++cnt)
+	  LoadLibraryW (buffered_shortname (d->forkedntname ()));
     }
 }
 
diff --git a/winsup/cygwin/release/3.6.1 b/winsup/cygwin/release/3.6.1
index 85c3f6c0554b..c1dbbfb24ae8 100644
--- a/winsup/cygwin/release/3.6.1
+++ b/winsup/cygwin/release/3.6.1
@@ -18,3 +18,5 @@ Fixes:
 
 - Fix reference counting when dlopen/dlclose is called on the Cygwin DLL.
   Addresses: https://cygwin.com/pipermail/cygwin/2025-March/257783.html
+
+- Fix reference counting when dlopen/dlclose a DLL with RTLD_NODELETE.