[newlib-cygwin/cygwin-3_6-branch] Cygwin: Ensure unused fd available for open()

https://sourceware.org/git/gitweb.cgi?p=3Dnewlib-cygwin.git;h=3D9dab20d03be=
12369a48c46ab1fb3ee76ba730203

commit 9dab20d03be12369a48c46ab1fb3ee76ba730203
Author: Mark Geisert <[email protected]>
Date:   Wed May 27 22:42:44 2026 -0700

    Cygwin: Ensure unused fd available for open()
   =20
    The existing logic for open() assumes an fd is always available in
    the fdtable for a created file.  This leads to a situation where, if
    there is no fd available due to the OPEN_MAX limit being hit, the
    file is created but cannot be referenced by a Cygwin fd.
   =20
    Move the fd reservation code to an earlier location within open().
   =20
    Reported-by: Christian Franke <[email protected]>
    Addresses: https://cygwin.com/pipermail/cygwin/2026-May/259664.html
    Signed-off-by: Mark Geisert <[email protected]>
    Fixes: e859706578ba (* autoload.cc (NtCreateFile): Add.)
    (cherry picked from commit 31bf91f867c5fadd7deb408cf06fe3af8e86bb74)

Diff:
---
 winsup/cygwin/syscalls.cc | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/winsup/cygwin/syscalls.cc b/winsup/cygwin/syscalls.cc
index 7a8e5d4fd..2bea79768 100644
--- a/winsup/cygwin/syscalls.cc
+++ b/winsup/cygwin/syscalls.cc
@@ -1547,6 +1547,13 @@ open (const char *unix_path, int flags, ...)
 	  fh =3D fh_file;
 	}
=20
+      /* Reserve an fdtable entry here, before calling open_with_arch() be=
low.
+         Otherwise there's a tiny chance of hitting OPEN_MAX further on wh=
ich
+         could create a new file without any way for Cygwin to refer to it=
. */
+      cygheap_fdnew fd;
+      if (fd < 0)
+        __leave;		/* errno already set */
+
       if (fh->dev () =3D=3D FH_PROCESSFD && fh->pc.follow_fd_symlink ())
 	{
 	  /* Reopen file by descriptor */
@@ -1573,14 +1580,6 @@ open (const char *unix_path, int flags, ...)
 	try_to_bin (fh->pc, fh->get_handle (), DELETE,
 		    FILE_OPEN_FOR_BACKUP_INTENT);
=20
-      cygheap_fdnew fd;
-
-      if (fd < 0)
-	{
-	  fh->close();
-	  __leave;		/* errno already set */
-	}
-
       fd =3D fh;
       if (fd <=3D 2)
 	set_std_handle (fd);