[newlib-cygwin/cygwin-3_6-branch] Cygwin: open: Unlock fdtab before open_with_arch()

https://sourceware.org/git/gitweb.cgi?p=3Dnewlib-cygwin.git;h=3D273e6617249=
4aa276f08155c12e6934f80abb10b

commit 273e66172494aa276f08155c12e6934f80abb10b
Author: Takashi Yano <[email protected]>
Date:   Fri Jul 17 10:48:11 2026 +0900

    Cygwin: open: Unlock fdtab before open_with_arch()
   =20
    Since the commit 31bf91f867c5, opening fifo causes a deadlock. This
    is because, open_with_arch() for fifo can be blocked until the other
    side of the fifo is opened. The commit 31bf91f867c5 moves the creating
    cygheap_fdnew before open_with_arch() to address the issue:
    https://cygwin.com/pipermail/cygwin/2026-May/259664.html
    However, cygheap_fdnew locks fdtab, so open() for the other side of
    fifo cannot create cygheap_fdnew until fdtab is unlocked. This is
    the cause of the deadlock.
   =20
    With this patch, fdtab is unlocked before open_with_arch(), but marked
    as used using tentative fhandler. The summary of open() is as follows.
     1) Lock fdtab.
     2) Create new fd.
     3) Mark fd as used using tentative fhandler.
     4) Unlock fdtab.
     5) Call open_with_arch().
     6) Set final fhandler to fd.
   =20
    The important point is that create fd before open_with_arch() to
    address https://cygwin.com/pipermail/cygwin/2026-May/259664.html,
    but unlock fdtab before open_with_arch() to address
    https://cygwin.com/pipermail/cygwin/2026-July/259884.html.
   =20
    Fixes: 31bf91f867c5 ("Cygwin: Ensure unused fd available for open()")
    Addresses: https://cygwin.com/pipermail/cygwin/2026-July/259884.html
    Reported-by: kikairoya <[email protected]>
    Signed-off-by: Takashi Yano <[email protected]>
    Reviewed-by: Mark Geisert <[email protected]>
    (cherry picked from commit 524d75ff73986b263161665af771cc90e55b5e01)

Diff:
---
 winsup/cygwin/release/3.6.11 |  5 +++++
 winsup/cygwin/syscalls.cc    | 20 +++++++++++++++++---
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/winsup/cygwin/release/3.6.11 b/winsup/cygwin/release/3.6.11
new file mode 100644
index 000000000..0c7046d58
--- /dev/null
+++ b/winsup/cygwin/release/3.6.11
@@ -0,0 +1,5 @@
+Fixes:
+------
+
+- Fix a deadlock in fifo open, that is a regression caused in 3.6.10
+  Addresses: https://cygwin.com/pipermail/cygwin/2026-July/259884.html
diff --git a/winsup/cygwin/syscalls.cc b/winsup/cygwin/syscalls.cc
index 2bea79768..e3ba8c65c 100644
--- a/winsup/cygwin/syscalls.cc
+++ b/winsup/cygwin/syscalls.cc
@@ -1451,6 +1451,7 @@ extern "C" int
 open (const char *unix_path, int flags, ...)
 {
   int res =3D -1;
+  int fd =3D -1;
   va_list ap;
   mode_t mode =3D 0;
   fhandler_base *fh =3D NULL;
@@ -1550,9 +1551,12 @@ open (const char *unix_path, int flags, ...)
       /* Reserve an fdtable entry here, before calling open_with_arch() be=
low.
          Otherwise there's a tiny chance of hitting OPEN_MAX further on wh=
ich
          could create a new file without any way for Cygwin to refer to it=
. */
-      cygheap_fdnew fd;
+      cygheap->fdtab.lock();
+      fd =3D cygheap->fdtab.find_unused_handle ();
       if (fd < 0)
-        __leave;		/* errno already set */
+	__leave;		/* errno already set */
+      cygheap->fdtab[fd] =3D fh; /* tentative setting to mark as used */
+      cygheap->fdtab.unlock();
=20
       if (fh->dev () =3D=3D FH_PROCESSFD && fh->pc.follow_fd_symlink ())
 	{
@@ -1580,13 +1584,23 @@ open (const char *unix_path, int flags, ...)
 	try_to_bin (fh->pc, fh->get_handle (), DELETE,
 		    FILE_OPEN_FOR_BACKUP_INTENT);
=20
-      fd =3D fh;
+      cygheap->fdtab.lock ();
+      cygheap->fdtab[fd] =3D fh;
+      fh->inc_refcnt ();
+      cygheap->fdtab.unlock ();
+
       if (fd <=3D 2)
 	set_std_handle (fd);
       res =3D fd;
     }
   __except (EFAULT) {}
   __endtry
+    if (res < 0 && fd >=3D 0)
+      {
+	cygheap->fdtab.lock ();
+	cygheap->fdtab[fd] =3D NULL; /* Mark as unused */
+	cygheap->fdtab.unlock ();
+      }
   if (res < 0 && fh)
     delete fh;
   syscall_printf ("%R =3D open(%s, %y)", res, unix_path, flags);