[PATCH v3] Cygwin: console: Fix undesired mode change at exit of non-cygwin apps

Previously, if two non-cygwin apps are started and one of them
exits first, the other one loosed appropriate console mode, since
the first one restored it to tty::cygwin. This patch counts the
active console process whose pgid is pgid of the tty and if the
result is zero (means the last non-cygwin foreground process),
restore console mode. To avoid race issue between non-cygwin apps
exiting simultaneously, this patch also introduce a named mutex
used only in (setup|cleanup)_for_non_cygwin_app().

Fixes: 48285aa36c2c ("Cygwin: console: Fix handling of Ctrl-S in Win7.")
Signed-off-by: Takashi Yano <[email protected]>
Reviewed-by: Johannes Schindelin <[email protected]>
---
v2: Stop counting up/down the counter by itself.
    Use num_active_non_cygwin_apps() instead.
v3: Guard setup_for_non_cygwin_app() by cons_mode_mutex as well.

 winsup/cygwin/fhandler/console.cc | 52 +++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/winsup/cygwin/fhandler/console.cc b/winsup/cygwin/fhandler/console.cc
index d4c87f29f..62650726d 100644
--- a/winsup/cygwin/fhandler/console.cc
+++ b/winsup/cygwin/fhandler/console.cc
@@ -977,15 +977,65 @@ fhandler_console::setup_for_non_cygwin_app ()
      console mode. */
   if (get_ttyp ()->getpgid () == myself->pgid)
     {
+      char buf[MAX_PATH];
+      shared_name (buf, "cygcons.cons_mode.mutex", unit);
+      HANDLE cons_mode_mutex = CreateMutex (&sec_none, FALSE, buf);
+      WaitForSingleObject (cons_mode_mutex, INFINITE);
       set_disable_master_thread (true, this);
       set_input_mode (tty::native, &tc ()->ti, get_handle_set ());
       set_output_mode (tty::native, &tc ()->ti, get_handle_set ());
+      ReleaseMutex (cons_mode_mutex);
+      CloseHandle (cons_mode_mutex);
     }
 }
 
+static int
+num_active_non_cygwin_apps (pid_t pgid)
+{
+  tmp_pathbuf tp;
+  DWORD *list = (DWORD *) tp.c_get ();
+  const DWORD buf_size = NT_MAX_PATH / sizeof (DWORD);
+
+  DWORD buf_size1 = 1;
+  DWORD num;
+  /* The buffer of too large size does not seem to be expected by new condrv.
+     https://github.com/microsoft/terminal/issues/18264#issuecomment-2515448548
+     Use the minimum buffer size in the loop. */
+  while ((num = GetConsoleProcessList (list, buf_size1)) > buf_size1)
+    {
+      if (num > buf_size)
+	return 0;
+      buf_size1 = num;
+    }
+  if (num == 0)
+    return 0;
+
+  int cnt = 0;
+  for (DWORD i = 0; i < num; i++)
+    {
+      pinfo p (cygwin_pid (list[i]));
+      if (!!p && p->pgid == pgid && ISSTATE (p, PID_NOTCYGWIN))
+	cnt++;
+    }
+  return cnt;
+}
+
 void
 fhandler_console::cleanup_for_non_cygwin_app (handle_set_t *p)
 {
+  if (cygheap->ctty->tc()->pgid != myself->pgid)
+    return;
+  char buf[MAX_PATH];
+  shared_name (buf, "cygcons.cons_mode.mutex", p->unit);
+  HANDLE cons_mode_mutex = CreateMutex (&sec_none, FALSE, buf);
+  WaitForSingleObject (cons_mode_mutex, INFINITE);
+  if (num_active_non_cygwin_apps (cygheap->ctty->tc()->pgid))
+    {
+      ReleaseMutex (cons_mode_mutex);
+      CloseHandle (cons_mode_mutex);
+      return;
+    }
+
   const _minor_t unit = p->unit;
   termios dummy = {0, };
   termios *ti = shared_console_info[unit] ?
@@ -999,6 +1049,8 @@ fhandler_console::cleanup_for_non_cygwin_app (handle_set_t *p)
     set_output_mode (conmode, ti, p);
   if (con.curr_input_mode != conmode)
     set_input_mode (conmode, ti, p);
+  ReleaseMutex (cons_mode_mutex);
+  CloseHandle (cons_mode_mutex);
 }
 
 /* Return the tty structure associated with a given tty number.  If the
-- 
2.51.0