[newlib-cygwin] Cygwin: pty: Use OpenConsole.exe if available

https://sourceware.org/git/gitweb.cgi?p=3Dnewlib-cygwin.git;h=3D95d0302d827=
f665884d3f0c089485a2ec814cb78

commit 95d0302d827f665884d3f0c089485a2ec814cb78
Author: Takashi Yano <[email protected]>
Date:   Tue Mar 24 12:08:14 2026 +0900

    Cygwin: pty: Use OpenConsole.exe if available
   =20
    This patch replaces legacy conhost.exe with OpenConsole.exe if
    it is available. This enables various new features such as mouse
    support in pseudo console and bug fixes. The legacy conhost has
    problems, e.g. character attributes are mangled or ignored, and
    terminal reports are not passed through. This patch resolve the
    issue by loading /usr/bin/OpenConsole.exe instead of conhost.exe
    if it is available.
   =20
    Signed-off-by: Takashi Yano <[email protected]>
    Suggested-by: Thomas Wolff <towo-JHPtuYzOr/[email protected]>
    Reviewed-by: Johannes Schindelin <[email protected]>

Diff:
---
 winsup/cygwin/environ.cc      |   1 +
 winsup/cygwin/fhandler/pty.cc | 177 ++++++++++++++++++++++++++++++++++++++=
+++-
 winsup/cygwin/globals.cc      |   1 +
 3 files changed, 176 insertions(+), 3 deletions(-)

diff --git a/winsup/cygwin/environ.cc b/winsup/cygwin/environ.cc
index d4cedcbdf..956a04a0c 100644
--- a/winsup/cygwin/environ.cc
+++ b/winsup/cygwin/environ.cc
@@ -114,6 +114,7 @@ static struct parse_thing
   } known[] NO_COPY =3D
 {
   {"disable_pcon", {&disable_pcon}, setbool, NULL, {{false}, {true}}},
+  {"use_legacy_pcon", {&use_legacy_pcon}, setbool, NULL, {{false}, {true}}=
},
   {"error_start", {func: error_start_init}, isfunc, NULL, {{0}, {0}}},
   {"export", {&export_settings}, setbool, NULL, {{false}, {true}}},
   {"glob", {func: glob_init}, isfunc, NULL, {{0}, {s: "normal"}}},
diff --git a/winsup/cygwin/fhandler/pty.cc b/winsup/cygwin/fhandler/pty.cc
index c7ad1d059..d6b783ae0 100644
--- a/winsup/cygwin/fhandler/pty.cc
+++ b/winsup/cygwin/fhandler/pty.cc
@@ -34,6 +34,170 @@ details. */
 #define PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE 0x00020016
 #endif /* PROC_THREAD_ATTRIBUTE_PSEUDOCONSOLE */
=20
+#define OPENCONSOLE_PATH "/usr/bin/OpenConsole.exe"
+
+/* The source code of following two functions, i.e. create_conhost_handle()
+   and CreatePseudoConsole_new(), are borrowed from
+   Microsoft WindowsTerminal project: https://github.com/microsoft/termina=
l/
+   that is licensed under MIT license. */
+
+/* -----------------------------------------------------------------------=
-----
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a co=
py
+of this software and associated documentation files (the "Software"), to d=
eal
+in the Software without restriction, including without limitation the righ=
ts
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in=
 all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FR=
OM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN =
THE
+SOFTWARE.
+--------------------------------------------------------------------------=
-- */
+
+static NTSTATUS
+create_conhost_handle (PHANDLE handle, PCWSTR device_name,
+		       ACCESS_MASK desired_access, HANDLE parent,
+		       BOOLEAN inheritable, ULONG open_options)
+{
+  ULONG flags =3D OBJ_CASE_INSENSITIVE;
+  if (inheritable)
+    flags |=3D OBJ_INHERIT;
+
+  UNICODE_STRING name;
+  RtlInitUnicodeString (&name, device_name);
+
+  OBJECT_ATTRIBUTES object_attributes;
+  InitializeObjectAttributes (&object_attributes, &name, flags, parent, NU=
LL);
+
+  IO_STATUS_BLOCK io;
+  return NtOpenFile (handle, desired_access, &object_attributes, &io,
+		     FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+		     open_options);
+}
+
+static HRESULT
+CreatePseudoConsole_new (COORD size, HANDLE h_input, HANDLE h_output,
+			 DWORD flags, HPCON *hpcon)
+{
+
+  HANDLE h_con_server, h_con_reference;
+  NTSTATUS status;
+  BOOL res;
+  HANDLE h_read_pipe, h_write_pipe;
+  BOOL inherit_cursor;
+  path_conv conhost (OPENCONSOLE_PATH);
+  size_t len;
+  HANDLE inherited_handles[4];
+  STARTUPINFOEXW si =3D {0, };
+  PROCESS_INFORMATION pi;
+  SIZE_T list_size =3D 0;
+  LPPROC_THREAD_ATTRIBUTE_LIST attr_list;
+  HPCON_INTERNAL *hpcon_internal;
+
+  status =3D create_conhost_handle (&h_con_server, L"\\Device\\ConDrv\\Ser=
ver",
+				  GENERIC_ALL, NULL, TRUE, 0);
+  if (!NT_SUCCESS (status))
+    goto cleanup;
+  status =3D create_conhost_handle (&h_con_reference, L"\\Reference",
+				  GENERIC_READ | GENERIC_WRITE | SYNCHRONIZE,
+				  h_con_server, FALSE,
+				  FILE_SYNCHRONOUS_IO_NONALERT);
+  if (!NT_SUCCESS (status))
+    goto cleanup_h_con_server;
+
+  res =3D CreatePipe (&h_read_pipe, &h_write_pipe, &sec_none, 0);
+  if (!res)
+    goto cleanup_h_con_reference;
+  res =3D SetHandleInformation (h_read_pipe,
+			      HANDLE_FLAG_INHERIT, HANDLE_FLAG_INHERIT);
+  if (!res)
+    goto cleanup_pipe;
+
+  inherit_cursor =3D (flags & PSEUDOCONSOLE_INHERIT_CURSOR) ? TRUE : FALSE;
+
+  WCHAR cmd[MAX_PATH];
+  len =3D conhost.get_wide_win32_path_len ();
+  conhost.get_wide_win32_path (cmd);
+  __small_swprintf (cmd + len,
+		    L" --headless %W"
+		    "--width %d --height %d --signal 0x%x --server 0x%x",
+		    inherit_cursor ? L"--inheritcursor " : L"",
+		    size.X, size.Y, h_read_pipe, h_con_server);
+
+  si.StartupInfo.cb =3D sizeof (STARTUPINFOEXW);
+  si.StartupInfo.hStdInput =3D h_input;
+  si.StartupInfo.hStdOutput =3D h_output;
+  si.StartupInfo.hStdError =3D h_output;
+  si.StartupInfo.dwFlags |=3D STARTF_USESTDHANDLES;
+
+  inherited_handles[0] =3D h_con_server;
+  inherited_handles[1] =3D h_input;
+  inherited_handles[2] =3D h_output;
+  inherited_handles[3] =3D h_read_pipe;
+
+  InitializeProcThreadAttributeList (NULL, 1, 0, &list_size);
+  attr_list =3D
+    (LPPROC_THREAD_ATTRIBUTE_LIST) HeapAlloc (GetProcessHeap (), 0, list_s=
ize);
+  if (!attr_list)
+    goto cleanup_pipe;
+
+  si.lpAttributeList =3D attr_list;
+  InitializeProcThreadAttributeList (si.lpAttributeList, 1, 0, &list_size);
+  UpdateProcThreadAttribute (si.lpAttributeList, 0,
+			     PROC_THREAD_ATTRIBUTE_HANDLE_LIST,
+			     inherited_handles, sizeof (inherited_handles),
+			     NULL, NULL);
+
+
+  res =3D CreateProcessW (NULL, cmd, NULL, NULL,
+			TRUE, EXTENDED_STARTUPINFO_PRESENT,
+			NULL, NULL, &si.StartupInfo, &pi);
+  if (!res)
+    goto cleanup_heap;
+
+  hpcon_internal =3D (HPCON_INTERNAL *)
+    HeapAlloc (GetProcessHeap (), 0, sizeof (HPCON_INTERNAL));
+  if (!hpcon_internal)
+    goto cleanup_heap;
+  hpcon_internal->hWritePipe =3D h_write_pipe;
+  hpcon_internal->hConDrvReference =3D h_con_reference;
+  hpcon_internal->hConHostProcess =3D pi.hProcess;
+  *hpcon =3D (HPCON) hpcon_internal;
+
+  DeleteProcThreadAttributeList (attr_list);
+  HeapFree (GetProcessHeap(), 0, attr_list);
+  CloseHandle (h_read_pipe);
+  CloseHandle (h_con_server);
+  CloseHandle (pi.hThread);
+
+  return S_OK;
+
+cleanup_heap:
+  DeleteProcThreadAttributeList (attr_list);
+  HeapFree (GetProcessHeap(), 0, attr_list);
+cleanup_pipe:
+  CloseHandle (h_read_pipe);
+  CloseHandle (h_write_pipe);
+cleanup_h_con_reference:
+  CloseHandle (h_con_reference);
+cleanup_h_con_server:
+  CloseHandle (h_con_server);
+cleanup:
+  return E_FAIL;
+}
+
+
 extern "C" int sscanf (const char *, const char *, ...);
=20
 #define close_maybe(h) \
@@ -3500,9 +3664,16 @@ fhandler_pty_slave::setup_pseudoconsole ()
       const DWORD inherit_cursor =3D 1;
       hpcon =3D NULL;
       SetLastError (ERROR_SUCCESS);
-      HRESULT res =3D CreatePseudoConsole (size, get_handle_nat (),
-					 get_output_handle_nat (),
-					 inherit_cursor, &hpcon);
+      /* Try OpenConsole.exe before conhost.exe */
+      HRESULT res =3D E_FAIL;
+      if (!use_legacy_pcon)
+	res =3D CreatePseudoConsole_new (size, get_handle_nat (),
+				       get_output_handle_nat (),
+				       inherit_cursor, &hpcon);
+      if (res !=3D S_OK) /* Fallback to legacy conhost.exe */
+        res =3D CreatePseudoConsole (size, get_handle_nat (),
+				   get_output_handle_nat (),
+				   inherit_cursor, &hpcon);
       if (res !=3D S_OK || GetLastError () =3D=3D ERROR_PROC_NOT_FOUND)
 	{
 	  if (res !=3D S_OK)
diff --git a/winsup/cygwin/globals.cc b/winsup/cygwin/globals.cc
index f73c35f88..71ffbe793 100644
--- a/winsup/cygwin/globals.cc
+++ b/winsup/cygwin/globals.cc
@@ -73,6 +73,7 @@ bool reset_com;
 bool wincmdln;
 winsym_t allow_winsymlinks =3D WSYM_default;
 bool disable_pcon;
+bool use_legacy_pcon;
 bool winjitdebug =3D false;
=20
 /* Taken from BSD libc: