[glibc] libio: Ignore doallocate for open_memstream and open_wmemstream [BZ #34019]

https://sourceware.org/git/gitweb.cgi?p=glibc.git;h=97bd3b3d56ed867e7a545ce3501991a92df0d1bb

commit 97bd3b3d56ed867e7a545ce3501991a92df0d1bb
Author: Xiang Gao <[email protected]>
Date:   Fri May 8 14:04:10 2026 +0800

    libio: Ignore doallocate for open_memstream and open_wmemstream [BZ #34019]
    
    setvbuf (stream, NULL, _IOFBF, 0) takes a special path in
    _IO_setvbuf: if the byte-oriented buffer base is NULL, it calls
    _IO_DOALLOCATE and returns without invoking the stream setbuf hook.
    
    For open_wmemstream, the byte-oriented buffer base is NULL although
    the wide result buffer has already been initialized in _wide_data.
    As a result, this path calls _IO_wdefault_doallocate, which may
    replace the wide buffer managed by open_wmemstream.
    
    Install an open_wmemstream-specific doallocate hook that leaves
    the growable result buffer unchanged. Add a regression test for this
    path.
    
    Install a narrow memstream doallocate hook as well. It keeps both
    memstream vtables consistent (generic stdio allocation must not
    replace the growable result buffer).
    
    Signed-off-by: Xiang Gao <[email protected]>
    Reviewed-by: Adhemerval Zanella  <[email protected]>

Diff:
---
 libio/Makefile                            |  2 +
 libio/libioP.h                            |  2 +
 libio/memstream.c                         |  9 +++++
 libio/tst-memstream-setvbuf-doallocate.c  | 61 +++++++++++++++++++++++++++++++
 libio/tst-wmemstream-setvbuf-doallocate.c | 20 ++++++++++
 libio/vtables.c                           |  6 ++-
 libio/wmemstream.c                        |  9 +++++
 7 files changed, 107 insertions(+), 2 deletions(-)

diff --git a/libio/Makefile b/libio/Makefile
index 916542f23b..616107ee10 100644
--- a/libio/Makefile
+++ b/libio/Makefile
@@ -120,6 +120,7 @@ tests = \
   tst-fwrite-error \
   tst-getdelim \
   tst-memstream-setbuf \
+  tst-memstream-setvbuf-doallocate \
   tst-memstream1 \
   tst-memstream2 \
   tst-memstream3 \
@@ -144,6 +145,7 @@ tests = \
   tst-wfiledoallocate-static \
   tst-widetext \
   tst-wmemstream-setbuf \
+  tst-wmemstream-setvbuf-doallocate \
   tst-wmemstream1 \
   tst-wmemstream2 \
   tst-wmemstream3 \
diff --git a/libio/libioP.h b/libio/libioP.h
index fd42307116..78e8ee6835 100644
--- a/libio/libioP.h
+++ b/libio/libioP.h
@@ -743,10 +743,12 @@ extern int _IO_mem_sync (FILE* fp) __THROW attribute_hidden;
 extern void _IO_mem_finish (FILE* fp, int) __THROW attribute_hidden;
 extern FILE *_IO_mem_setbuf (FILE *fp, char *buf, ssize_t size)
   __THROW attribute_hidden;
+extern int _IO_mem_doallocate (FILE *fp) __THROW attribute_hidden;
 extern int _IO_wmem_sync (FILE* fp) __THROW attribute_hidden;
 extern void _IO_wmem_finish (FILE* fp, int) __THROW attribute_hidden;
 extern FILE *_IO_wmem_setbuf (FILE *fp, char *buf, ssize_t size)
   __THROW attribute_hidden;
+extern int _IO_wmem_doallocate (FILE *fp) __THROW attribute_hidden;
 
 /* Other strfile functions */
 struct _IO_strfile_;
diff --git a/libio/memstream.c b/libio/memstream.c
index 69b400d928..1af7224ab5 100644
--- a/libio/memstream.c
+++ b/libio/memstream.c
@@ -120,3 +120,12 @@ _IO_mem_setbuf (FILE *fp, char *p, ssize_t len)
   /* Memstream manage a growable buffer internally.  */
   return fp;
 }
+
+
+int
+_IO_mem_doallocate (FILE *fp)
+{
+  /* Memstream manage a growable buffer internally.  The doallocate
+     hook must not replace it with a generic stdio buffer.  */
+  return 1;
+}
diff --git a/libio/tst-memstream-setvbuf-doallocate.c b/libio/tst-memstream-setvbuf-doallocate.c
new file mode 100644
index 0000000000..aa6f86f02d
--- /dev/null
+++ b/libio/tst-memstream-setvbuf-doallocate.c
@@ -0,0 +1,61 @@
+/* Test setvbuf on open_memstream, BZ #34019.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <libio/tst-memstream.h>
+
+static int
+do_test (void)
+{
+  /* Regression test for setvbuf doallocate on open_memstream and
+     open_wmemstream.  This test covers the _IO_setvbuf path for:
+
+     setvbuf (stream, NULL, _IOFBF, 0)
+
+     This path may call _IO_DOALLOCATE and return without invoking
+     the stream setbuf hook.  The generic doallocate hook must not
+     replace the growable result buffer.  */
+
+  CHAR_T *buf = NULL;
+  size_t len = 0;
+  FILE *fp = OPEN_MEMSTREAM (&buf, &len);
+
+  TEST_VERIFY_EXIT (fp != NULL);
+
+  TEST_COMPARE (setvbuf (fp, NULL, _IOFBF, 0), 0);
+
+  TEST_COMPARE (FPUTC (W('A'), fp), W('A'));
+  TEST_COMPARE (fflush (fp), 0);
+  TEST_COMPARE (len, 1);
+  TEST_VERIFY_EXIT (buf != NULL);
+  TEST_VERIFY (buf[0] == W('A'));
+  TEST_VERIFY (buf[1] == W('\0'));
+
+  TEST_COMPARE (FPUTC (W('B'), fp), W('B'));
+  TEST_COMPARE (fclose (fp), 0);
+  TEST_COMPARE (len, 2);
+  TEST_VERIFY_EXIT (buf != NULL);
+  TEST_VERIFY (buf[0] == W('A'));
+  TEST_VERIFY (buf[1] == W('B'));
+  TEST_VERIFY (buf[2] == W('\0'));
+
+  free (buf);
+
+  return 0;
+}
+
+#include <support/test-driver.c>
diff --git a/libio/tst-wmemstream-setvbuf-doallocate.c b/libio/tst-wmemstream-setvbuf-doallocate.c
new file mode 100644
index 0000000000..857239aa85
--- /dev/null
+++ b/libio/tst-wmemstream-setvbuf-doallocate.c
@@ -0,0 +1,20 @@
+/* Test setvbuf on open_wmemstream, BZ #34019.
+   Copyright (C) 2026 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define TEST_WCHAR
+#include <libio/tst-memstream-setvbuf-doallocate.c>
diff --git a/libio/vtables.c b/libio/vtables.c
index ca9f1b2dc4..ba3f9566f8 100644
--- a/libio/vtables.c
+++ b/libio/vtables.c
@@ -79,10 +79,12 @@
 # pragma weak _IO_mem_finish
 # pragma weak _IO_mem_setbuf
 # pragma weak _IO_mem_sync
+# pragma weak _IO_mem_doallocate
 
 # pragma weak _IO_wmem_finish
 # pragma weak _IO_wmem_setbuf
 # pragma weak _IO_wmem_sync
+# pragma weak _IO_wmem_doallocate
 
 # pragma weak __printf_buffer_as_file_overflow
 # pragma weak __printf_buffer_as_file_xsputn
@@ -338,7 +340,7 @@ const struct _IO_jump_t __io_vtables[] attribute_relro =
     JUMP_INIT (seekpos, _IO_default_seekpos),
     JUMP_INIT (setbuf, _IO_mem_setbuf),
     JUMP_INIT (sync, _IO_mem_sync),
-    JUMP_INIT (doallocate, _IO_default_doallocate),
+    JUMP_INIT (doallocate, _IO_mem_doallocate),
     JUMP_INIT (read, _IO_default_read),
     JUMP_INIT (write, _IO_default_write),
     JUMP_INIT (seek, _IO_default_seek),
@@ -361,7 +363,7 @@ const struct _IO_jump_t __io_vtables[] attribute_relro =
     JUMP_INIT (seekpos, _IO_default_seekpos),
     JUMP_INIT (setbuf, _IO_wmem_setbuf),
     JUMP_INIT (sync, _IO_wmem_sync),
-    JUMP_INIT (doallocate, _IO_wdefault_doallocate),
+    JUMP_INIT (doallocate, _IO_wmem_doallocate),
     JUMP_INIT (read, _IO_default_read),
     JUMP_INIT (write, _IO_default_write),
     JUMP_INIT (seek, _IO_default_seek),
diff --git a/libio/wmemstream.c b/libio/wmemstream.c
index cdee2a4151..dffbb4a808 100644
--- a/libio/wmemstream.c
+++ b/libio/wmemstream.c
@@ -125,3 +125,12 @@ _IO_wmem_setbuf (FILE *fp, char *p, ssize_t len)
   /* Wmemstreams manage a growable buffer internally.  */
   return fp;
 }
+
+
+int
+_IO_wmem_doallocate (FILE *fp)
+{
+  /* Wmemstreams manage a growable buffer internally.  The doallocate
+     hook must not replace it with a generic stdio buffer.  */
+  return 1;
+}