[PATCH v2] posix: Avoid allocation while holding atfork_lock [BZ 34321]
Adhemerval Zanella <[email protected]>
| Newsgroups | gmane.comp.lib.glibc.alpha |
|---|---|
| Message-ID | <[email protected]> |
__register_atfork inserted the new handler into a dynamic array while
holding atfork_lock, and growing that array calls the public/iterposable
malloc. An allocator replacement whose own lock is also taken from a
registered pthread_atfork prepare handler can therefore deadlock:
Thread A: fork Thread B: __register_atfork
------------------------------------- -----------------------------------
:139 unlock(atfork_lock)
:141 prepare() -> lock(L) [holds L]
:39 lock(atfork_lock) [holds atfork]
:44 emplace -> malloc -> lock(L)
BLOCKS on L (held by A)
:144 lock(atfork_lock)
BLOCKS on atfork (held by B)
=> DEADLOCK (A: holds L, waits atfork; B: holds atfork, waits L)
Replace the dynamic array with a doubly linked list (include/list.h).
A new node is allocated outside atfork_lock and only linked in while the
lock is held. __unregister_atfork moves nodes to a free pool instead of
freeing them, and __register_atfork reuses a pooled node when one is
available, so neither path allocates or frees under atfork_lock in the
steady state; malloc runs only to grow the pool, always outside the lock.
This preserves the current allocation guarantee of the dynarray.
Checked on x86_64-linux-gnu and aarch64-linux-gnu.
---
include/register-atfork.h | 5 +-
posix/register-atfork.c | 278 ++++++++++++++++++-------------
sysdeps/pthread/Makefile | 7 +
sysdeps/pthread/tst-atfork5.c | 115 +++++++++++++
sysdeps/pthread/tst-atfork5mod.c | 175 +++++++++++++++++++
5 files changed, 464 insertions(+), 116 deletions(-)
create mode 100644 sysdeps/pthread/tst-atfork5.c
create mode 100644 sysdeps/pthread/tst-atfork5mod.c
diff --git a/include/register-atfork.h b/include/register-atfork.h
index 25286a25aad..2fc0064d9d8 100644
--- a/include/register-atfork.h
+++ b/include/register-atfork.h
@@ -19,7 +19,9 @@
#ifndef _REGISTER_ATFORK_H
#define _REGISTER_ATFORK_H
-/* Elements of the fork handler lists. */
+#include <list_t.h>
+
+/* Elements of the fork handler list. */
struct fork_handler
{
void (*prepare_handler) (void);
@@ -27,6 +29,7 @@ struct fork_handler
void (*child_handler) (void);
void *dso_handle;
uint64_t id;
+ list_t list;
};
/* Function to call to unregister fork handlers. */
diff --git a/posix/register-atfork.c b/posix/register-atfork.c
index 580c6b77caf..ef20f5cbf7e 100644
--- a/posix/register-atfork.c
+++ b/posix/register-atfork.c
@@ -17,94 +17,121 @@
#include <libc-lock.h>
#include <stdbool.h>
+#include <stdlib.h>
+#include <errno.h>
#include <register-atfork.h>
#include <intprops.h>
+#include <list.h>
#include <stdio.h>
-#define DYNARRAY_ELEMENT struct fork_handler
-#define DYNARRAY_STRUCT fork_handler_list
-#define DYNARRAY_PREFIX fork_handler_list_
-#define DYNARRAY_INITIAL_SIZE 48
-#include <malloc/dynarray-skeleton.c>
+/* New handlers are added at the front, so the list runs from the newest
+ (highest ID) at the front to the oldest (lowest ID) at the back. A new
+ node is allocated with malloc only when no reusable node is available, and
+ always *outside* of ATFORK_LOCK: holding ATFORK_LOCK across an allocation
+ would deadlock with interposable allocators whose own locks are also taken
+ from an atfork handler. */
+static LIST_HEAD (fork_handlers);
+
+/* Unregistered nodes are moved here (under ATFORK_LOCK) instead of being
+ freed, and a subsequent registration reuses one. This keeps both
+ registration and unregistration allocation-free in the steady state (in
+ particular avoids any free under ATFORK_LOCK). The pool is released only
+ by __libc_atfork_freemem. */
+static LIST_HEAD (fork_handlers_free);
-static struct fork_handler_list fork_handlers;
static uint64_t fork_handler_counter;
+/* Bumped on every list mutation (registration or unregistration). The fork
+ handler runners drop ATFORK_LOCK while a handler executes; comparing this
+ counter across that window tells them whether the list changed and a saved
+ list position is therefore still valid. */
+static uint64_t fork_handler_modcount;
+
static int atfork_lock = LLL_LOCK_INITIALIZER;
+#define fork_handler_entry(pos) list_entry (pos, struct fork_handler, list)
+
+/* True if the list LP has no elements. */
+static inline bool
+list_is_empty (list_t *lp)
+{
+ return lp->next == lp;
+}
+
+/* Initialize NEWP from the supplied handlers and link it at the front of the
+ active list. ATFORK_LOCK must be held. */
+static void
+fork_handler_init (struct fork_handler *newp, void (*prepare) (void),
+ void (*parent) (void), void (*child) (void),
+ void *dso_handle)
+{
+ newp->prepare_handler = prepare;
+ newp->parent_handler = parent;
+ newp->child_handler = child;
+ newp->dso_handle = dso_handle;
+
+ /* IDs assigned to handlers start at 1 and increment with handler
+ registration. Un-registering a handler discards the corresponding ID.
+ It is not reused in future registrations. */
+ if (INT_ADD_OVERFLOW (fork_handler_counter, 1))
+ __libc_fatal ("fork handler counter overflow");
+ newp->id = ++fork_handler_counter;
+
+ /* Add at the front: the highest ID ends up first. */
+ list_add (&newp->list, &fork_handlers);
+
+ ++fork_handler_modcount;
+}
+
int
__register_atfork (void (*prepare) (void), void (*parent) (void),
void (*child) (void), void *dso_handle)
{
lll_lock (atfork_lock, LLL_PRIVATE);
- if (fork_handler_counter == 0)
- fork_handler_list_init (&fork_handlers);
-
- struct fork_handler *newp = fork_handler_list_emplace (&fork_handlers);
- if (newp != NULL)
+ bool free_empty = list_is_empty (&fork_handlers_free);
+ if (!free_empty)
{
- newp->prepare_handler = prepare;
- newp->parent_handler = parent;
- newp->child_handler = child;
- newp->dso_handle = dso_handle;
-
- /* IDs assigned to handlers start at 1 and increment with handler
- registration. Un-registering a handlers discards the corresponding
- ID. It is not reused in future registrations. */
- if (INT_ADD_OVERFLOW (fork_handler_counter, 1))
- __libc_fatal ("fork handler counter overflow");
- newp->id = ++fork_handler_counter;
+ list_t *reuse = fork_handlers_free.next;
+ list_del (reuse);
+ fork_handler_init (fork_handler_entry (reuse), prepare, parent, child,
+ dso_handle);
}
- /* Release the lock. */
lll_unlock (atfork_lock, LLL_PRIVATE);
- return newp == NULL ? ENOMEM : 0;
+ if (!free_empty)
+ return 0;
+
+ struct fork_handler *newp = malloc (sizeof (*newp));
+ if (newp == NULL)
+ return ENOMEM;
+
+ lll_lock (atfork_lock, LLL_PRIVATE);
+ fork_handler_init (newp, prepare, parent, child, dso_handle);
+ lll_unlock (atfork_lock, LLL_PRIVATE);
+
+ return 0;
}
libc_hidden_def (__register_atfork)
-static struct fork_handler *
-fork_handler_list_find (struct fork_handler_list *fork_handlers,
- void *dso_handle)
-{
- for (size_t i = 0; i < fork_handler_list_size (fork_handlers); i++)
- {
- struct fork_handler *elem = fork_handler_list_at (fork_handlers, i);
- if (elem->dso_handle == dso_handle)
- return elem;
- }
- return NULL;
-}
-
void
__unregister_atfork (void *dso_handle)
{
+ list_t *runp, *prevp;
+
lll_lock (atfork_lock, LLL_PRIVATE);
- struct fork_handler *first = fork_handler_list_find (&fork_handlers,
- dso_handle);
- /* Removing is done by shifting the elements in the way the elements
- that are not to be removed appear in the beginning in dynarray.
- This avoid the quadradic run-time if a naive strategy to remove and
- shift one element at time. */
- if (first != NULL)
- {
- struct fork_handler *new_end = first;
- first++;
- for (; first != fork_handler_list_end (&fork_handlers); ++first)
- {
- if (first->dso_handle != dso_handle)
- {
- *new_end = *first;
- ++new_end;
- }
- }
-
- ptrdiff_t removed = first - new_end;
- for (size_t i = 0; i < removed; i++)
- fork_handler_list_remove_last (&fork_handlers);
- }
+ /* Move the matching nodes to the free pool rather than freeing them: this
+ avoids a free under ATFORK_LOCK and lets a later registration reuse
+ them. */
+ list_for_each_prev_safe (runp, prevp, &fork_handlers)
+ if (fork_handler_entry (runp)->dso_handle == dso_handle)
+ {
+ list_del (runp);
+ list_add (runp, &fork_handlers_free);
+ ++fork_handler_modcount;
+ }
lll_unlock (atfork_lock, LLL_PRIVATE);
}
@@ -117,47 +144,55 @@ __run_prefork_handlers (_Bool do_locking)
if (do_locking)
lll_lock (atfork_lock, LLL_PRIVATE);
- /* We run prepare handlers from last to first. After fork, only
- handlers up to the last handler found here (pre-fork) will be run.
- Handlers registered during __run_prefork_handlers or
- __run_postfork_handlers will be positioned after this last handler, and
- since their prepare handlers won't be run now, their parent/child
- handlers should also be ignored. */
+ /* We run prepare handlers from newest to oldest (highest ID first). After
+ fork, only handlers up to the last handler found here (pre-fork) will be
+ run. Handlers registered during __run_prefork_handlers or
+ __run_postfork_handlers will have a higher ID, and since their prepare
+ handlers will not be run now, their parent/child handlers should also be
+ ignored. */
lastrun = fork_handler_counter;
- size_t sl = fork_handler_list_size (&fork_handlers);
- for (size_t i = sl; i > 0;)
+ /* The newest handlers are at the front; skip any with ID > LASTRUN (those
+ were registered after this function was entered). */
+ list_t *pos = fork_handlers.next;
+ while (pos != &fork_handlers && fork_handler_entry (pos)->id > lastrun)
+ pos = pos->next;
+
+ while (pos != &fork_handlers)
{
- struct fork_handler *runp
- = fork_handler_list_at (&fork_handlers, i - 1);
-
+ struct fork_handler *runp = fork_handler_entry (pos);
uint64_t id = runp->id;
+ void (*prepare_handler) (void) = runp->prepare_handler;
- if (runp->prepare_handler != NULL)
+ /* Remember where to continue and whether the list changed across the
+ handler call. Both are read while the lock is held. */
+ uint64_t saved_modcount = fork_handler_modcount;
+ list_t *next = pos->next;
+
+ if (prepare_handler != NULL)
{
if (do_locking)
lll_unlock (atfork_lock, LLL_PRIVATE);
- runp->prepare_handler ();
+ prepare_handler ();
if (do_locking)
lll_lock (atfork_lock, LLL_PRIVATE);
}
- /* We unlocked, ran the handler, and locked again. In the
- meanwhile, one or more deregistrations could have occurred leading
- to the current (just run) handler being moved up the list or even
- removed from the list itself. Since handler IDs are guaranteed to
- to be in increasing order, the next handler has to have: */
-
- /* A. An earlier position than the current one has. */
- i--;
-
- /* B. A lower ID than the current one does. The code below skips
- any newly added handlers with higher IDs. */
- while (i > 0
- && fork_handler_list_at (&fork_handlers, i - 1)->id >= id)
- i--;
+ /* Advance to the next (lower ID) handler. If nothing was registered or
+ unregistered while the lock was dropped, the saved position is still
+ valid. Otherwise it may be stale (a node could even have been
+ freed), so re-find the next handler by ID: the one with the greatest
+ ID strictly below the just-run handler. */
+ if (fork_handler_modcount == saved_modcount)
+ pos = next;
+ else
+ {
+ pos = fork_handlers.next;
+ while (pos != &fork_handlers && fork_handler_entry (pos)->id >= id)
+ pos = pos->next;
+ }
}
return lastrun;
@@ -167,10 +202,12 @@ void
__run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
uint64_t lastrun)
{
- size_t sl = fork_handler_list_size (&fork_handlers);
- for (size_t i = 0; i < sl;)
+ /* Run parent/child handlers from oldest to newest (lowest ID first). The
+ oldest handlers are at the back of the list. */
+ list_t *pos = fork_handlers.prev;
+ while (pos != &fork_handlers)
{
- struct fork_handler *runp = fork_handler_list_at (&fork_handlers, i);
+ struct fork_handler *runp = fork_handler_entry (pos);
uint64_t id = runp->id;
/* prepare handlers were not run for handlers with ID > LASTRUN.
@@ -178,37 +215,36 @@ __run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
if (id > lastrun)
break;
- if (do_locking)
- lll_unlock (atfork_lock, LLL_PRIVATE);
+ void (*handler) (void) = NULL;
+ if (who == atfork_run_child)
+ handler = runp->child_handler;
+ else if (who == atfork_run_parent)
+ handler = runp->parent_handler;
- if (who == atfork_run_child && runp->child_handler)
- runp->child_handler ();
- else if (who == atfork_run_parent && runp->parent_handler)
- runp->parent_handler ();
+ uint64_t saved_modcount = fork_handler_modcount;
+ list_t *prev = pos->prev;
- if (do_locking)
- lll_lock (atfork_lock, LLL_PRIVATE);
+ if (handler != NULL)
+ {
+ if (do_locking)
+ lll_unlock (atfork_lock, LLL_PRIVATE);
- /* We unlocked, ran the handler, and locked again. In the meanwhile,
- one or more [de]registrations could have occurred. Due to this,
- the list size must be updated. */
- sl = fork_handler_list_size (&fork_handlers);
+ handler ();
- /* The just-run handler could also have moved up the list. */
+ if (do_locking)
+ lll_lock (atfork_lock, LLL_PRIVATE);
+ }
- if (sl > i && fork_handler_list_at (&fork_handlers, i)->id == id)
- /* The position of the recently run handler hasn't changed. The
- next handler to be run is an easy increment away. */
- i++;
+ /* Advance to the next (higher ID) handler. Fast path when the list did
+ not change; otherwise re-find by ID the handler with the smallest ID
+ strictly above the just-run one. */
+ if (fork_handler_modcount == saved_modcount)
+ pos = prev;
else
{
- /* The next handler to be run is the first handler in the list
- to have an ID higher than the current one. */
- for (i = 0; i < sl; i++)
- {
- if (fork_handler_list_at (&fork_handlers, i)->id > id)
- break;
- }
+ pos = fork_handlers.prev;
+ while (pos != &fork_handlers && fork_handler_entry (pos)->id <= id)
+ pos = pos->prev;
}
}
@@ -220,9 +256,21 @@ __run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
void
__libc_atfork_freemem (void)
{
+ /* Detach both the active list and the free pool under the lock and free
+ them afterwards. */
+ LIST_HEAD (removed);
+ list_t *runp, *prevp;
+
lll_lock (atfork_lock, LLL_PRIVATE);
- fork_handler_list_free (&fork_handlers);
+ list_splice (&fork_handlers, &removed);
+ INIT_LIST_HEAD (&fork_handlers);
+
+ list_splice (&fork_handlers_free, &removed);
+ INIT_LIST_HEAD (&fork_handlers_free);
lll_unlock (atfork_lock, LLL_PRIVATE);
+
+ list_for_each_prev_safe (runp, prevp, &removed)
+ free (fork_handler_entry (runp));
}
diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile
index d0f3cd59ac6..d5d1df2d56a 100644
--- a/sysdeps/pthread/Makefile
+++ b/sysdeps/pthread/Makefile
@@ -348,6 +348,7 @@ tests += \
tst-atfork2 \
tst-atfork3 \
tst-atfork4 \
+ tst-atfork5 \
tst-create1 \
tst-fini1 \
tst-pt-tls4 \
@@ -364,6 +365,7 @@ modules-names += \
tst-atfork2mod \
tst-atfork3mod \
tst-atfork4mod \
+ tst-atfork5mod \
tst-create1mod \
tst-fini1mod \
tst-stack2-mod \
@@ -459,10 +461,15 @@ $(objpfx)tst-atfork4: $(shared-thread-library)
LDFLAGS-tst-atfork4 = -rdynamic
$(objpfx)tst-atfork4mod.so: $(shared-thread-library)
+$(objpfx)tst-atfork5: $(shared-thread-library)
+tst-atfork5-ENV = LD_PRELOAD=$(objpfx)tst-atfork5mod.so
+$(objpfx)tst-atfork5mod.so: $(shared-thread-library)
+
ifeq ($(build-shared),yes)
$(objpfx)tst-atfork2.out: $(objpfx)tst-atfork2mod.so
$(objpfx)tst-atfork3.out: $(objpfx)tst-atfork3mod.so
$(objpfx)tst-atfork4.out: $(objpfx)tst-atfork4mod.so
+$(objpfx)tst-atfork5.out: $(objpfx)tst-atfork5mod.so
endif
ifeq ($(build-shared),yes)
diff --git a/sysdeps/pthread/tst-atfork5.c b/sysdeps/pthread/tst-atfork5.c
new file mode 100644
index 00000000000..e3adfc9292d
--- /dev/null
+++ b/sysdeps/pthread/tst-atfork5.c
@@ -0,0 +1,115 @@
+/* Test that atfork registration does not deadlock against a malloc
+ replacement (BZ 34321).
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Check if a malloc replacement that takes its own lock both in malloc and
+ in a registered fork prepare handler does not deadlock. One thread holds
+ the atfork lock and waits for the allocator lock inside __register_atfork,
+ while the forking thread holds the allocator lock in the prepare handler
+ and waits for the atfork lock.
+
+ The malloc replacement is provided by tst-atfork5mod.so through
+ LD_PRELOAD. */
+
+#include <dlfcn.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/wait.h>
+
+#include <support/check.h>
+#include <support/xthread.h>
+#include <support/xunistd.h>
+#include <support/xdlfcn.h>
+
+#define NREG 3 /* Number of 'tf' threads. */
+#define FORK_TARGET 20 /* Forks to complete when there is no deadlock. */
+#define REG_CAP 40000U /* Upper bound on total handler registrations. */
+
+/* Synchronizes the NREG tf threads with the first prepare handler run, so
+ that they only start registering once the allocator lock is held by the
+ preloaded module's prepare handler. */
+static pthread_barrier_t *window_barrier;
+
+static atomic_int running = 1;
+static atomic_int forks_done;
+static atomic_uint reg_count;
+
+static void *
+tf (void *closure)
+{
+ /* Start registering only once a prepare window is open (the module's
+ allocator lock held), so the first handler-list growth happens under
+ that lock. The total number of registrations is bounded so that the
+ handler list stays small and, on a fixed library, the fork handlers
+ run quickly. */
+ xpthread_barrier_wait (window_barrier);
+ while (atomic_load (&running)
+ && atomic_fetch_add (®_count, 1) < REG_CAP)
+ pthread_atfork (NULL, NULL, NULL);
+ return NULL;
+}
+
+static void *
+forker (void *closure)
+{
+ while (atomic_load (&running))
+ {
+ pid_t pid = xfork ();
+ if (pid == 0)
+ _exit (0);
+ xwaitpid (pid, NULL, 0);
+ if (atomic_fetch_add (&forks_done, 1) + 1 >= FORK_TARGET)
+ break;
+ }
+ return NULL;
+}
+
+static int
+do_test (void)
+{
+ /* Check if tst-atfork5mod.so was preloaded. */
+ void (*mod_prepare) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_prepare");
+ void (*mod_parent) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_parent");
+ void (*mod_child) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_child");
+ atomic_uint *malloc_count = xdlsym (RTLD_DEFAULT,
+ "atfork5mod_malloc_count");
+ window_barrier = xdlsym (RTLD_DEFAULT, "atfork5mod_window_barrier");
+
+ xpthread_barrier_init (window_barrier, NULL, NREG + 1);
+ TEST_COMPARE (pthread_atfork (mod_prepare, mod_parent, mod_child), 0);
+
+ pthread_t reg[NREG];
+ for (int i = 0; i < NREG; i++)
+ reg[i] = xpthread_create (NULL, tf, NULL);
+ pthread_t fork_tid = xpthread_create (NULL, forker, NULL);
+
+ xpthread_join (fork_tid);
+ atomic_store (&running, 0);
+ for (int i = 0; i < NREG; i++)
+ xpthread_join (reg[i]);
+
+ TEST_VERIFY (atomic_load (malloc_count) > 0);
+
+ xpthread_barrier_destroy (window_barrier);
+ return 0;
+}
+
+#define TIMEOUT 8
+#include <support/test-driver.c>
diff --git a/sysdeps/pthread/tst-atfork5mod.c b/sysdeps/pthread/tst-atfork5mod.c
new file mode 100644
index 00000000000..bb59560db8f
--- /dev/null
+++ b/sysdeps/pthread/tst-atfork5mod.c
@@ -0,0 +1,175 @@
+/* Malloc replacement module for tst-atfork5 (BZ 34321).
+ Copyright (C) 2026 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <https://www.gnu.org/licenses/>. */
+
+/* Minimal malloc interposer, applied to tst-atfork5 through LD_PRELOAD:
+ every allocation takes ARENA_LOCK, and so does the fork prepare handler
+ below (registered by the test program via pthread_atfork). */
+
+#include <dlfcn.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+/* Microseconds the prepare handler holds ARENA_LOCK. */
+#define WINDOW_US 4000
+
+static pthread_mutex_t arena_lock = PTHREAD_MUTEX_INITIALIZER;
+static void *(*real_malloc) (size_t);
+static void *(*real_calloc) (size_t, size_t);
+static void *(*real_realloc) (void *, size_t);
+static void (*real_free) (void *);
+
+/* Number of allocations that went through the interposer; the test program
+ uses it to check that this module was actually preloaded. */
+atomic_uint atfork5mod_malloc_count;
+
+/* Small bump buffer to provide the few allocations that happen while dlsym
+ itself runs. */
+static __thread bool in_dlsym;
+static char bootbuf[1 << 20];
+static size_t bootoff;
+
+static int
+is_boot (void *p)
+{
+ return (char *) p >= bootbuf && (char *) p < bootbuf + sizeof bootbuf;
+}
+
+static void *
+boot_alloc (size_t n)
+{
+ size_t o = (bootoff + 15) & ~(size_t) 15;
+ bootoff = o + n;
+ return bootbuf + o;
+}
+
+static void *
+next_sym (const char *name)
+{
+ void *sym = dlsym (RTLD_NEXT, name);
+ if (sym == NULL)
+ abort ();
+ return sym;
+}
+
+static void
+init_real (void)
+{
+ if (real_malloc != NULL)
+ return;
+ in_dlsym = true;
+ real_malloc = next_sym ("malloc");
+ real_calloc = next_sym ("calloc");
+ real_realloc = next_sym ("realloc");
+ real_free = next_sym ("free");
+ in_dlsym = false;
+}
+
+void *
+malloc (size_t n)
+{
+ if (real_malloc == NULL)
+ {
+ if (in_dlsym)
+ return boot_alloc (n);
+ init_real ();
+ }
+ atomic_fetch_add (&atfork5mod_malloc_count, 1);
+ pthread_mutex_lock (&arena_lock);
+ void *p = real_malloc (n);
+ pthread_mutex_unlock (&arena_lock);
+ return p;
+}
+
+void *
+calloc (size_t a, size_t b)
+{
+ if (real_malloc == NULL)
+ {
+ if (in_dlsym)
+ return boot_alloc (a * b);
+ init_real ();
+ }
+ atomic_fetch_add (&atfork5mod_malloc_count, 1);
+ pthread_mutex_lock (&arena_lock);
+ void *p = real_calloc (a, b);
+ pthread_mutex_unlock (&arena_lock);
+ return p;
+}
+
+void *
+realloc (void *old, size_t n)
+{
+ if (real_malloc == NULL)
+ init_real ();
+ atomic_fetch_add (&atfork5mod_malloc_count, 1);
+ pthread_mutex_lock (&arena_lock);
+ void *p = real_realloc (old, n);
+ pthread_mutex_unlock (&arena_lock);
+ return p;
+}
+
+void
+free (void *p)
+{
+ if (is_boot (p))
+ return;
+ if (real_free == NULL)
+ init_real ();
+ pthread_mutex_lock (&arena_lock);
+ real_free (p);
+ pthread_mutex_unlock (&arena_lock);
+}
+
+/* Synchronizes the registering threads of the test program with the first
+ prepare handler run, so that they only start registering once ARENA_LOCK
+ is held by the prepare handler below. Initialized by the test program. */
+pthread_barrier_t atfork5mod_window_barrier;
+static atomic_int synced;
+
+void
+atfork5mod_prepare (void)
+{
+ pthread_mutex_lock (&arena_lock);
+ /* Release the registering threads on the first fork only; on later forks
+ they are no longer waiting on the barrier. */
+ if (!atomic_exchange (&synced, 1))
+ pthread_barrier_wait (&atfork5mod_window_barrier);
+ usleep (WINDOW_US);
+}
+
+void
+atfork5mod_parent (void)
+{
+ pthread_mutex_unlock (&arena_lock);
+}
+
+void
+atfork5mod_child (void)
+{
+ pthread_mutex_unlock (&arena_lock);
+}
+
+__attribute__ ((constructor))
+static void
+init (void)
+{
+ init_real ();
+}
--
2.53.0