Re: [PATCH v2] posix: Avoid allocation while holding atfork_lock [BZ 34321]
Adhemerval Zanella Netto <[email protected]>
| Newsgroups | gmane.comp.lib.glibc.alpha |
|---|---|
| Organization | Linaro |
| Message-ID | <[email protected]> |
Ping.
On 14/08/26 12:43, Adhemerval Zanella wrote:
> __register_atfork inserted the new handler into a dynamic array while
> holding atfork_lock, and growing that array calls the public/iterposable
> malloc. An allocator replacement whose own lock is also taken from a
> registered pthread_atfork prepare handler can therefore deadlock:
>
> Thread A: fork Thread B: __register_atfork
> ------------------------------------- -----------------------------------
> :139 unlock(atfork_lock)
> :141 prepare() -> lock(L) [holds L]
> :39 lock(atfork_lock) [holds atfork]
> :44 emplace -> malloc -> lock(L)
> BLOCKS on L (held by A)
> :144 lock(atfork_lock)
> BLOCKS on atfork (held by B)
> => DEADLOCK (A: holds L, waits atfork; B: holds atfork, waits L)
>
> Replace the dynamic array with a doubly linked list (include/list.h).
> A new node is allocated outside atfork_lock and only linked in while the
> lock is held. __unregister_atfork moves nodes to a free pool instead of
> freeing them, and __register_atfork reuses a pooled node when one is
> available, so neither path allocates or frees under atfork_lock in the
> steady state; malloc runs only to grow the pool, always outside the lock.
> This preserves the current allocation guarantee of the dynarray.
>
> Checked on x86_64-linux-gnu and aarch64-linux-gnu.
> ---
> include/register-atfork.h | 5 +-
> posix/register-atfork.c | 278 ++++++++++++++++++-------------
> sysdeps/pthread/Makefile | 7 +
> sysdeps/pthread/tst-atfork5.c | 115 +++++++++++++
> sysdeps/pthread/tst-atfork5mod.c | 175 +++++++++++++++++++
> 5 files changed, 464 insertions(+), 116 deletions(-)
> create mode 100644 sysdeps/pthread/tst-atfork5.c
> create mode 100644 sysdeps/pthread/tst-atfork5mod.c
>
> diff --git a/include/register-atfork.h b/include/register-atfork.h
> index 25286a25aad..2fc0064d9d8 100644
> --- a/include/register-atfork.h
> +++ b/include/register-atfork.h
> @@ -19,7 +19,9 @@
> #ifndef _REGISTER_ATFORK_H
> #define _REGISTER_ATFORK_H
>
> -/* Elements of the fork handler lists. */
> +#include <list_t.h>
> +
> +/* Elements of the fork handler list. */
> struct fork_handler
> {
> void (*prepare_handler) (void);
> @@ -27,6 +29,7 @@ struct fork_handler
> void (*child_handler) (void);
> void *dso_handle;
> uint64_t id;
> + list_t list;
> };
>
> /* Function to call to unregister fork handlers. */
> diff --git a/posix/register-atfork.c b/posix/register-atfork.c
> index 580c6b77caf..ef20f5cbf7e 100644
> --- a/posix/register-atfork.c
> +++ b/posix/register-atfork.c
> @@ -17,94 +17,121 @@
>
> #include <libc-lock.h>
> #include <stdbool.h>
> +#include <stdlib.h>
> +#include <errno.h>
> #include <register-atfork.h>
> #include <intprops.h>
> +#include <list.h>
> #include <stdio.h>
>
> -#define DYNARRAY_ELEMENT struct fork_handler
> -#define DYNARRAY_STRUCT fork_handler_list
> -#define DYNARRAY_PREFIX fork_handler_list_
> -#define DYNARRAY_INITIAL_SIZE 48
> -#include <malloc/dynarray-skeleton.c>
> +/* New handlers are added at the front, so the list runs from the newest
> + (highest ID) at the front to the oldest (lowest ID) at the back. A new
> + node is allocated with malloc only when no reusable node is available, and
> + always *outside* of ATFORK_LOCK: holding ATFORK_LOCK across an allocation
> + would deadlock with interposable allocators whose own locks are also taken
> + from an atfork handler. */
> +static LIST_HEAD (fork_handlers);
> +
> +/* Unregistered nodes are moved here (under ATFORK_LOCK) instead of being
> + freed, and a subsequent registration reuses one. This keeps both
> + registration and unregistration allocation-free in the steady state (in
> + particular avoids any free under ATFORK_LOCK). The pool is released only
> + by __libc_atfork_freemem. */
> +static LIST_HEAD (fork_handlers_free);
>
> -static struct fork_handler_list fork_handlers;
> static uint64_t fork_handler_counter;
>
> +/* Bumped on every list mutation (registration or unregistration). The fork
> + handler runners drop ATFORK_LOCK while a handler executes; comparing this
> + counter across that window tells them whether the list changed and a saved
> + list position is therefore still valid. */
> +static uint64_t fork_handler_modcount;
> +
> static int atfork_lock = LLL_LOCK_INITIALIZER;
>
> +#define fork_handler_entry(pos) list_entry (pos, struct fork_handler, list)
> +
> +/* True if the list LP has no elements. */
> +static inline bool
> +list_is_empty (list_t *lp)
> +{
> + return lp->next == lp;
> +}
> +
> +/* Initialize NEWP from the supplied handlers and link it at the front of the
> + active list. ATFORK_LOCK must be held. */
> +static void
> +fork_handler_init (struct fork_handler *newp, void (*prepare) (void),
> + void (*parent) (void), void (*child) (void),
> + void *dso_handle)
> +{
> + newp->prepare_handler = prepare;
> + newp->parent_handler = parent;
> + newp->child_handler = child;
> + newp->dso_handle = dso_handle;
> +
> + /* IDs assigned to handlers start at 1 and increment with handler
> + registration. Un-registering a handler discards the corresponding ID.
> + It is not reused in future registrations. */
> + if (INT_ADD_OVERFLOW (fork_handler_counter, 1))
> + __libc_fatal ("fork handler counter overflow");
> + newp->id = ++fork_handler_counter;
> +
> + /* Add at the front: the highest ID ends up first. */
> + list_add (&newp->list, &fork_handlers);
> +
> + ++fork_handler_modcount;
> +}
> +
> int
> __register_atfork (void (*prepare) (void), void (*parent) (void),
> void (*child) (void), void *dso_handle)
> {
> lll_lock (atfork_lock, LLL_PRIVATE);
>
> - if (fork_handler_counter == 0)
> - fork_handler_list_init (&fork_handlers);
> -
> - struct fork_handler *newp = fork_handler_list_emplace (&fork_handlers);
> - if (newp != NULL)
> + bool free_empty = list_is_empty (&fork_handlers_free);
> + if (!free_empty)
> {
> - newp->prepare_handler = prepare;
> - newp->parent_handler = parent;
> - newp->child_handler = child;
> - newp->dso_handle = dso_handle;
> -
> - /* IDs assigned to handlers start at 1 and increment with handler
> - registration. Un-registering a handlers discards the corresponding
> - ID. It is not reused in future registrations. */
> - if (INT_ADD_OVERFLOW (fork_handler_counter, 1))
> - __libc_fatal ("fork handler counter overflow");
> - newp->id = ++fork_handler_counter;
> + list_t *reuse = fork_handlers_free.next;
> + list_del (reuse);
> + fork_handler_init (fork_handler_entry (reuse), prepare, parent, child,
> + dso_handle);
> }
>
> - /* Release the lock. */
> lll_unlock (atfork_lock, LLL_PRIVATE);
>
> - return newp == NULL ? ENOMEM : 0;
> + if (!free_empty)
> + return 0;
> +
> + struct fork_handler *newp = malloc (sizeof (*newp));
> + if (newp == NULL)
> + return ENOMEM;
> +
> + lll_lock (atfork_lock, LLL_PRIVATE);
> + fork_handler_init (newp, prepare, parent, child, dso_handle);
> + lll_unlock (atfork_lock, LLL_PRIVATE);
> +
> + return 0;
> }
> libc_hidden_def (__register_atfork)
>
> -static struct fork_handler *
> -fork_handler_list_find (struct fork_handler_list *fork_handlers,
> - void *dso_handle)
> -{
> - for (size_t i = 0; i < fork_handler_list_size (fork_handlers); i++)
> - {
> - struct fork_handler *elem = fork_handler_list_at (fork_handlers, i);
> - if (elem->dso_handle == dso_handle)
> - return elem;
> - }
> - return NULL;
> -}
> -
> void
> __unregister_atfork (void *dso_handle)
> {
> + list_t *runp, *prevp;
> +
> lll_lock (atfork_lock, LLL_PRIVATE);
>
> - struct fork_handler *first = fork_handler_list_find (&fork_handlers,
> - dso_handle);
> - /* Removing is done by shifting the elements in the way the elements
> - that are not to be removed appear in the beginning in dynarray.
> - This avoid the quadradic run-time if a naive strategy to remove and
> - shift one element at time. */
> - if (first != NULL)
> - {
> - struct fork_handler *new_end = first;
> - first++;
> - for (; first != fork_handler_list_end (&fork_handlers); ++first)
> - {
> - if (first->dso_handle != dso_handle)
> - {
> - *new_end = *first;
> - ++new_end;
> - }
> - }
> -
> - ptrdiff_t removed = first - new_end;
> - for (size_t i = 0; i < removed; i++)
> - fork_handler_list_remove_last (&fork_handlers);
> - }
> + /* Move the matching nodes to the free pool rather than freeing them: this
> + avoids a free under ATFORK_LOCK and lets a later registration reuse
> + them. */
> + list_for_each_prev_safe (runp, prevp, &fork_handlers)
> + if (fork_handler_entry (runp)->dso_handle == dso_handle)
> + {
> + list_del (runp);
> + list_add (runp, &fork_handlers_free);
> + ++fork_handler_modcount;
> + }
>
> lll_unlock (atfork_lock, LLL_PRIVATE);
> }
> @@ -117,47 +144,55 @@ __run_prefork_handlers (_Bool do_locking)
> if (do_locking)
> lll_lock (atfork_lock, LLL_PRIVATE);
>
> - /* We run prepare handlers from last to first. After fork, only
> - handlers up to the last handler found here (pre-fork) will be run.
> - Handlers registered during __run_prefork_handlers or
> - __run_postfork_handlers will be positioned after this last handler, and
> - since their prepare handlers won't be run now, their parent/child
> - handlers should also be ignored. */
> + /* We run prepare handlers from newest to oldest (highest ID first). After
> + fork, only handlers up to the last handler found here (pre-fork) will be
> + run. Handlers registered during __run_prefork_handlers or
> + __run_postfork_handlers will have a higher ID, and since their prepare
> + handlers will not be run now, their parent/child handlers should also be
> + ignored. */
> lastrun = fork_handler_counter;
>
> - size_t sl = fork_handler_list_size (&fork_handlers);
> - for (size_t i = sl; i > 0;)
> + /* The newest handlers are at the front; skip any with ID > LASTRUN (those
> + were registered after this function was entered). */
> + list_t *pos = fork_handlers.next;
> + while (pos != &fork_handlers && fork_handler_entry (pos)->id > lastrun)
> + pos = pos->next;
> +
> + while (pos != &fork_handlers)
> {
> - struct fork_handler *runp
> - = fork_handler_list_at (&fork_handlers, i - 1);
> -
> + struct fork_handler *runp = fork_handler_entry (pos);
> uint64_t id = runp->id;
> + void (*prepare_handler) (void) = runp->prepare_handler;
>
> - if (runp->prepare_handler != NULL)
> + /* Remember where to continue and whether the list changed across the
> + handler call. Both are read while the lock is held. */
> + uint64_t saved_modcount = fork_handler_modcount;
> + list_t *next = pos->next;
> +
> + if (prepare_handler != NULL)
> {
> if (do_locking)
> lll_unlock (atfork_lock, LLL_PRIVATE);
>
> - runp->prepare_handler ();
> + prepare_handler ();
>
> if (do_locking)
> lll_lock (atfork_lock, LLL_PRIVATE);
> }
>
> - /* We unlocked, ran the handler, and locked again. In the
> - meanwhile, one or more deregistrations could have occurred leading
> - to the current (just run) handler being moved up the list or even
> - removed from the list itself. Since handler IDs are guaranteed to
> - to be in increasing order, the next handler has to have: */
> -
> - /* A. An earlier position than the current one has. */
> - i--;
> -
> - /* B. A lower ID than the current one does. The code below skips
> - any newly added handlers with higher IDs. */
> - while (i > 0
> - && fork_handler_list_at (&fork_handlers, i - 1)->id >= id)
> - i--;
> + /* Advance to the next (lower ID) handler. If nothing was registered or
> + unregistered while the lock was dropped, the saved position is still
> + valid. Otherwise it may be stale (a node could even have been
> + freed), so re-find the next handler by ID: the one with the greatest
> + ID strictly below the just-run handler. */
> + if (fork_handler_modcount == saved_modcount)
> + pos = next;
> + else
> + {
> + pos = fork_handlers.next;
> + while (pos != &fork_handlers && fork_handler_entry (pos)->id >= id)
> + pos = pos->next;
> + }
> }
>
> return lastrun;
> @@ -167,10 +202,12 @@ void
> __run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
> uint64_t lastrun)
> {
> - size_t sl = fork_handler_list_size (&fork_handlers);
> - for (size_t i = 0; i < sl;)
> + /* Run parent/child handlers from oldest to newest (lowest ID first). The
> + oldest handlers are at the back of the list. */
> + list_t *pos = fork_handlers.prev;
> + while (pos != &fork_handlers)
> {
> - struct fork_handler *runp = fork_handler_list_at (&fork_handlers, i);
> + struct fork_handler *runp = fork_handler_entry (pos);
> uint64_t id = runp->id;
>
> /* prepare handlers were not run for handlers with ID > LASTRUN.
> @@ -178,37 +215,36 @@ __run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
> if (id > lastrun)
> break;
>
> - if (do_locking)
> - lll_unlock (atfork_lock, LLL_PRIVATE);
> + void (*handler) (void) = NULL;
> + if (who == atfork_run_child)
> + handler = runp->child_handler;
> + else if (who == atfork_run_parent)
> + handler = runp->parent_handler;
>
> - if (who == atfork_run_child && runp->child_handler)
> - runp->child_handler ();
> - else if (who == atfork_run_parent && runp->parent_handler)
> - runp->parent_handler ();
> + uint64_t saved_modcount = fork_handler_modcount;
> + list_t *prev = pos->prev;
>
> - if (do_locking)
> - lll_lock (atfork_lock, LLL_PRIVATE);
> + if (handler != NULL)
> + {
> + if (do_locking)
> + lll_unlock (atfork_lock, LLL_PRIVATE);
>
> - /* We unlocked, ran the handler, and locked again. In the meanwhile,
> - one or more [de]registrations could have occurred. Due to this,
> - the list size must be updated. */
> - sl = fork_handler_list_size (&fork_handlers);
> + handler ();
>
> - /* The just-run handler could also have moved up the list. */
> + if (do_locking)
> + lll_lock (atfork_lock, LLL_PRIVATE);
> + }
>
> - if (sl > i && fork_handler_list_at (&fork_handlers, i)->id == id)
> - /* The position of the recently run handler hasn't changed. The
> - next handler to be run is an easy increment away. */
> - i++;
> + /* Advance to the next (higher ID) handler. Fast path when the list did
> + not change; otherwise re-find by ID the handler with the smallest ID
> + strictly above the just-run one. */
> + if (fork_handler_modcount == saved_modcount)
> + pos = prev;
> else
> {
> - /* The next handler to be run is the first handler in the list
> - to have an ID higher than the current one. */
> - for (i = 0; i < sl; i++)
> - {
> - if (fork_handler_list_at (&fork_handlers, i)->id > id)
> - break;
> - }
> + pos = fork_handlers.prev;
> + while (pos != &fork_handlers && fork_handler_entry (pos)->id <= id)
> + pos = pos->prev;
> }
> }
>
> @@ -220,9 +256,21 @@ __run_postfork_handlers (enum __run_fork_handler_type who, _Bool do_locking,
> void
> __libc_atfork_freemem (void)
> {
> + /* Detach both the active list and the free pool under the lock and free
> + them afterwards. */
> + LIST_HEAD (removed);
> + list_t *runp, *prevp;
> +
> lll_lock (atfork_lock, LLL_PRIVATE);
>
> - fork_handler_list_free (&fork_handlers);
> + list_splice (&fork_handlers, &removed);
> + INIT_LIST_HEAD (&fork_handlers);
> +
> + list_splice (&fork_handlers_free, &removed);
> + INIT_LIST_HEAD (&fork_handlers_free);
>
> lll_unlock (atfork_lock, LLL_PRIVATE);
> +
> + list_for_each_prev_safe (runp, prevp, &removed)
> + free (fork_handler_entry (runp));
> }
> diff --git a/sysdeps/pthread/Makefile b/sysdeps/pthread/Makefile
> index d0f3cd59ac6..d5d1df2d56a 100644
> --- a/sysdeps/pthread/Makefile
> +++ b/sysdeps/pthread/Makefile
> @@ -348,6 +348,7 @@ tests += \
> tst-atfork2 \
> tst-atfork3 \
> tst-atfork4 \
> + tst-atfork5 \
> tst-create1 \
> tst-fini1 \
> tst-pt-tls4 \
> @@ -364,6 +365,7 @@ modules-names += \
> tst-atfork2mod \
> tst-atfork3mod \
> tst-atfork4mod \
> + tst-atfork5mod \
> tst-create1mod \
> tst-fini1mod \
> tst-stack2-mod \
> @@ -459,10 +461,15 @@ $(objpfx)tst-atfork4: $(shared-thread-library)
> LDFLAGS-tst-atfork4 = -rdynamic
> $(objpfx)tst-atfork4mod.so: $(shared-thread-library)
>
> +$(objpfx)tst-atfork5: $(shared-thread-library)
> +tst-atfork5-ENV = LD_PRELOAD=$(objpfx)tst-atfork5mod.so
> +$(objpfx)tst-atfork5mod.so: $(shared-thread-library)
> +
> ifeq ($(build-shared),yes)
> $(objpfx)tst-atfork2.out: $(objpfx)tst-atfork2mod.so
> $(objpfx)tst-atfork3.out: $(objpfx)tst-atfork3mod.so
> $(objpfx)tst-atfork4.out: $(objpfx)tst-atfork4mod.so
> +$(objpfx)tst-atfork5.out: $(objpfx)tst-atfork5mod.so
> endif
>
> ifeq ($(build-shared),yes)
> diff --git a/sysdeps/pthread/tst-atfork5.c b/sysdeps/pthread/tst-atfork5.c
> new file mode 100644
> index 00000000000..e3adfc9292d
> --- /dev/null
> +++ b/sysdeps/pthread/tst-atfork5.c
> @@ -0,0 +1,115 @@
> +/* Test that atfork registration does not deadlock against a malloc
> + replacement (BZ 34321).
> + Copyright (C) 2026 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +/* Check if a malloc replacement that takes its own lock both in malloc and
> + in a registered fork prepare handler does not deadlock. One thread holds
> + the atfork lock and waits for the allocator lock inside __register_atfork,
> + while the forking thread holds the allocator lock in the prepare handler
> + and waits for the atfork lock.
> +
> + The malloc replacement is provided by tst-atfork5mod.so through
> + LD_PRELOAD. */
> +
> +#include <dlfcn.h>
> +#include <pthread.h>
> +#include <stdatomic.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +#include <sys/wait.h>
> +
> +#include <support/check.h>
> +#include <support/xthread.h>
> +#include <support/xunistd.h>
> +#include <support/xdlfcn.h>
> +
> +#define NREG 3 /* Number of 'tf' threads. */
> +#define FORK_TARGET 20 /* Forks to complete when there is no deadlock. */
> +#define REG_CAP 40000U /* Upper bound on total handler registrations. */
> +
> +/* Synchronizes the NREG tf threads with the first prepare handler run, so
> + that they only start registering once the allocator lock is held by the
> + preloaded module's prepare handler. */
> +static pthread_barrier_t *window_barrier;
> +
> +static atomic_int running = 1;
> +static atomic_int forks_done;
> +static atomic_uint reg_count;
> +
> +static void *
> +tf (void *closure)
> +{
> + /* Start registering only once a prepare window is open (the module's
> + allocator lock held), so the first handler-list growth happens under
> + that lock. The total number of registrations is bounded so that the
> + handler list stays small and, on a fixed library, the fork handlers
> + run quickly. */
> + xpthread_barrier_wait (window_barrier);
> + while (atomic_load (&running)
> + && atomic_fetch_add (®_count, 1) < REG_CAP)
> + pthread_atfork (NULL, NULL, NULL);
> + return NULL;
> +}
> +
> +static void *
> +forker (void *closure)
> +{
> + while (atomic_load (&running))
> + {
> + pid_t pid = xfork ();
> + if (pid == 0)
> + _exit (0);
> + xwaitpid (pid, NULL, 0);
> + if (atomic_fetch_add (&forks_done, 1) + 1 >= FORK_TARGET)
> + break;
> + }
> + return NULL;
> +}
> +
> +static int
> +do_test (void)
> +{
> + /* Check if tst-atfork5mod.so was preloaded. */
> + void (*mod_prepare) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_prepare");
> + void (*mod_parent) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_parent");
> + void (*mod_child) (void) = xdlsym (RTLD_DEFAULT, "atfork5mod_child");
> + atomic_uint *malloc_count = xdlsym (RTLD_DEFAULT,
> + "atfork5mod_malloc_count");
> + window_barrier = xdlsym (RTLD_DEFAULT, "atfork5mod_window_barrier");
> +
> + xpthread_barrier_init (window_barrier, NULL, NREG + 1);
> + TEST_COMPARE (pthread_atfork (mod_prepare, mod_parent, mod_child), 0);
> +
> + pthread_t reg[NREG];
> + for (int i = 0; i < NREG; i++)
> + reg[i] = xpthread_create (NULL, tf, NULL);
> + pthread_t fork_tid = xpthread_create (NULL, forker, NULL);
> +
> + xpthread_join (fork_tid);
> + atomic_store (&running, 0);
> + for (int i = 0; i < NREG; i++)
> + xpthread_join (reg[i]);
> +
> + TEST_VERIFY (atomic_load (malloc_count) > 0);
> +
> + xpthread_barrier_destroy (window_barrier);
> + return 0;
> +}
> +
> +#define TIMEOUT 8
> +#include <support/test-driver.c>
> diff --git a/sysdeps/pthread/tst-atfork5mod.c b/sysdeps/pthread/tst-atfork5mod.c
> new file mode 100644
> index 00000000000..bb59560db8f
> --- /dev/null
> +++ b/sysdeps/pthread/tst-atfork5mod.c
> @@ -0,0 +1,175 @@
> +/* Malloc replacement module for tst-atfork5 (BZ 34321).
> + Copyright (C) 2026 Free Software Foundation, Inc.
> + This file is part of the GNU C Library.
> +
> + The GNU C Library is free software; you can redistribute it and/or
> + modify it under the terms of the GNU Lesser General Public
> + License as published by the Free Software Foundation; either
> + version 2.1 of the License, or (at your option) any later version.
> +
> + The GNU C Library is distributed in the hope that it will be useful,
> + but WITHOUT ANY WARRANTY; without even the implied warranty of
> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + Lesser General Public License for more details.
> +
> + You should have received a copy of the GNU Lesser General Public
> + License along with the GNU C Library; if not, see
> + <https://www.gnu.org/licenses/>. */
> +
> +/* Minimal malloc interposer, applied to tst-atfork5 through LD_PRELOAD:
> + every allocation takes ARENA_LOCK, and so does the fork prepare handler
> + below (registered by the test program via pthread_atfork). */
> +
> +#include <dlfcn.h>
> +#include <pthread.h>
> +#include <stdatomic.h>
> +#include <stdbool.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +
> +/* Microseconds the prepare handler holds ARENA_LOCK. */
> +#define WINDOW_US 4000
> +
> +static pthread_mutex_t arena_lock = PTHREAD_MUTEX_INITIALIZER;
> +static void *(*real_malloc) (size_t);
> +static void *(*real_calloc) (size_t, size_t);
> +static void *(*real_realloc) (void *, size_t);
> +static void (*real_free) (void *);
> +
> +/* Number of allocations that went through the interposer; the test program
> + uses it to check that this module was actually preloaded. */
> +atomic_uint atfork5mod_malloc_count;
> +
> +/* Small bump buffer to provide the few allocations that happen while dlsym
> + itself runs. */
> +static __thread bool in_dlsym;
> +static char bootbuf[1 << 20];
> +static size_t bootoff;
> +
> +static int
> +is_boot (void *p)
> +{
> + return (char *) p >= bootbuf && (char *) p < bootbuf + sizeof bootbuf;
> +}
> +
> +static void *
> +boot_alloc (size_t n)
> +{
> + size_t o = (bootoff + 15) & ~(size_t) 15;
> + bootoff = o + n;
> + return bootbuf + o;
> +}
> +
> +static void *
> +next_sym (const char *name)
> +{
> + void *sym = dlsym (RTLD_NEXT, name);
> + if (sym == NULL)
> + abort ();
> + return sym;
> +}
> +
> +static void
> +init_real (void)
> +{
> + if (real_malloc != NULL)
> + return;
> + in_dlsym = true;
> + real_malloc = next_sym ("malloc");
> + real_calloc = next_sym ("calloc");
> + real_realloc = next_sym ("realloc");
> + real_free = next_sym ("free");
> + in_dlsym = false;
> +}
> +
> +void *
> +malloc (size_t n)
> +{
> + if (real_malloc == NULL)
> + {
> + if (in_dlsym)
> + return boot_alloc (n);
> + init_real ();
> + }
> + atomic_fetch_add (&atfork5mod_malloc_count, 1);
> + pthread_mutex_lock (&arena_lock);
> + void *p = real_malloc (n);
> + pthread_mutex_unlock (&arena_lock);
> + return p;
> +}
> +
> +void *
> +calloc (size_t a, size_t b)
> +{
> + if (real_malloc == NULL)
> + {
> + if (in_dlsym)
> + return boot_alloc (a * b);
> + init_real ();
> + }
> + atomic_fetch_add (&atfork5mod_malloc_count, 1);
> + pthread_mutex_lock (&arena_lock);
> + void *p = real_calloc (a, b);
> + pthread_mutex_unlock (&arena_lock);
> + return p;
> +}
> +
> +void *
> +realloc (void *old, size_t n)
> +{
> + if (real_malloc == NULL)
> + init_real ();
> + atomic_fetch_add (&atfork5mod_malloc_count, 1);
> + pthread_mutex_lock (&arena_lock);
> + void *p = real_realloc (old, n);
> + pthread_mutex_unlock (&arena_lock);
> + return p;
> +}
> +
> +void
> +free (void *p)
> +{
> + if (is_boot (p))
> + return;
> + if (real_free == NULL)
> + init_real ();
> + pthread_mutex_lock (&arena_lock);
> + real_free (p);
> + pthread_mutex_unlock (&arena_lock);
> +}
> +
> +/* Synchronizes the registering threads of the test program with the first
> + prepare handler run, so that they only start registering once ARENA_LOCK
> + is held by the prepare handler below. Initialized by the test program. */
> +pthread_barrier_t atfork5mod_window_barrier;
> +static atomic_int synced;
> +
> +void
> +atfork5mod_prepare (void)
> +{
> + pthread_mutex_lock (&arena_lock);
> + /* Release the registering threads on the first fork only; on later forks
> + they are no longer waiting on the barrier. */
> + if (!atomic_exchange (&synced, 1))
> + pthread_barrier_wait (&atfork5mod_window_barrier);
> + usleep (WINDOW_US);
> +}
> +
> +void
> +atfork5mod_parent (void)
> +{
> + pthread_mutex_unlock (&arena_lock);
> +}
> +
> +void
> +atfork5mod_child (void)
> +{
> + pthread_mutex_unlock (&arena_lock);
> +}
> +
> +__attribute__ ((constructor))
> +static void
> +init (void)
> +{
> + init_real ();
> +}