Re: svn commit: r360233 - in head: contrib/jemalloc . . . : This partially breaks a 2-socket 32-bit powerpc (old PowerMac G4) based on head -r360311

[This explores process crashes that happen during system
shutdown, in a context not having MALLOC_PRODUCTION= .
So assert failures are reported as the stopping points.]

It looks like shutdown -p now, shutdown -r now, and the
like can lead some processes to assert during their exit
attempt, including a sshd failure (that I've not seen
before), rpcbind, and nfsd. I show information about the
observed asserts for those below.


sshd hit an assert, failing slab == extent_slab_get(extent) :

(gdb) bt 
#0  thr_kill () at thr_kill.S:4
#1  0x50927170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52
#2  0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
#3  0x508834b0 in arena_dalloc (tsdn=<optimized out>, ptr=<optimized out>, tcache=<optimized out>, alloc_ctx=<optimized out>, slow_path=<optimized out>)
    at /usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315
#4  idalloctm (tsdn=0x500dd040, ptr=0x5008a180, tcache=0x500dd160, alloc_ctx=<optimized out>, is_internal=<optimized out>, slow_path=<optimized out>)
    at /usr/src/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h:118
#5  0x5087b0a4 in ifree (tsd=0x500dd040, ptr=0x5008a180, tcache=0x500dd160, slow_path=<optimized out>) at jemalloc_jemalloc.c:2590
#6  0x5087acac in __je_free_default (ptr=0x5008a180) at jemalloc_jemalloc.c:2784
#7  0x5087b294 in __free (ptr=0x5008a180) at jemalloc_jemalloc.c:2852
#8  0x10029464 in server_accept_loop (config_s=<optimized out>, sock_in=<optimized out>, sock_out=<optimized out>, newsock=<optimized out>) at /usr/src/crypto/openssh/sshd.c:1185
#9  main (ac=<optimized out>, av=0xffffde3c) at /usr/src/crypto/openssh/sshd.c:2009

. . .
(gdb) up
#2  0x50886cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
67		(void)raise(SIGABRT);
(gdb) up
#3  0x508834b0 in arena_dalloc (tsdn=<optimized out>, ptr=<optimized out>, tcache=<optimized out>, alloc_ctx=<optimized out>, slow_path=<optimized out>)
    at /usr/src/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h:315
315			assert(slab == extent_slab_get(extent));

(gdb) list
310			rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
311			extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
312			    rtree_ctx, (uintptr_t)ptr, true);
313			assert(szind == extent_szind_get(extent));
314			assert(szind < SC_NSIZES);
315			assert(slab == extent_slab_get(extent));
316		}
317	
318		if (likely(slab)) {
319			/* Small allocation. */

More fully:

285	JEMALLOC_ALWAYS_INLINE void
286	arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
287	    alloc_ctx_t *alloc_ctx, bool slow_path) {
288		assert(!tsdn_null(tsdn) || tcache == NULL);
289		assert(ptr != NULL);
290	
291		if (unlikely(tcache == NULL)) {
292			arena_dalloc_no_tcache(tsdn, ptr);
293			return;
294		}
295	
296		szind_t szind;
297		bool slab;
298		rtree_ctx_t *rtree_ctx;
299		if (alloc_ctx != NULL) {
300			szind = alloc_ctx->szind;
301			slab = alloc_ctx->slab;
302			assert(szind != SC_NSIZES);
303		} else {
304			rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
305			rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx,
306			    (uintptr_t)ptr, true, &szind, &slab);
307		}
308	
309		if (config_debug) {
310			rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn));
311			extent_t *extent = rtree_extent_read(tsdn, &extents_rtree,
312			    rtree_ctx, (uintptr_t)ptr, true);
313			assert(szind == extent_szind_get(extent));
314			assert(szind < SC_NSIZES);
315			assert(slab == extent_slab_get(extent));
316		}
317	
318		if (likely(slab)) {
319			/* Small allocation. */
320			tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind,
321			    slow_path);
322		} else {
323			arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path);
324		}
325	}


rpcbind hit an assert, failing ret == sz_size2index_compute(size) :

(gdb) bt
#0  thr_kill () at thr_kill.S:4
#1  0x502f2170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52
#2  0x50251d04 in abort () at /usr/src/lib/libc/stdlib/abort.c:79
#3  0x5024f260 in sz_size2index_lookup (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159
#4  sz_size2index (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:166
#5  imalloc_body (sopts=0xffffb360, dopts=0xffffb340, tsd=0x5009a018) at jemalloc_jemalloc.c:2066
#6  0x50244874 in imalloc (sopts=0xffffb360, dopts=0xffffb340) at /usr/src/contrib/jemalloc/include/jemalloc/internal/tsd.h:331
#7  0x50244fe8 in __calloc (num=1, size=96) at jemalloc_jemalloc.c:2498
#8  0x50265690 in svc_xprt_alloc () at /usr/src/lib/libc/rpc/svc.c:541
#9  0x502635f4 in makefd_xprt (fd=14, sendsize=9000, recvsize=9000) at /usr/src/lib/libc/rpc/svc_vc.c:250
#10 0x502644b4 in rendezvous_request (xprt=0x5004c000, msg=<optimized out>) at /usr/src/lib/libc/rpc/svc_vc.c:315
#11 0x50265a98 in svc_getreq_common (fd=<optimized out>) at /usr/src/lib/libc/rpc/svc.c:640
#12 0x50265d1c in svc_getreq_poll (pfdp=<optimized out>, pollretval=1) at /usr/src/lib/libc/rpc/svc.c:739
#13 0x10018568 in my_svc_run () at /usr/src/usr.sbin/rpcbind/rpcb_svc_com.c:1167
#14 0x10014ad8 in main (argc=<optimized out>, argv=<optimized out>) at /usr/src/usr.sbin/rpcbind/rpcbind.c:250
(gdb) up 3
#3  0x5024f260 in sz_size2index_lookup (size=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:159
159		assert(ret == sz_size2index_compute(size));
(gdb) print ret
$1 = 0

154	JEMALLOC_ALWAYS_INLINE szind_t
155	sz_size2index_lookup(size_t size) {
156		assert(size <= SC_LOOKUP_MAXCLASS);
157		szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1)
158						 >> SC_LG_TINY_MIN]);
159		assert(ret == sz_size2index_compute(size));
160		return ret;
161	}


nfsd hit an assert, failing ret == sz_size2index_compute(size)
(also, but a different caller of sz_size2index):

(gdb) bt
#0  thr_kill () at thr_kill.S:4
#1  0x502b2170 in __raise (s=6) at /usr/src/lib/libc/gen/raise.c:52
#2  0x50211cc0 in abort () at /usr/src/lib/libc/stdlib/abort.c:67
#3  0x50206104 in sz_index2size_lookup (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200
#4  sz_index2size (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:207
#5  ifree (tsd=0x50094018, ptr=0x50041028, tcache=0x50094138, slow_path=<optimized out>) at jemalloc_jemalloc.c:2583
#6  0x50205cac in __je_free_default (ptr=0x50041028) at jemalloc_jemalloc.c:2784
#7  0x50206294 in __free (ptr=0x50041028) at jemalloc_jemalloc.c:2852
#8  0x50287ec8 in ns_src_free (src=0x50329004, srclistsize=<optimized out>) at /usr/src/lib/libc/net/nsdispatch.c:452
#9  ns_dbt_free (dbt=0x50329000) at /usr/src/lib/libc/net/nsdispatch.c:436
#10 vector_free (vec=0x50329000, count=<optimized out>, esize=12, free_elem=<optimized out>) at /usr/src/lib/libc/net/nsdispatch.c:253
#11 nss_atexit () at /usr/src/lib/libc/net/nsdispatch.c:578
#12 0x5028d958 in __cxa_finalize (dso=0x0) at /usr/src/lib/libc/stdlib/atexit.c:240
#13 0x502117f8 in exit (status=0) at /usr/src/lib/libc/stdlib/exit.c:74
#14 0x10013f9c in child_cleanup (signo=<optimized out>) at /usr/src/usr.sbin/nfsd/nfsd.c:969
#15 <signal handler called>
#16 0x00000000 in ?? ()

(gdb) up 3
#3  0x50206104 in sz_index2size_lookup (index=<optimized out>) at /usr/src/contrib/jemalloc/include/jemalloc/internal/sz.h:200
200		assert(ret == sz_index2size_compute(index));

(ret is optimized out.)

197	JEMALLOC_ALWAYS_INLINE size_t
198	sz_index2size_lookup(szind_t index) {
199		size_t ret = (size_t)sz_index2size_tab[index];
200		assert(ret == sz_index2size_compute(index));
201		return ret;
202	}


Booting and immediately trying something like:

service nfsd stop

did not lead to a failure. But may be after
a while it would and be less drastic than a
reboot or power down.

===
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)