[ ssic-linux-Bugs-686748 ] Filesystem stacking deadlock
"SourceForge.net" <[email protected]>
| Newsgroups | gmane.linux.cluster.ssic.devel |
|---|---|
| Message-ID | <[email protected]> |
Bugs item #686748, was opened at 2003-02-14 15:10
Message generated for change (Settings changed) made by rogertsang
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=405834&aid=686748&group_id=32541
Please note that this message will contain a full copy of the comment thread,
including the initial issue submission, for this request,
not just the latest update.
Category: Filesystem
Group: v1.2.0
Status: Open
>Resolution: Fixed
Priority: 5
Private: No
Submitted By: David Zafman (dzafman)
Assigned to: Roger Tsang (rogertsang)
Summary: Filesystem stacking deadlock
Initial Comment:
This has only been seen once. There are other
processes wedged processes, which based on their
stacks, are also blocked on the same inodes i_sem.
This stack trace below shows that if the same processes
recursively calls generic_file_write() twice, it will,
of course, deadlock, leaving the i_sem locked on the
inode.
We might look into the Non-stop Clusters code, to see
how we dealt with this issue before, although it might
be somewhat OS specific.
Stack traceback for pid 8
EBP EIP Function (args)
0xc11f7aa0 0xc011a9c8 schedule+0x478 (0x1, 0xc11f6000,
0xc30cfbe8, 0xc1ff5c08, 0xc3cd94ec)
kernel .text 0xc0100000
0xc011a550 0xc011aa80
0xc11f7ac0 0xc0106199 __down+0x69 (0xc3cd94ec,
0xc11f7b78, 0xc0c5c3a0)
kernel .text 0xc0100000
0xc0106130 0xc0106200
0xc11f7ad4 0xc010634b __down_failed+0xb (0xc11f7b58,
0xffffffff, 0xc3f96000, 0xc11f7b04, 0xc4859a04)
kernel .text 0xc0100000
0xc0106340 0xc0106354
0xc0135868 _text_lock_filemap+0x27b
kernel .text 0xc0100000
0xc01355ed 0xc0135880
0xc11f7ae8 0xc0135561 generic_file_write+0x21
(0xc11f7b58, 0xc3f96000, 0x1000, 0xc11f7b78, 0xc11f6000)
kernel .text 0xc0100000
0xc0135540 0xc0135590
0xc11f7b04 0xc4859a04 [ext3]ext3_file_write+0x44
(0xc11f7b58, 0xc3f96000, 0x1000, 0xc11f7b78, 0xc11f7b58)
ext3 .text 0xc4858060
0xc48599c0 0xc4859a10
0xc11f7bd4 0xc01e3859 cfsd_write+0xb9 (0xc2345360,
0x8a000, 0x0, 0xc3f96000, 0x1000)
kernel .text 0xc0100000
0xc01e37a0 0xc01e38d0
0xc11f7c40 0xc01e012c cfs_proc_write+0x8c (0xc0e8bba0,
0x8a000, 0x0, 0x1000, 0xc3f96000)
kernel .text 0xc0100000
0xc01e00a0 0xc01e0220
0xc11f7c70 0xc01ddf97 cfs_writepage_sync+0x57 (0x0,
0xc0e8bba0, 0xc111e238, 0x0, 0x1000)
kernel .text 0xc0100000
0xc01ddf40 0xc01de020
0xc11f7c9c 0xc01de0be cfs_writepage+0x9e (0xc111e238,
0xc3c1bd90, 0xc11f6000, 0x3b, 0x256)
kernel .text 0xc0100000
0xc01de020 0xc01de100
[1]more>
0xc11f7ccc 0xc0139c96 shrink_cache+0x1f6 (0x6, 0x3,
0x20, 0xc03c4908, 0x6)
kernel .text 0xc0100000
0xc0139aa0 0xc0139e40
0xc11f7cf0 0xc0139f83 shrink_caches+0x53 (0x20,
0xc11f6000, 0x1d2, 0xc03c4908)
kernel .text 0xc0100000
0xc0139f30 0xc0139fb0
0xc11f7d08 0xc0139fe3 try_to_free_pages+0x33 (0x0, 0x0,
0x0, 0xc03c4a84, 0x81)
kernel .text 0xc0100000
0xc0139fb0 0xc013a010
0xc11f7d28 0xc013aa98 balance_classzone+0x68
(0xc11f7d40, 0xc03c4908, 0xc03c4a78, 0x1d2, 0xab)
kernel .text 0xc0100000
0xc013aa30 0xc013ac00
0xc11f7d50 0xc013ad0a __alloc_pages+0x10a (0xc11f7da4)
kernel .text 0xc0100000
0xc013ac00 0xc013ad70
0xc11f7d58 0xc013aa26 _alloc_pages+0x16 (0xc11f7d88,
0x0, 0x1000, 0xfffffff4, 0x0)
kernel .text 0xc0100000
0xc013aa10 0xc013aa30
0xc0135259 __generic_file_write+0x419
(0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f7e38)
kernel .text 0xc0100000
0xc0134e40 0xc0135540
0xc11f7dc8 0xc0135570 generic_file_write+0x30
(0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f6000)
kernel .text 0xc0100000
0xc0135540 0xc0135590
0xc11f7de4 0xc4859a04 [ext3]ext3_file_write+0x44
(0xc11f7e38, 0xc2436000, 0x1000, 0xc11f7e58, 0xc11f7e38)
ext3 .text 0xc4858060
0xc48599c0 0xc4859a10
0xc11f7eb4 0xc01e3859 cfsd_write+0xb9 (0x202, 0x202,
0xc11dd780, 0x0, 0x0)
kernel .text 0xc0100000
0xc01e37a0 0xc01e38d0
0xc11f7e3c 0xc48090c8
[scsi_mod]scsi_queue_next_request+0x48 (0xc044aee0)
scsi_mod .text
0xc4802060 0xc4809080 0xc48091a0
0xc11f7ef4 0xc0123aac bh_action+0x4c (0x8, 0x0, 0x1,
0xc041d500, 0xfffffffe)
[1]more>
kernel .text 0xc0100000
0xc0123a60 0xc0123af0
0xc11f7f10 0xc0123951 tasklet_hi_action+0x61
(0xc0e8bba0, 0xab000, 0x0, 0x1000, 0xc2436000)
kernel .text 0xc0100000
0xc01238f0 0xc01239a0
0xc11f7f50 0xc01ddf97 cfs_writepage_sync+0x57 (0x0,
0xc0e8bba0, 0xc10a2f38, 0x0, 0x1000)
kernel .text 0xc0100000
0xc01ddf40 0xc01de020
0xc11f7f7c 0xc01de0be cfs_writepage+0x9e (0xc10a2f38,
0xc0e8bc64, 0xc01de020, 0x0, 0x4)
kernel .text 0xc0100000
0xc01de020 0xc01de100
0xc11f7fa0 0xc01324b8 filemap_fdatasync+0x88
(0xc0e8bc54, 0x0, 0xc326e464, 0xc326e400, 0xc11f6000)
kernel .text 0xc0100000
0xc0132430 0xc0132510
0xc11f7fc4 0xc015623a sync_unlocked_inodes+0xaa
(0xc11f6000)
kernel .text 0xc0100000
0xc0156190 0xc01564d0
0xc11f7fd0 0xc0144938 sync_old_buffers+0x28 (0x30f00,
0xc11f6000, 0x30f00, 0xc115bf9c, 0xc0105000)
kernel .text 0xc0100000
0xc0144910 0xc01449a0
0xc11f7fec 0xc0144cec kupdate+0x13c
kernel .text 0xc0100000
0xc0144bb0 0xc0144d00
0xc0105ab6 kernel_thread+0x26
kernel .text 0xc0100000
0xc0105a90 0xc0105ad0
----------------------------------------------------------------------
Comment By: Roger Tsang (rogertsang)
Date: 2008-10-06 04:32
Message:
Testing a bug fix. The fix is OS specific (for Linux kernel-2.6.11).
This bug appears on 1.9.x (pre-1.9.6) by putting VM pressure on non-chard
CFS stacked device mapper. The following was reproduced with CFS
FLUSH_SYNC.
Stack traceback for pid 252853
0xf3dd10c0 252853 248638 0 0 D 0xf3dd1280 rsync
EBP EIP Function (args)
0xcf4b5318 0xc03d7bee schedule+0x31e (0xf3dd10c0, 0x1, 0xf3dd10c0,
0xc0119bd0, 0xd1483d30)
0xcf4b5350 0xc03d76c5 __down+0x75 (0x1, 0xc04d1880)
0xcf4b5360 0xc03d7866 __down_failed+0xa (0x0, 0xd1483cac, 0xcf4b5394,
0xc0280e80, 0xcf4b5444)
0xc01426a1 .text.lock.filemap+0x56
0xcf4b5394 0xc01424ca generic_file_writev+0x2a (0xcf4b53f8, 0xdf623aa0,
0x1, 0xcf4b54b0, 0xffffffff)
0xcf4b54a4 0xc028153a cfsd_write+0x1aa (0xcf02f540, 0x802, 0x0,
0xdf623aa0, 0x1)
0xcf4b5534 0xc027be80 cfs_proc_write+0x300 (0x7fe, 0xcf4b55d8,
0xe148ef00)
0xcf4b5580 0xc0142a39 mempool_alloc+0x69 (0xe148ef00, 0xcf4b5560, 0x8,
0x0, 0x0)
0xcf4b5580 0xc0277c88 cfs_async_handler_write+0x68 (0xcf4b55b0,
0xe148ef00, 0x7fe, 0x0, 0x0)
0xcf4b55c0 0xc02780a4 cfs_flush_one+0x184 (0xcf4b55d8, 0xe48bee6c, 0x0,
0xfffe0d59, 0xcf4b55d8)
0xcf4b55ec 0xc02781d1 cfs_flush_list+0x51 (0xcf4b5608, 0x8, 0x0, 0x0,
0xe48bed40)
0xcf4b561c 0xc0278838 cfs_flush_inode+0x88 (0xe48bee6c, 0x0, 0x0, 0x0,
0x0)
0xcf4b5650 0xc02771ac cfs_writepages+0xcc (0xe48bef1c, 0xcf4b575c,
0xc017ce1b)
0xcf4b5664 0xc01457a7 do_writepages+0x27 (0xe48bef1c, 0xcf4b575c, 0x0,
0x0, 0xf2fbb200)
0xcf4b568c 0xc0185990 __sync_single_inode+0x60 (0xe48bee6c, 0xcf4b575c,
0xd8a15aac, 0xcf9d853c, 0xcf9d853c)
0xcf4b56e8 0xc0185b95 __writeback_single_inode+0x35 (0xe48bee6c,
0xcf4b575c, 0xcf4b4000, 0xf2fbb29c, 0xf2fbb294)
0xcf4b5718 0xc0185e5a sync_sb_inodes+0x1aa (0xf2fbb200, 0xcf4b575c, 0x600,
0xcf4b575c, 0x2515b)
0xcf4b5734 0xc01861c6 writeback_inodes+0x106 (0xcf4b575c, 0xcf4b5754,
0xcf4b5758, 0xd1483d5c, 0xf61792b8)
0xcf4b57a8 0xc0145217 balance_dirty_pages+0x47 (0xd1483d5c, 0xd1483d5c,
0x7fe, 0xc146e6f0)
[0]more>
0xcf4b57c0 0xc014538c balance_dirty_pages_ratelimited+0x6c (0xd1483d5c,
0xc146e6f0, 0x2, 0x800, 0x800)
0xcf4b5874 0xc0141679 generic_file_buffered_write+0x2e9 (0xcf4b593c,
0xe7abf7c0, 0x1, 0x1f2a7800, 0x0)
0xcf4b5924 0xc0141dbc __generic_file_aio_write_nolock+0x2cc (0xcf4b593c,
0xe7abf7c0, 0x1, 0xcf4b5b10, 0xf895633a)
0xcf4b59c0 0xc0142140 __generic_file_write_nolock+0x90 (0xcf4b5a58,
0xe7abf7c0, 0x1, 0xcf4b5b10, 0xcf4b5aa4)
0xcf4b59f4 0xc01424f6 generic_file_writev+0x56 (0xcf4b5a58, 0xe7abf7c0,
0x1, 0xcf4b5b10, 0xc0000000)
0xcf4b5b04 0xc028153a cfsd_write+0x1aa (0xcf02f540, 0x1f2a7002, 0x0,
0xe7abf7c0, 0x1)
0xcf4b5b94 0xc027be80 cfs_proc_write+0x300 (0x7fe, 0xcf4b5c38,
0xe148ef80)
0xcf4b5be0 0xc0142a39 mempool_alloc+0x69 (0xe148ef80, 0xcf4b5bc0, 0x8,
0x0, 0x0)
0xcf4b5be0 0xc0277c88 cfs_async_handler_write+0x68 (0xcf4b5c10,
0xe148ef80, 0x7fe, 0x0, 0x4)
0xcf4b5c20 0xc02780a4 cfs_flush_one+0x184 (0xcf4b5c38, 0xe48bee6c, 0x4,
0xffffffff, 0xcf4b5c38)
0xcf4b5c4c 0xc02781d1 cfs_flush_list+0x51 (0xcf4b5c68, 0x8, 0x4, 0x1,
0xe48bed40)
0xcf4b5c7c 0xc0278838 cfs_flush_inode+0x88 (0xe48bee6c, 0x1f2a7, 0x1, 0x4,
0x2)
0xcf4b5ca4 0xc0278959 cfs_sync_inode+0x59 (0xe48bee6c, 0x1f2a7, 0x1, 0x6,
0x3fd)
0xcf4b5cd0 0xc0277ba8 cfs_updatepage+0x158 (0xf46b9dc0, 0xc14acae0, 0x802,
0x3fd, 0xbff)
0xcf4b5d84 0xc0141603 generic_file_buffered_write+0x273 (0xcf4b5ee0,
0xcf4b5e58, 0x1, 0x1f2a7802, 0x0)
0xcf4b5e34 0xc0141dbc __generic_file_aio_write_nolock+0x2cc (0xcf4b5ee0,
0xcf4b5e58, 0x1, 0xcf4b5f1c, 0x0)
0xcf4b5e6c 0xc0142286 generic_file_aio_write+0x76 (0xcf4b5ee0, 0x80923fa,
0x3fd, 0x1f2a7802, 0x0)
0xcf4b5ea8 0xc0275756 __cfs_file_write+0xe6 (0xcf4b5ee0, 0x0, 0x80923fa,
0x3fd, 0xcf4b5ed8)
0xcf4b5ec4 0xc027580e cfs_file_aio_write+0x2e (0xcf4b5ee0, 0x80923fa,
0x3fd, 0x1f2a7802, 0x0)
0xcf4b5f6c 0xc015f7eb do_sync_write+0xab (0x0, 0x1, 0xffffffff,
0xf46b9dc0, 0xf4d7f754)
----------------------------------------------------------------------
Comment By: Roger Tsang (rogertsang)
Date: 2005-09-29 01:19
Message:
Logged In: YES
user_id=1246761
Is this a loopback filesystem?
----------------------------------------------------------------------
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=405834&aid=686748&group_id=32541
-------------------------------------------------------------------------
This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
Build the coolest Linux based applications with Moblin SDK & win great prizes
Grand prize is a trip for two to an Open Source event anywhere in the world
http://moblin-contest.org/redirect.php?banner_id=100&url=/