fix stgt crash in conn_close
fx chen <[email protected]> Fri, 22 Jul 2016 16:26:01 +0800
| Newsgroups | org.kernel.vger.stgt |
|---|---|
| Message-ID | <CAKbOEtuRKZXM4VGMRErbtWwtEA=0Mk5siH3WRQgOdH7jqzOONg@mail.gmail.com> |
--94eb2c09ccaa5976160538352fe0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
hello!
we found tgtd happen core dump and fix it=E3=80=82
1=EF=BC=9A we use the stgt code
git clone https://github.com/fujita/tgt.git
stgt git commit version: 44c11763d71dc019741c84a857080cd0b4a2f265
2: our test cases
export 10 sata disk with stgt=EF=BC=8Ceach sata disk capacity is 100G=
B,
each target have one lun=EF=BC=8C in iscsi initiator import=EF=BC=8C using =
FIO to to
parallel random write test for 10 devices=EF=BC=8C
during this period the frequent disconnection of iscsi networks=E3=80=
=82
3: tgtd crash, happen core dump
test about 5 hours=EF=BC=8Ctgtd process crash=EF=BC=8C happen core du=
mp=EF=BC=8C the
stack trace showed below:
[New LWP 121362]
[New LWP 121349]
[New LWP 121350]
[New LWP 121370]
[New LWP 121404]
[New LWP 121365]
[New LWP 121348]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `tgtd -d 1'.
Program terminated with signal 11, Segmentation fault.
#0 0x000000000040a259 in __list_del (prev=3D0x7cd38c0,
next=3D0x5aff340) at ./list.h:83
83 prev->next =3D next;
Missing separate debuginfos, use: debuginfo-install
libgcc-4.8.3-9.el7.x86_64 libgcc-4.8.5-4.el7.x86_64
(gdb) bt
#0 0x000000000040a259 in __list_del (prev=3D0x7cd38c0,
next=3D0x5aff340) at ./list.h:83
#1 0x000000000040a296 in list_del (entry=3D0x623d080) at ./list.h:88
#2 0x000000000040edd6 in iscsi_free_cmd_task (task=3D0x623d010) at
iscsi/iscsid.c:1254
#3 0x000000000040ee6a in iscsi_scsi_cmd_done (nid=3D3356, result=3D0,
scmd=3D0x623d0e0) at iscsi/iscsid.c:1269
#4 0x0000000000432dc0 in target_cmd_io_done (cmd=3D0x623d0e0,
result=3D0) at target.c:1236
#5 0x000000000045be39 in bs_sig_request_done (fd=3D10, events=3D1,
data=3D0x0) at bs.c:210
#6 0x0000000000428ff2 in event_loop () at tgtd.c:432
#7 0x0000000000429fca in main (argc=3D3, argv=3D0x7ffffcec25f8) at tgt=
d.c:624
4: Our analysis of tgtd crash
beause the IO during the frequent connection/disconnection of
iscsi networks=EF=BC=8Cthis will trigger close current iscsi session=EF=BC=
=8C
lead to stgt frequent call conn_close function, conn_close will
clear all iscsi task=EF=BC=8Cwe found some ISCSI_OP_SCSI_CMD type's task,
no list_del from session->cmd_list before free it.
5: Our patch for tgtd crash
From 2abf9229ea206172f2fe244539f7f87ed404eff8 Mon Sep 17 00:00:00 2001
From: Chen Fangxian <[email protected]>
Date: Fri, 22 Jul 2016 10:37:23 +0800
Subject: [PATCH] iscsi: fix segfault at conn_close
Remove some iscsi task from conn->session->cmd_list before free it,
otherwise it may cause tgtd process crash. Below is a backtrace info:
Program terminated with signal 11, Segmentation fault.
#0 0x000000000040a259 in __list_del (prev=3D0x7cd38c0, next=3D0x5aff340)
at ./list.h:83
83 prev->next =3D next;
(gdb) bt
#0 0x000000000040a259 in __list_del (prev=3D0x7cd38c0, next=3D0x5aff340)
at ./list.h:83
#1 0x000000000040a296 in list_del (entry=3D0x623d080) at ./list.h:88
#2 0x000000000040edd6 in iscsi_free_cmd_task (task=3D0x623d010) at
iscsi/iscsid.c:1254
#3 0x000000000040ee6a in iscsi_scsi_cmd_done (nid=3D3356, result=3D0,
scmd=3D0x623d0e0) at iscsi/iscsid.c:1269
#4 0x0000000000432dc0 in target_cmd_io_done (cmd=3D0x623d0e0,
result=3D0) at target.c:1236
#5 0x000000000045be39 in bs_sig_request_done (fd=3D10, events=3D1,
data=3D0x0) at bs.c:210
#6 0x0000000000428ff2 in event_loop () at tgtd.c:432
#7 0x0000000000429fca in main (argc=3D3, argv=3D0x7ffffcec25f8) at tgtd.c=
:624
Signed-off-by: Meng Lingkun <[email protected]>
Signed-off-by: Wang Zhengyong <[email protected]>
Reviewed-by: Wang Dongxu <[email protected]>
Signed-off-by: Chen Fangxian <[email protected]>
---
usr/iscsi/conn.c | 32 ++++++++++++++++++++++++++++++--
1 file changed, 30 insertions(+), 2 deletions(-)
diff --git a/usr/iscsi/conn.c b/usr/iscsi/conn.c
index e7d4e8c..3ed08a8 100644
--- a/usr/iscsi/conn.c
+++ b/usr/iscsi/conn.c
@@ -83,6 +83,22 @@ void conn_exit(struct iscsi_connection *conn)
session_put(session);
}
+static int find_task_in_cmd_list(struct iscsi_connection *conn,
+ struct iscsi_task *find_task)
+{
+ struct iscsi_task *task =3D NULL;
+ int find =3D 0;
+
+ list_for_each_entry(task, &conn->session->cmd_list, c_hlist) {
+ if (task =3D=3D find_task) {
+ find =3D 1;
+ break;
+ }
+ }
+
+ return find;
+}
+
void conn_close(struct iscsi_connection *conn)
{
struct iscsi_task *task, *tmp;
@@ -134,6 +150,7 @@ void conn_close(struct iscsi_connection *conn)
list_for_each_entry_safe(task, tmp, &conn->tx_clist, c_list) {
uint8_t op;
+ list_del(&task->c_list);
op =3D task->req.opcode & ISCSI_OPCODE_MASK;
eprintf("Forcing release of tx task %p %" PRIx64 " %x\n",
@@ -146,10 +163,14 @@ void conn_close(struct iscsi_connection *conn)
* would be a better way to see
* task->scmd.c_target though.
*/
- if (task->scmd.c_target)
+ if (task->scmd.c_target) {
iscsi_free_cmd_task(task);
- else
+ } else {
+ if (find_task_in_cmd_list(conn, task))
+ list_del(&task->c_hlist);
+
iscsi_free_task(task);
+ }
break;
case ISCSI_OP_NOOP_IN:
/* NOOP_IN req is allocated within iscsi_tcp
@@ -181,6 +202,9 @@ void conn_close(struct iscsi_connection *conn)
if (conn->rx_task) {
eprintf("Forcing release of rx task %p %" PRIx64 "\n",
conn->rx_task, conn->rx_task->tag);
+ if (find_task_in_cmd_list(conn, conn->rx_task))
+ list_del(&conn->rx_task->c_hlist);
+
iscsi_free_task(conn->rx_task);
}
conn->rx_task =3D NULL;
@@ -193,6 +217,10 @@ void conn_close(struct iscsi_connection *conn)
*/
if (task_in_scsi(task))
continue;
+
+ if (find_task_in_cmd_list(conn, task))
+ list_del(&task->c_hlist);
+
iscsi_free_task(task);
}
done:
--
1.8.3.1
--94eb2c09ccaa5976160538352fe0
Content-Type: application/octet-stream;
name="0001-iscsi-fix-segfault-at-conn_close.patch"
Content-Disposition: attachment;
filename="0001-iscsi-fix-segfault-at-conn_close.patch"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_iqxhfcyq0
RnJvbSAyYWJmOTIyOWVhMjA2MTcyZjJmZTI0NDUzOWY3Zjg3ZWQ0MDRlZmY4IE1vbiBTZXAgMTcg
MDA6MDA6MDAgMjAwMQpGcm9tOiBDaGVuIEZhbmd4aWFuIDxjaGVuZmFuZ3hpYW5AY21zcy5jaGlu
YW1vYmlsZS5jb20+CkRhdGU6IEZyaSwgMjIgSnVsIDIwMTYgMTA6Mzc6MjMgKzA4MDAKU3ViamVj
dDogW1BBVENIXSBpc2NzaTogZml4IHNlZ2ZhdWx0IGF0IGNvbm5fY2xvc2UKClJlbW92ZSBzb21l
IGlzY3NpIHRhc2sgZnJvbSBjb25uLT5zZXNzaW9uLT5jbWRfbGlzdCBiZWZvcmUgZnJlZSBpdCwK
b3RoZXJ3aXNlIGl0IG1heSBjYXVzZSB0Z3RkIHByb2Nlc3MgY3Jhc2guIEJlbG93IGlzIGEgYmFj
a3RyYWNlIGluZm86CgogUHJvZ3JhbSB0ZXJtaW5hdGVkIHdpdGggc2lnbmFsIDExLCBTZWdtZW50
YXRpb24gZmF1bHQuCiAjMCAgMHgwMDAwMDAwMDAwNDBhMjU5IGluIF9fbGlzdF9kZWwgKHByZXY9
MHg3Y2QzOGMwLCBuZXh0PTB4NWFmZjM0MCkgYXQgLi9saXN0Lmg6ODMKIDgzICBwcmV2LT5uZXh0
ID0gbmV4dDsKIChnZGIpIGJ0CiAjMCAgMHgwMDAwMDAwMDAwNDBhMjU5IGluIF9fbGlzdF9kZWwg
KHByZXY9MHg3Y2QzOGMwLCBuZXh0PTB4NWFmZjM0MCkgYXQgLi9saXN0Lmg6ODMKICMxICAweDAw
MDAwMDAwMDA0MGEyOTYgaW4gbGlzdF9kZWwgKGVudHJ5PTB4NjIzZDA4MCkgYXQgLi9saXN0Lmg6
ODgKICMyICAweDAwMDAwMDAwMDA0MGVkZDYgaW4gaXNjc2lfZnJlZV9jbWRfdGFzayAodGFzaz0w
eDYyM2QwMTApIGF0IGlzY3NpL2lzY3NpZC5jOjEyNTQKICMzICAweDAwMDAwMDAwMDA0MGVlNmEg
aW4gaXNjc2lfc2NzaV9jbWRfZG9uZSAobmlkPTMzNTYsIHJlc3VsdD0wLCBzY21kPTB4NjIzZDBl
MCkgYXQgaXNjc2kvaXNjc2lkLmM6MTI2OQogIzQgIDB4MDAwMDAwMDAwMDQzMmRjMCBpbiB0YXJn
ZXRfY21kX2lvX2RvbmUgKGNtZD0weDYyM2QwZTAsIHJlc3VsdD0wKSBhdCB0YXJnZXQuYzoxMjM2
CiAjNSAgMHgwMDAwMDAwMDAwNDViZTM5IGluIGJzX3NpZ19yZXF1ZXN0X2RvbmUgKGZkPTEwLCBl
dmVudHM9MSwgZGF0YT0weDApIGF0IGJzLmM6MjEwCiAjNiAgMHgwMDAwMDAwMDAwNDI4ZmYyIGlu
IGV2ZW50X2xvb3AgKCkgYXQgdGd0ZC5jOjQzMgogIzcgIDB4MDAwMDAwMDAwMDQyOWZjYSBpbiBt
YWluIChhcmdjPTMsIGFyZ3Y9MHg3ZmZmZmNlYzI1ZjgpIGF0IHRndGQuYzo2MjQKClNpZ25lZC1v
ZmYtYnk6IE1lbmcgTGluZ2t1biA8bWVuZ2xpbmdrdW5AY21zcy5jaGluYW1vYmlsZS5jb20+ClNp
Z25lZC1vZmYtYnk6IFdhbmcgWmhlbmd5b25nIDx3YW5nemhlbmd5b25nQGNtc3MuY2hpbmFtb2Jp
bGUuY29tPgpSZXZpZXdlZC1ieTogV2FuZyBEb25neHUgPHdhbmdkb25neHVAY21zcy5jaGluYW1v
YmlsZS5jb20+ClNpZ25lZC1vZmYtYnk6IENoZW4gRmFuZ3hpYW4gPGNoZW5mYW5neGlhbkBjbXNz
LmNoaW5hbW9iaWxlLmNvbT4KLS0tCiB1c3IvaXNjc2kvY29ubi5jIHwgMzIgKysrKysrKysrKysr
KysrKysrKysrKysrKysrKysrLS0KIDEgZmlsZSBjaGFuZ2VkLCAzMCBpbnNlcnRpb25zKCspLCAy
IGRlbGV0aW9ucygtKQoKZGlmZiAtLWdpdCBhL3Vzci9pc2NzaS9jb25uLmMgYi91c3IvaXNjc2kv
Y29ubi5jCmluZGV4IGU3ZDRlOGMuLjNlZDA4YTggMTAwNjQ0Ci0tLSBhL3Vzci9pc2NzaS9jb25u
LmMKKysrIGIvdXNyL2lzY3NpL2Nvbm4uYwpAQCAtODMsNiArODMsMjIgQEAgdm9pZCBjb25uX2V4
aXQoc3RydWN0IGlzY3NpX2Nvbm5lY3Rpb24gKmNvbm4pCiAJCXNlc3Npb25fcHV0KHNlc3Npb24p
OwogfQogCitzdGF0aWMgaW50IGZpbmRfdGFza19pbl9jbWRfbGlzdChzdHJ1Y3QgaXNjc2lfY29u
bmVjdGlvbiAqY29ubiwKKwkJCQlzdHJ1Y3QgaXNjc2lfdGFzayAqZmluZF90YXNrKQoreworCXN0
cnVjdCBpc2NzaV90YXNrICp0YXNrID0gTlVMTDsKKwlpbnQgZmluZCA9IDA7CisKKwlsaXN0X2Zv
cl9lYWNoX2VudHJ5KHRhc2ssICZjb25uLT5zZXNzaW9uLT5jbWRfbGlzdCwgY19obGlzdCkgewor
CQlpZiAodGFzayA9PSBmaW5kX3Rhc2spIHsKKwkJCWZpbmQgPSAxOworCQkJYnJlYWs7CisJCX0K
Kwl9CisKKwlyZXR1cm4gZmluZDsKK30KKwogdm9pZCBjb25uX2Nsb3NlKHN0cnVjdCBpc2NzaV9j
b25uZWN0aW9uICpjb25uKQogewogCXN0cnVjdCBpc2NzaV90YXNrICp0YXNrLCAqdG1wOwpAQCAt
MTM0LDYgKzE1MCw3IEBAIHZvaWQgY29ubl9jbG9zZShzdHJ1Y3QgaXNjc2lfY29ubmVjdGlvbiAq
Y29ubikKIAlsaXN0X2Zvcl9lYWNoX2VudHJ5X3NhZmUodGFzaywgdG1wLCAmY29ubi0+dHhfY2xp
c3QsIGNfbGlzdCkgewogCQl1aW50OF90IG9wOwogCisJCWxpc3RfZGVsKCZ0YXNrLT5jX2xpc3Qp
OwogCQlvcCA9IHRhc2stPnJlcS5vcGNvZGUgJiBJU0NTSV9PUENPREVfTUFTSzsKIAogCQllcHJp
bnRmKCJGb3JjaW5nIHJlbGVhc2Ugb2YgdHggdGFzayAlcCAlIiBQUkl4NjQgIiAleFxuIiwKQEAg
LTE0NiwxMCArMTYzLDE0IEBAIHZvaWQgY29ubl9jbG9zZShzdHJ1Y3QgaXNjc2lfY29ubmVjdGlv
biAqY29ubikKIAkJCSAqIHdvdWxkIGJlIGEgYmV0dGVyIHdheSB0byBzZWUKIAkJCSAqIHRhc2st
PnNjbWQuY190YXJnZXQgdGhvdWdoLgogCQkJICovCi0JCQlpZiAodGFzay0+c2NtZC5jX3Rhcmdl
dCkKKwkJCWlmICh0YXNrLT5zY21kLmNfdGFyZ2V0KSB7CiAJCQkJaXNjc2lfZnJlZV9jbWRfdGFz
ayh0YXNrKTsKLQkJCWVsc2UKKwkJCX0gZWxzZSB7CisJCQkJaWYgKGZpbmRfdGFza19pbl9jbWRf
bGlzdChjb25uLCB0YXNrKSkKKwkJCQkJbGlzdF9kZWwoJnRhc2stPmNfaGxpc3QpOworCiAJCQkJ
aXNjc2lfZnJlZV90YXNrKHRhc2spOworCQkJfQogCQkJYnJlYWs7CiAJCWNhc2UgSVNDU0lfT1Bf
Tk9PUF9JTjoKIAkJCS8qIE5PT1BfSU4gcmVxIGlzIGFsbG9jYXRlZCB3aXRoaW4gaXNjc2lfdGNw
CkBAIC0xODEsNiArMjAyLDkgQEAgdm9pZCBjb25uX2Nsb3NlKHN0cnVjdCBpc2NzaV9jb25uZWN0
aW9uICpjb25uKQogCWlmIChjb25uLT5yeF90YXNrKSB7CiAJCWVwcmludGYoIkZvcmNpbmcgcmVs
ZWFzZSBvZiByeCB0YXNrICVwICUiIFBSSXg2NCAiXG4iLAogCQkJY29ubi0+cnhfdGFzaywgY29u
bi0+cnhfdGFzay0+dGFnKTsKKwkJaWYgKGZpbmRfdGFza19pbl9jbWRfbGlzdChjb25uLCBjb25u
LT5yeF90YXNrKSkKKwkJCWxpc3RfZGVsKCZjb25uLT5yeF90YXNrLT5jX2hsaXN0KTsKKwogCQlp
c2NzaV9mcmVlX3Rhc2soY29ubi0+cnhfdGFzayk7CiAJfQogCWNvbm4tPnJ4X3Rhc2sgPSBOVUxM
OwpAQCAtMTkzLDYgKzIxNywxMCBAQCB2b2lkIGNvbm5fY2xvc2Uoc3RydWN0IGlzY3NpX2Nvbm5l
Y3Rpb24gKmNvbm4pCiAJCSAqLwogCQlpZiAodGFza19pbl9zY3NpKHRhc2spKQogCQkJY29udGlu
dWU7CisKKwkJaWYgKGZpbmRfdGFza19pbl9jbWRfbGlzdChjb25uLCB0YXNrKSkKKwkJCWxpc3Rf
ZGVsKCZ0YXNrLT5jX2hsaXN0KTsKKwogCQlpc2NzaV9mcmVlX3Rhc2sodGFzayk7CiAJfQogZG9u
ZToKLS0gCjEuOC4zLjEKCg==
--94eb2c09ccaa5976160538352fe0--