[SSI] openssi/kernel/cluster/ssi/ipc ipcmsg_svr.c, 1.10, 1.11 ipcsem_svr.c, 1.10, 1.11 ipcshm_svr.c, 1.23, 1.24 namesvr_func.c, 1.21, 1.22
Roger Tsang <[email protected]> Mon, 29 Mar 2010 06:17:02 +0000
| Newsgroups | gmane.linux.cluster.ssic.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv16378/kernel/cluster/ssi/ipc
Modified Files:
Tag: OPENSSI-FC
ipcmsg_svr.c ipcsem_svr.c ipcshm_svr.c namesvr_func.c
Log Message:
IPC:
- Optimize away redundant initialization in newseg() and do_ssi_shm_noclients().
- Initialize shm_node inside newseg(). Optimize away shm_un/lock() dance.
- Test shm_sem_owned() shm_ids.sem semaphore in ssi_shm_noclients().
- Fix shmctl(IPC_RMID) was not setting SHM_DEST flag on other clients in the cluster.
- Fix IPC id 0 race. ipc_get_locks() / ipc_drop_locks() skipped id 0.
IPC (#ifdef IPC_SHM_RACE_FIX):
- Fix shm_svr_addid() array bounds violation when /proc/sys/kernel/shmmni is greater than 4096 (SHMNMI).
- Fix dereferencing invalid shm_kernel_svr structure after calling ssi_shm_noclients().
- Fix do_ssi_shm_noclients() remote clients did not wait for server to failover.
cluster/ssi/ipc/ipcmsg_svr.c | 4
cluster/ssi/ipc/ipcsem_svr.c | 4
cluster/ssi/ipc/ipcshm_svr.c | 114 +++++++------
cluster/ssi/ipc/namesvr_func.c | 8
include/cluster/ssi/ipc/shm.h | 3
ipc/shm.c | 281 ++++++++++++++++++---------------
ipc/util.c | 9 -
ipc/util.h | 2
8 files changed, 235 insertions(+), 190 deletions(-)
Index: ipcmsg_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc/ipcmsg_svr.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- ipcmsg_svr.c 2 Feb 2010 05:19:30 -0000 1.10
+++ ipcmsg_svr.c 29 Mar 2010 06:16:59 -0000 1.11
@@ -54,7 +54,7 @@
#define DATALEN_SEG (PAGE_SIZE-sizeof(struct msg_msgseg))
typedef unsigned long lid_t;
-extern void ipc_drop_locks(int, struct kern_ipc_perm *, struct ipc_ids *, int);
+extern void ipc_drop_locks(struct kern_ipc_perm *, struct ipc_ids *);
extern struct kern_ipc_perm * ipc_get_locks(int, struct ipc_ids *, int);
/*
@@ -83,7 +83,7 @@
if (buf != NULL)
{
memcpy (*msq, buf, sizeof(struct msg_queue));
- ipc_drop_locks(msgid, (struct kern_ipc_perm *)buf, &msg_ids, 0);
+ ipc_drop_locks((struct kern_ipc_perm *)buf, NULL);
*len = sizeof(struct msg_queue);
}
done:
Index: ipcshm_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc/ipcshm_svr.c,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- ipcshm_svr.c 21 Mar 2010 05:33:50 -0000 1.23
+++ ipcshm_svr.c 29 Mar 2010 06:17:00 -0000 1.24
@@ -54,7 +54,7 @@
extern struct ipc_ids shm_ids;
extern struct ipc_ids shm_ids_svr;
extern void ipc_rcu_putref(void *ptr);
-extern void ipc_drop_locks(int, struct kern_ipc_perm *, struct ipc_ids *, int);
+extern void ipc_drop_locks(struct kern_ipc_perm *, struct ipc_ids *);
extern struct kern_ipc_perm * ipc_get_locks(int, struct ipc_ids *, int);
#ifdef IPC_SHM_RACE_FIX
extern void ipc_rcu_getref(void *ptr);
@@ -93,7 +93,7 @@
memcpy (*shmidk, buf, sizeof(struct shmid_kernel));
*segsize = shm_get_segsize(buf);
*cprid = shm_get_cpid(buf);
- ipc_drop_locks(shmid, (struct kern_ipc_perm *)buf, &shm_ids, 0);
+ ipc_drop_locks((struct kern_ipc_perm *)buf, NULL);
*len = sizeof(struct shmid_kernel);
}
done:
@@ -174,7 +174,7 @@
shp = (struct shmid_kernel *)ipc_get_locks(shmid, &shm_ids, 1);
if (!shp) {
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
*rval = -EINVAL;
return 0;
}
@@ -187,10 +187,10 @@
*nattch = shp->shm_nattch;
*rval = 0;
if (dest != 2)
- ipc_drop_locks(shmid, (struct kern_ipc_perm *)shp, &shm_ids, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)shp, &shm_ids);
else {
shp->shm_flags |= SHM_LOCK_DEST;
- ipc_drop_locks(shmid, (struct kern_ipc_perm *)shp, &shm_ids, 0);
+ ipc_drop_locks((struct kern_ipc_perm *)shp, NULL);
#ifdef IPC_SHM_RACE_FIX
ipc_set_lock_dest(&shm_ids, shmid);
#endif
@@ -239,26 +239,35 @@
clusternode_t node;
nsc_nlcookie_t cookie;
int ret, cnt;
- int nattch = 0;
- int rval = 0;
- int nm_svr_num = 0;
+ int rval, nattch, nm_svr_num;
+#ifdef IPC_SHM_RACE_FIX
+retry:
+#endif
if (svrnode != this_node) {
ret = RIPC_SHM_NOCLIENTS(svrnode, &rval, id, dest);
- if (ret == -EREMOTE)
+ if (ret == -EREMOTE) {
+#ifdef IPC_SHM_RACE_FIX
+ idelay(HZ);
+ goto retry;
+#else
rval = 1;
+#endif
+ }
if (rval == -EIDRM) {
- ipc_get_locks(0, &shm_ids, 1);
+ ipc_get_locks(-1, &shm_ids, 1);
cli_ipcname_rmid(NAME_SERVICE_SHM, id);
rval = ripc_shm_rmid(this_node, &rval, id, 1);
+ printk(KERN_WARNING "%s: id %d not found on server\n",
+ __FUNCTION__, id);
}
-
return rval;
}
- if (dest != 2)
- ripc_shm_get_nattchs(this_node, &rval, id, &nattch, dest);
- nm_svr_num = nattch;
+ if (dest != 2) {
+ ripc_shm_get_nattchs(this_node, &rval, id, &nm_svr_num, dest);
+ } else
+ nm_svr_num = 0;
#ifdef IPC_SHM_RACE_FIX
nl = NSC_NODELIST_ALLOC();
@@ -320,22 +329,24 @@
return cnt;
}
-/* It is assumed that a non-null shp is passed as the first argument */
+/* Called with shp locked. Returns with shp unlocked */
int
-ssi_shm_noclients(struct shmid_kernel *shp, int dest, int tbl, int lck)
+ssi_shm_noclients(struct shmid_kernel *shp, int dest)
{
- int id = shp->id;
clusternode_t svrnode = shp->shm_node;
- int ret = 0;
+ int ret, id = shp->id;
- ipc_drop_locks((lck ? id : 0), (struct kern_ipc_perm *)shp,
- &shm_ids, tbl);
+ WARN_ON(shm_sem_owned());
+
+ /* Drop the locks since we might go remote */
+ ipc_drop_locks((struct kern_ipc_perm *)shp, NULL);
ret = do_ssi_shm_noclients(id, svrnode, dest);
- shp = (struct shmid_kernel *)ipc_get_locks(
- (lck ? id : 0), &shm_ids, tbl);
- WARN_ON(lck && !shp);
+#ifndef IPC_SHM_RACE_FIX
+ shp = (struct shmid_kernel *)ipc_get_locks(id, &shm_ids, 0);
+ WARN_ON(!shp);
+#endif
return ret;
}
@@ -414,12 +425,12 @@
#ifdef IPC_SHM_RACE_FIX
shmp = shm_svr_lock(id);
#else
- ipc_get_locks(0, &shm_ids, 1);
+ ipc_get_locks(-1, &shm_ids, 1);
shmp = (struct shmid_kernel_svr *)shm_svr_get(id);
#endif
if (shmp == NULL) {
#ifndef IPC_SHM_RACE_FIX
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
#endif
*rval = -EINVAL;
return 0;
@@ -429,7 +440,7 @@
#ifdef IPC_SHM_RACE_FIX
shm_svr_unlock(shmp);
#else
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
#endif
*rval = 0;
return 0;
@@ -451,7 +462,7 @@
shp = (struct shmid_kernel *)ipc_get_locks(id, &shm_ids, 0);
if (!shp) {
/* Drop the locks acquired by caller */
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
*rval = -EIDRM;
return 0;
}
@@ -462,9 +473,9 @@
#ifdef IPC_SHM_RACE_FIX
/* ssi_local_destroy() returns with shp unlocked */
ipc_set_lock_dest(&shm_ids, -1);
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
#else
- ipc_drop_locks(id, (struct kern_ipc_perm *)shp, &shm_ids, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)shp, &shm_ids);
#endif
*rval = 0;
return 0;
@@ -502,7 +513,7 @@
#ifdef IPC_SHM_RACE_FIX
struct shmid_kernel *shp = shm_cli_lock(id);
if (!shp) {
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
*rval = -EIDRM;
return 0;
}
@@ -513,7 +524,7 @@
if (shp)
shp->shm_flags &= ~SHM_LOCK_DEST;
#endif /* !IPC_SHM_LOCK_DEST_FIX */
- ipc_drop_locks(id, (struct kern_ipc_perm *)shp, &shm_ids, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)shp, &shm_ids);
*rval = 0;
return 0;
}
@@ -538,15 +549,15 @@
memcpy(nl, svp->shm_nodelist, sizeof(*nl));
shm_svr_unlock(svp);
#else
- ipc_get_locks(0, &shm_ids_svr, 1);
+ ipc_get_locks(-1, &shm_ids_svr, 1);
svp = (struct shmid_kernel_svr *)shm_svr_get(id);
if (!svp) {
- ipc_drop_locks(0, NULL, &shm_ids_svr, 1);
+ ipc_drop_locks(NULL, &shm_ids_svr);
return;
}
nl = NSC_NODELIST_COPY(svp->shm_nodelist);
- ipc_drop_locks(0, (struct kern_ipc_perm *)svp, &shm_ids_svr, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)svp, &shm_ids_svr);
#endif /* !IPC_SHM_RACE_FIX */
cookie = CLUSTERNODE_INVAL;
@@ -599,14 +610,14 @@
} else {
/* caller is remote. Called with no locks held */
shp = NULL;
- ipc_get_locks(0, &shm_ids, 1);
+ ipc_get_locks(-1, &shm_ids, 1);
}
/* shm_ids.sem held */
ret = do_ssi_shm_noclients(id, svrnode, 2);
if (ret) {
if (!do_locks)
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
#ifdef IPC_SHM_DESTROY_FIX
/* [ ssic-linux-Bugs-1944781 ]
@@ -625,8 +636,8 @@
shp = shm_cli_lock(id);
svp = (struct shmid_kernel_svr *)shm_svr_get(id);
if (!svp || shp->shm_nattch) {
- ipc_drop_locks(id, (struct kern_ipc_perm *)shp,
- &shm_ids, !do_locks);
+ ipc_drop_locks((struct kern_ipc_perm *)shp,
+ (!do_locks) ? &shm_ids : NULL);
/* drop remote locks because has attachs */
abort_rmid(id);
@@ -646,7 +657,7 @@
(void) ipc_get_locks(id, &shm_ids_svr, 1);
svp = shm_svr_rmid(id);
ipc_rcu_getref(svp);
- ipc_drop_locks(id, (struct kern_ipc_perm *)svp, &shm_ids_svr, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)svp, &shm_ids_svr);
nl = svp->shm_nodelist;
shm_svr_cleanup(svp); /* might sleep */
@@ -656,9 +667,9 @@
ipc_rcu_putref(svp); /* active */
shm_svr_unlock(svp);
#else /* IPC_SHM_RACE_FIX */
- ipc_get_locks(0, &shm_ids_svr, 1);
+ ipc_get_locks(-1, &shm_ids_svr, 1);
shm_svr_rmid(id);
- ipc_drop_locks(0, NULL, &shm_ids_svr, 1);
+ ipc_drop_locks(NULL, &shm_ids_svr);
nl = svp->shm_nodelist;
shm_svr_cleanup(svp);
@@ -680,7 +691,7 @@
}
NSC_NODELIST_FREE(nl); /* svp->shm_nodelist */
if (!do_locks)
- ipc_drop_locks(0, NULL, &shm_ids, 1);
+ ipc_drop_locks(NULL, &shm_ids);
}
else {
shp = shm_cli_get(id);
@@ -688,14 +699,14 @@
/* Must drop locks since RIPC_SHM_CLEANUP() does
* RIPC_SHM_GET_NATTCHS() callback which acquires shm_ids.sem
*/
- ipc_drop_locks(id, (struct kern_ipc_perm *)shp, &shm_ids, 1);
+ ipc_drop_locks((struct kern_ipc_perm *)shp, &shm_ids);
ret = RIPC_SHM_CLEANUP(svrnode, &rval, id, clinode);
+#ifndef IPC_SHM_RACE_FIX
/* Reacquire the locks */
- ipc_get_locks(0, &shm_ids, 1);
+ ipc_get_locks(-1, &shm_ids, 1);
-#ifndef IPC_SHM_RACE_FIX
#ifdef IPC_SHM_DESTROY_FIX
/* Ignore -EIDRM already removed at server */
if (ret == -EREMOTE) {
@@ -704,10 +715,8 @@
#endif
cli_ipcname_rmid(NAME_SERVICE_SHM, id);
ret = ripc_shm_rmid(this_node, &rval, id, 1);
-#ifndef IPC_SHM_RACE_FIX
/* ripc_shm_rmid() returns with shm_ids.sem unlocked */
- ipc_get_locks(0, &shm_ids, 1);
-#endif
+ ipc_get_locks(-1, &shm_ids, 1);
}
if (ret)
rval = ret;
@@ -715,12 +724,17 @@
printk(KERN_WARNING "%s: Failed to cleanup IPC shm "
"structures %d\n", __FUNCTION__, rval);
#else /* !IPC_SHM_RACE_FIX */
- /* Ignore -EIDRM already removed at server */
- /* Ignore -EREMOTE cleaned up by ipc_shm_nodedown() */
+ /* Reacquire the locks */
+ ipc_get_locks(-1, &shm_ids, 1);
+
if (ret)
rval = ret;
switch (rval) {
case 0:
+ case -EIDRM:
+ /* Ignore -EIDRM already removed at server */
+ case -EREMOTE:
+ /* Ignore -EREMOTE cleaned up by ipc_shm_nodedown() */
case -EBUSY:
case -ESRCH:
break;
Index: ipcsem_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc/ipcsem_svr.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- ipcsem_svr.c 2 Feb 2010 05:19:30 -0000 1.10
+++ ipcsem_svr.c 29 Mar 2010 06:17:00 -0000 1.11
@@ -47,7 +47,7 @@
extern clusternode_t name_server_node;
extern struct ipc_ids sem_ids;
-extern void ipc_drop_locks(int, struct kern_ipc_perm *, struct ipc_ids *, int);
+extern void ipc_drop_locks(struct kern_ipc_perm *, struct ipc_ids *);
extern struct kern_ipc_perm * ipc_get_locks(int, struct ipc_ids *, int);
int
@@ -72,7 +72,7 @@
if (buf != NULL)
{
memcpy (*sema, buf, sizeof(struct sem_array));
- ipc_drop_locks(semid, (struct kern_ipc_perm *)buf, &sem_ids, 0);
+ ipc_drop_locks((struct kern_ipc_perm *)buf, NULL);
*len = sizeof(struct sem_array);
}
done:
Index: namesvr_func.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc/namesvr_func.c,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- namesvr_func.c 21 Mar 2010 05:26:04 -0000 1.21
+++ namesvr_func.c 29 Mar 2010 06:17:00 -0000 1.22
@@ -73,7 +73,7 @@
extern struct ipc_ids sem_ids;
extern struct ipc_ids shm_ids;
#ifdef IPC_SHM_RACE_FIX
-extern void ipc_drop_locks(int, struct kern_ipc_perm *, struct ipc_ids *, int);
+extern void ipc_drop_locks(struct kern_ipc_perm *, struct ipc_ids *);
extern struct kern_ipc_perm * ipc_get_locks(int, struct ipc_ids *, int);
#endif
extern struct shmid_kernel_svr *shm_svr_get(int);
@@ -1137,10 +1137,10 @@
*/
for (i = 0; i < NAME_SERVICE_MAX; i++) {
#ifdef IPC_SHM_RACE_FIX
- ipc_get_locks(0, ipcname_svc_dbs[i], 1);
+ ipc_get_locks(-1, ipcname_svc_dbs[i], 1);
num_objects[i] = ipc_get_inuse(ipcname_svc_dbs[i]);
if (num_objects[i] == 0)
- ipc_drop_locks(0, NULL, ipcname_svc_dbs[i], 1);
+ ipc_drop_locks(NULL, ipcname_svc_dbs[i]);
#else
num_objects[i] = ipc_get_inuse(ipcname_svc_dbs[i]);
#endif
@@ -1216,7 +1216,7 @@
break;
}
#ifdef IPC_SHM_RACE_FIX
- ipc_drop_locks(0, NULL, ipcname_svc_dbs[i], 1);
+ ipc_drop_locks(NULL, ipcname_svc_dbs[i]);
#endif
}
------------------------------------------------------------------------------
Download Intel® Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev