[SSI] openssi/kernel/cluster/ssi/ipc ipcshm_svr.c,1.22,1.23

Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv21204/kernel/cluster/ssi/ipc

Modified Files:
      Tag: OPENSSI-FC
	ipcshm_svr.c 
Log Message:
IPC (#ifdef IPC_SHM_RACE_FIX):
- Fix livelock in do_ssi_shm_noclients() in svr_ripc_shm_cleanup() path due to deadlock with icssvr_nodedown_svc_wait() in ipc_shm_nodedown(). In do_ssi_shm_noclients() when RIPC_SHM_GET_NATTCHS() fails shm_nodelist in shmid_kernel_svr structure is re-read before retrying. It retries indefinitely. There is no livelock as long as down nodes are promptly removed from shm_nodelist, but the down node is not removed from shm_nodelist until after icssvr_nodedown_svc_wait() completes in ipc_shm_nodedown(). A deadlock occurs because icssvr_nodedown_svr_wait() waits for svr_ripc_shm_cleanup() to complete.
- Remove shm_sem_owned(). No longer used.
- Fix bug introduced in CVS tag OPENSSI-FC-1-9-6-PRE20. Remote SHM client stuck in shm_destroy(). cli_ripc_shm_cleanup() deadlock with ripc_shm_get_nattchs(). shm_ids.sem mutex is held during cli_ripc_shm_cleanup() which is a blocking RPC and triggers ripc_shm_get_nattchs() callback. A deadlock occurs when the callback waits for shm_ids.sem already held prior the RPC.
- ssi_shm_cleanup() at remote client no longer handles -EREMOTE (aka. node down); SHM segment is cleaned up during nodedown. Now ripc_shm_rmid() drops shm_ids.sem for caller since there is no longer need to return with shm_ids.sem held.
- Make abort_rmid() more robust during low memory.
- Fix RIPC_SHMCTL() in sys_shmctl(SHM_INFO) path deadlock with remote RIPC_SHM_GET_NATTCHS() in ssi_shm_cleanup() path.  Both callers hold shm_ids.sem at their node during RPC.

 cluster/ssi/ipc/ipcshm_svr.c  |   88 ++++++++++++++++++++--------------
 include/cluster/ssi/ipc/shm.h |    1 
 ipc/shm.c                     |   22 +++++---
 3 files changed, 67 insertions(+), 44 deletions(-)


Index: ipcshm_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/ipc/ipcshm_svr.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- ipcshm_svr.c	7 Mar 2010 05:40:14 -0000	1.22
+++ ipcshm_svr.c	21 Mar 2010 05:33:50 -0000	1.23
@@ -251,9 +251,6 @@
 			ipc_get_locks(0, &shm_ids, 1);
 			cli_ipcname_rmid(NAME_SERVICE_SHM, id);
 			rval = ripc_shm_rmid(this_node, &rval, id, 1);
-#ifdef IPC_SHM_RACE_FIX
-			ipc_drop_locks(0, NULL, &shm_ids, 1);
-#endif
 		}
 
 		return rval;
@@ -266,14 +263,7 @@
 #ifdef IPC_SHM_RACE_FIX
 	nl = NSC_NODELIST_ALLOC();
 try_again:
-	if (dest == 2) {
-		WARN_ON(!shm_sem_owned());
-		svp = shm_svr_get(id);
-		if (svp)
-			ipc_lock_by_ptr(&svp->shm_perm);
-	} else
-		svp = shm_svr_lock(id);
-
+	svp = shm_svr_lock(id);
 	if (!svp) {
 		NSC_NODELIST_FREE(nl);
 		return -EIDRM;
@@ -285,6 +275,18 @@
 	cookie = CLUSTERNODE_INVAL;
 	while ((node = NSC_NODELIST_GET_NEXT(&cookie, nl))
 							!= CLUSTERNODE_INVAL) {
+		if (node == this_node)
+			continue;
+
+		ret = RIPC_SHM_GET_NATTCHS(node, &rval, id, &nattch, dest);
+		if (ret && ret != -EREMOTE) {
+			idelay(HZ);
+			goto try_again;
+		}
+		if (!rval && nattch)
+			cnt += nattch;
+	}
+	NSC_NODELIST_FREE(nl);
 #else /* IPC_SHM_RACE_FIX */
 try_again:
 	cnt = nm_svr_num;
@@ -294,13 +296,17 @@
 	cookie = CLUSTERNODE_INVAL;
 	while ((node = NSC_NODELIST_GET_NEXT(&cookie, svp->shm_nodelist))
 							!= CLUSTERNODE_INVAL) {
-#endif /* !IPC_SHM_RACE_FIX */
 		if (node == this_node)
 			continue;
 
 		ret = RIPC_SHM_GET_NATTCHS(node, &rval, id, &nattch, dest);
 
 		/* if error should get cleaned up by nodedown */
+		/* SSI_XXX: does not get cleaned up if we are
+		 * in svr_ripc_shm_cleanup() code path because nodedown
+		 * waits for us before removing node from shm_nodelist.
+		 *	-Roger
+		 */
 		if (ret) {
 			idelay(HZ);
 			goto try_again;
@@ -309,9 +315,7 @@
 		if (!rval && nattch)
 			cnt = cnt + nattch;
 	}
-#ifdef IPC_SHM_RACE_FIX
-	NSC_NODELIST_FREE(nl);
-#endif
+#endif /* !IPC_SHM_RACE_FIX */
 
 	return cnt;
 }
@@ -434,6 +438,7 @@
 /* Caller must assure shm_ids.sem is held.
  * Returns with shm_ids.sem released.
  */
+/* NB: shm_ids.sem held by ripc_shm_get_nattchs() during ssi_shm_cleanup() */
 int
 ripc_shm_rmid(
 	clusternode_t node,
@@ -445,14 +450,8 @@
 
 	shp = (struct shmid_kernel *)ipc_get_locks(id, &shm_ids, 0);
 	if (!shp) {
-#ifdef IPC_SHM_RACE_FIX
-		/* Drop the lock acquired during ripc_shm_get_nattchs() */
-		if (ssi_isremote())
-			ipc_drop_locks(0, NULL, &shm_ids, 1);
-#else
 		/* Drop the locks acquired by caller */
 		ipc_drop_locks(0, NULL, &shm_ids, 1);
-#endif
 		*rval = -EIDRM;
 		return 0;
 	}
@@ -461,10 +460,9 @@
 	BUG_ON(!size);
 	ssi_local_destroy(shp);
 #ifdef IPC_SHM_RACE_FIX
+	/* ssi_local_destroy() returns with shp unlocked */
 	ipc_set_lock_dest(&shm_ids, -1);
-	/* Drop the lock acquired during ripc_shm_get_nattchs() */
-	if (ssi_isremote())
-		ipc_drop_locks(0, NULL, &shm_ids, 1);
+	ipc_drop_locks(0, NULL, &shm_ids, 1);
 #else
 	ipc_drop_locks(id, (struct kern_ipc_perm *)shp, &shm_ids, 1);
 #endif
@@ -531,7 +529,7 @@
 
 #ifdef IPC_SHM_RACE_FIX
 	nl = NSC_NODELIST_ALLOC();
-
+retry:
 	svp = shm_svr_lock(id);
 	if (!svp) {
 		NSC_NODELIST_FREE(nl);
@@ -559,9 +557,14 @@
 
 		ret = RIPC_DROP_LOCKS(node, &rval, id);
 		(void) rval;
+#ifdef IPC_SHM_RACE_FIX
+		if (ret && ret != -EREMOTE) {
+			idelay(HZ);
+			goto retry;
+		}
+#endif
 	}
 	NSC_NODELIST_FREE(nl);
-	return;
 }
 
 int
@@ -610,7 +613,8 @@
 			 * unlocking unlocked lock in ripc_drop_locks.
 			 */
 			/* -EIDRM ? No. */
-			if (ret > 0)
+			//if (ret > 0)
+			if (ret != -EIDRM)
 #endif
 			/* drop remote locks because has attachs */
 			abort_rmid(id);
@@ -679,22 +683,19 @@
 			ipc_drop_locks(0, NULL, &shm_ids, 1);
 	}
 	else {
-		/* Drop the locks since will be going remote */
 		shp = shm_cli_get(id);
-#ifdef IPC_SHM_RACE_FIX
-		shm_cli_unlock(shp);
-		/* shm_ids.sem held */
-#else
+
+		/* Must drop locks since RIPC_SHM_CLEANUP() does
+		 * RIPC_SHM_GET_NATTCHS() callback which acquires shm_ids.sem
+		 */
 		ipc_drop_locks(id, (struct kern_ipc_perm *)shp, &shm_ids, 1);
-#endif
 
 		ret = RIPC_SHM_CLEANUP(svrnode, &rval, id, clinode);
 
-#ifndef IPC_SHM_RACE_FIX
 		/* Reacquire the locks */
 		ipc_get_locks(0, &shm_ids, 1);
-#endif
 
+#ifndef IPC_SHM_RACE_FIX
 #ifdef IPC_SHM_DESTROY_FIX
 		/* Ignore -EIDRM already removed at server */
 		if (ret == -EREMOTE) {
@@ -713,6 +714,23 @@
 		if (rval && rval != -EBUSY && rval != -ESRCH)
 			printk(KERN_WARNING "%s: Failed to cleanup IPC shm "
 				"structures %d\n", __FUNCTION__, rval);
+#else /* !IPC_SHM_RACE_FIX */
+		/* Ignore -EIDRM already removed at server */
+		/* Ignore -EREMOTE cleaned up by ipc_shm_nodedown() */
+		if (ret)
+			rval = ret;
+		switch (rval) {
+		case 0:
+		case -EBUSY:
+		case -ESRCH:
+			break;
+		default:
+			printk(KERN_WARNING "%s: Failed to cleanup IPC shm "
+				"structures %d\n", __FUNCTION__, rval);
+			break;
+		}
+		/* Return with shp unlocked */
+#endif /* IPC_SHM_RACE_FIX */
 	}
 	return 0;
 }


------------------------------------------------------------------------------
Download Intel® Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev