[SSI] openssi/kernel/ipc msg.c, 1.13, 1.14 sem.c, 1.29, 1.30 shm.c, 1.24, 1.25 util.c, 1.8, 1.9 util.h, 1.10, 1.11

Update of /cvsroot/ssic-linux/openssi/kernel/ipc
In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv16224/kernel/ipc

Modified Files:
      Tag: OPENSSI-FC
	msg.c sem.c shm.c util.c util.h 
Log Message:
Bug fixes and enhancements. See ChangeLog.


Index: util.h
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/ipc/util.h,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- util.h	26 May 2005 10:02:43 -0000	1.10
+++ util.h	27 Oct 2009 03:18:30 -0000	1.11
@@ -34,6 +34,11 @@
 	int max_id;
 	unsigned short seq;
 	unsigned short seq_max;
+#ifdef CONFIG_SSI
+#ifdef IPC_SHM_RACE_FIX
+	int lock_dest_id; /* id of entry locked for destroy aka. SHM_LOCK_DEST */
+#endif
+#endif
 	struct semaphore sem;	
 	struct ipc_id_ary nullentry;
 	struct ipc_id_ary* entries;

Index: sem.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/ipc/sem.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- sem.c	28 Mar 2009 23:44:16 -0000	1.29
+++ sem.c	27 Oct 2009 03:18:30 -0000	1.30
@@ -745,6 +745,7 @@
 						}
 					}
 				}
+				NSC_NODELIST_FREE(nl);
 				seminfo.semusz += sem_ids.in_use;
 				seminfo.semaem += used_sems;
 				max_id += sem_ids.max_id;
@@ -2121,7 +2122,7 @@
 #ifdef CONFIG_SSI
 	struct ssi_nodeid_pair *node_id_pairs = NULL;
 	int local_view = ssi_get_localview();
-	int id_count, size;
+	int id_count, size = 0;
 	char viewstr[10];
 #endif /* CONFIG_SSI */
 
@@ -2131,19 +2132,15 @@
 #ifdef CONFIG_SSI
 	bzero(viewstr, 10);
 	if (!local_view) {
-		size = 30; /* Random guess */
-		for (;;) {
-			int allocated = size;
+		(void) cli_ipcname_gettotal(NAME_SERVICE_SEM, &node_id_pairs, &size);
+		if (!size)
+			goto done;
 
-			node_id_pairs = kmalloc(size * sizeof(*node_id_pairs), GFP_KERNEL);
-			if (node_id_pairs == NULL)
-				goto done;
+		node_id_pairs = kmalloc(size * sizeof(*node_id_pairs), GFP_KERNEL);
+		if (node_id_pairs == NULL)
+			goto done;
 
-			cli_ipcname_gettotal(NAME_SERVICE_SEM, &node_id_pairs, &size);
-			if (size <= allocated)
-				break;
-			kfree (node_id_pairs);
-		}
+		(void) cli_ipcname_gettotal(NAME_SERVICE_SEM, &node_id_pairs, &size);
 	}
 	id_count = SEM_MAX_ID;
 
@@ -2157,19 +2154,19 @@
 			node_num = node_id_pairs[i].node_num;
 
 			if (node_num == CLUSTERNODE_INVAL) {
-				node_num = -1;
+				/* node_num = -1; */
+				continue;
 			} else if (node_num == this_node) {
 				sma = sem_lock(ipc_id);
-			} else {
-				sma = kzmalloc(sizeof(struct sem_array), GFP_KERNEL);
-				if (sma == NULL)
-					break;
-				(void) ssi_sem_get_sem_array(node_num, ipc_id, (char **) &sma);
+			} else if (!(sma = kmalloc(sizeof(*sma), GFP_KERNEL)) ||
+				   ssi_sem_get_sem_array(node_num, ipc_id,
+							 (char **) &sma) != 0) {
+				continue;
 			}
 		} else {
 			sma = sem_lock(i);
 			node_num = this_node;
-			ipc_id = 0; /* shut up compiler */
+			ipc_id = i; /* satisfy compiler */
 		}
 		if(sma) {
 			len += sprintf(buffer + len, "%10d %10d  %4o %10lu %5u %5u %5u %5u %10lu %10lu",
@@ -2188,14 +2185,10 @@
 			else
 				strcpy(viewstr, "default");
 			len += sprintf(buffer + len, " %10s %10d\n", viewstr, node_num);
-			if (local_view)
+			if (node_num == this_node)
 				sem_unlock(sma);
-			else {
-				if (node_num == this_node)
-					sem_unlock(sma);
-				else
-					kfree(sma);
-			}
+			else
+				kfree(sma);
 			pos += len;
 			if(pos < offset) {
 				len = 0;
@@ -2236,8 +2229,9 @@
 	*eof = 1;
 done:
 	up(&sem_ids.sem);
-#ifdef	CONFIG_SSI
-	kfree (node_id_pairs);
+#ifdef CONFIG_SSI
+	if (!local_view)
+		kfree(node_id_pairs);
 #endif
 	*start = buffer + (offset - begin);
 	len -= (offset - begin);

Index: shm.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/ipc/shm.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- shm.c	28 Mar 2009 23:44:16 -0000	1.24
+++ shm.c	27 Oct 2009 03:18:30 -0000	1.25
@@ -347,7 +347,7 @@
 
 	error = -ENOSPC;
 #ifdef CONFIG_SSI
-	set_mb(shp->id, newid);
+	shp->id = newid;
 #endif /* CONFIG_SSI */
 	id = shm_addid(shp);
 	if(id == -1) 
@@ -673,6 +673,7 @@
         				}
                 		}
 			}
+			NSC_NODELIST_FREE(nl);
 			memset(buf,0,sizeof(shm_info));
 			shm_info.used_ids += shm_ids.in_use;
 			shm_get_stat (&(((struct shm_info *)buf)->shm_rss), &(((struct shm_info *)buf)->shm_swp));
@@ -1383,18 +1384,138 @@
 ipc_shm_nodedown(clusternode_t node)
 {
 	int id;
-	struct ipc_ids *ids = &shm_ids;
 	struct shmid_kernel *shp;
+#ifdef IPC_SHM_LOCK_DEST_FIX
+	struct ipc_id_ary* entries;
+	int max_id, shm_id, shm_ids_sem_held, shm_node_found = 1;
+#else
+	struct ipc_ids *ids = &shm_ids;
+#endif
 
 	icssvr_nodedown_svc_wait(node, cluster_ipc_svc);
+
+#ifdef IPC_SHM_LOCK_DEST_FIX
+again:
+	shm_ids_sem_held = 1;
+	if (shm_node_found) {
+		/* Cannot call down() since we are not certain if a SHM segment
+		 * for DOWN node has shm_ids.sem for SHM_LOCK_DEST.
+		 */
+		while (down_trylock(&shm_ids.sem)) {
+			if (shm_ids.lock_dest_id != -1) {
+				/* SHM_LOCK_DEST for some segment not necessarily
+				 * the segment for DOWN node.
+				 */
+				shm_ids_sem_held = 0;
+				break;
+			}
+			nidelay(HZ/20);
+		}
+		/* If none of the SHM segments for DOWN node
+		 * has SHM_LOCK_DEST we can call down().
+		 */
+		shm_node_found = 0;
+	} else
+		down(&shm_ids.sem);
+
+ 	max_id = shm_ids_sem_held ? shm_ids.max_id : shm_ctlmni;
+ 	for (id = 0; id <= max_id; id++) {
+		if (!shm_ids_sem_held) {
+			/* avoid races; based on ipc_lock() */
+			rcu_read_lock();
+			entries = rcu_dereference(shm_ids.entries);
+			shp = (struct shmid_kernel *)entries->p[id];
+			if (!shp || shp->shm_perm.deleted) {
+				rcu_read_unlock();
+				continue;
+			}
+			if (shp->shm_node != node) {
+				rcu_read_unlock();
+				continue;
+			}
+			shm_id = shp->id;
+			rcu_read_unlock();
+
+			/* barrier for shm_ids.lock_dest_id */
+			shm_ids_sem_held = !down_trylock(&shm_ids.sem);
+			if (shm_ids_sem_held) {
+				if (id != 0)
+					id = -1; /* restart for loop */
+				max_id = shm_ids.max_id;
+			}
+
+			shm_node_found = 1;
+
+			if (shm_id != shm_ids.lock_dest_id)
+				continue;
+			/* Now we can obtain shm_ids.sem without possibility of
+			 * deadlock since this segment has shm_ids.lock_dest_id
+			 * (SHM_LOCK_DEST). Even if we raced we don't expect
+			 * another thread for DOWN node to acquire SHM_LOCK_DEST.
+			 */
+		} else {
+			/* shm_ids.sem held */
+			shp = (struct shmid_kernel *)shm_ids.entries->p[id];
+			if (!shp)
+				continue;
+			if (shp->shm_node == this_node)
+				shm_svr_unregister(shp->id, node);
+			if (shp->shm_node != node)
+				continue;
+			shm_id = shp->id;
+		}
+
+		shp = (struct shmid_kernel *)ipc_get_locks(shm_id, &shm_ids, 0);
+
+		if (!shm_ids_sem_held) {
+			shm_ids_sem_held = 1;
+			if (!shp || shp->id != shm_ids.lock_dest_id) {
+				/* Lost race with ripc_shm_rmid(), etc. */
+				if (shp) {
+					ipc_drop_locks(shp->id,
+							(struct kern_ipc_perm *)shp,
+							&shm_ids, 0);
+					shp = (struct shmid_kernel *)
+						ipc_get_locks(shm_id, &shm_ids, 1);
+				} else
+					down(&shm_ids.sem);
+				max_id = shm_ids.max_id;
+			} else {
+				/* shm_ids.sem held for SHM_LOCK_DEST */
+				max_id = shm_ids.max_id; /* avoided barrier */
+			}
+			if (id != 0)
+				id = -1; /* restart for loop */
+		}
+		/* shm_ids.sem held */
+		if (!shp)
+			continue;
+
+		shp->shm_flags |= SHM_DEST;
+		if (shp->shm_nattch == 0) {
+			SSI_ASSERT(shm_ids_sem_held);
+
+			ssi_local_destroy(shp);
+			cli_ipcname_rmid(NAME_SERVICE_SHM, shm_id);
+			/* ssi_local_destroy() released shp.shm_perm.lock */
+			/* shm_ids.sem still held */
+		} else
+			ipc_drop_locks(shp->id, (struct kern_ipc_perm *)shp,
+					&shm_ids, 0);
+	}
+	if (!shm_ids_sem_held)
+		goto again;
+
+	up(&shm_ids.sem);
+#else /* !IPC_SHM_LOCK_DEST_FIX */
  	for (id = 0; id <= ids->max_id; id++) {
 		if(ids->entries->p[id] == NULL)
 			continue;
-		
 		shp = (struct shmid_kernel *)ids->entries->p[id];
 		if (shp->shm_node == this_node)
 			shm_svr_unregister(shp->id, node);
 		if (shp->shm_node == node) {
+			/* SSI_XXX: testing shm_flags without required locks */
 			if (shp->shm_flags & SHM_LOCK_DEST) {
 				(void) ipc_get_locks(shp->id, &shm_ids, 0);
 				if (shp->shm_nattch == 0) {
@@ -1425,6 +1546,7 @@
 			}
 		}
 	}
+#endif /* !IPC_SHM_LOCK_DEST_FIX */
 	if (cfs_shm_node_mnts[node]) {
 		mntput(cfs_shm_node_mnts[node]);
 		cfs_shm_node_mnts[node] = NULL;
@@ -1458,6 +1580,9 @@
 	return 0;
 }
 
+/* It has to be called with shp and shm_ids.sem locked,
+ * but returns with shp unlocked and freed.
+ */
 void
 ssi_local_destroy(struct shmid_kernel *shp)
 {
@@ -1591,7 +1716,7 @@
 #ifdef CONFIG_SSI
 	struct ssi_nodeid_pair *node_id_pairs = NULL;
 	int local_view = ssi_get_localview();
-	int id_count, size;
+	int id_count, size = 0;
 	char viewstr[10];
 #endif /* CONFIG_SSI */
 
@@ -1601,19 +1726,15 @@
 #ifdef CONFIG_SSI
 	bzero(viewstr, 10);
 	if (!local_view) {
-		size = 30; /* Random guess */
-		for (;;) {	/* Maybe limit tries? */
-			int allocated = size;
+		(void) cli_ipcname_gettotal(NAME_SERVICE_SHM, &node_id_pairs, &size);
+		if (!size)
+			goto done;
 
-			node_id_pairs = kmalloc(size * sizeof *node_id_pairs, GFP_KERNEL);
-			if (!node_id_pairs)
-				goto done;
+		node_id_pairs = kmalloc(size * sizeof *node_id_pairs, GFP_KERNEL);
+		if (!node_id_pairs)
+			goto done;
 
-			cli_ipcname_gettotal(NAME_SERVICE_SHM, &node_id_pairs, &size);
-			if (size <= allocated)
-				break;
-			kfree (node_id_pairs);
-		}
+		(void) cli_ipcname_gettotal(NAME_SERVICE_SHM, &node_id_pairs, &size);
 	}
 	id_count = SHM_MAX_ID;
 
@@ -1623,30 +1744,29 @@
 #endif /* CONFIG_SSI */
 		struct shmid_kernel* shp;
 #ifdef CONFIG_SSI
-		key_t ipc_id = i;
+		key_t ipc_id;
 		int node_num, segsize=0, cprid=0;
-		shp = NULL;
+
 		if (!local_view) {
 			ipc_id = node_id_pairs[i].ipc_id;
 			node_num = node_id_pairs[i].node_num;
 
 			if (node_num == CLUSTERNODE_INVAL) {
-                                node_num = -1;
+                                /* node_num = -1; */
+				continue;
                         } else if (node_num == this_node) {
 				shp = shm_lock(ipc_id);
 				segsize = shm_get_segsize(shp);
 				cprid = shm_get_cpid(shp);
-			} else {
-				shp = kzmalloc(sizeof(struct shmid_kernel),
-								GFP_KERNEL);
-				if (shp == NULL)
-					break;
-				ssi_shm_get_shmid_kernel(node_num, ipc_id,
-					(char **)&shp, &segsize, &cprid);
+			} else if (!(shp = kmalloc(sizeof(*shp), GFP_KERNEL)) ||
+				   ssi_shm_get_shmid_kernel(node_num, ipc_id,
+					(char **)&shp, &segsize, &cprid) != 0) {
+				continue;
 			}
 		} else {
 			shp = shm_lock(i);
 			node_num = this_node;
+			ipc_id = i; /* satisfy compiler */
 		}
 #else
 		shp = shm_lock(i);
@@ -1705,15 +1825,10 @@
 			else
 				strcpy(viewstr, "default");
 			len += sprintf(buffer + len, " %10s %10d\n", viewstr, node_num);
-			if (local_view)
+			if (node_num == this_node)
 				shm_unlock(shp);
-			else {
-				if (node_num == this_node)
-					shm_unlock(shp);
-				else {
-					kfree(shp);
-				}
-			}
+			else
+				kfree(shp);
 #else
 			shm_unlock(shp);
 #endif /* CONFIG_SSI */
@@ -1730,7 +1845,8 @@
 done:
 	up(&shm_ids.sem);
 #ifdef CONFIG_SSI
-	kfree(node_id_pairs);
+	if (!local_view)
+		kfree(node_id_pairs);
 #endif /* CONFIG_SSI */
 	*start = buffer + (offset - begin);
 	len -= (offset - begin);

Index: util.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/ipc/util.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- util.c	26 May 2005 10:02:42 -0000	1.8
+++ util.c	27 Oct 2009 03:18:30 -0000	1.9
@@ -75,6 +75,11 @@
 		 else
 		 	ids->seq_max = seq_limit;
 	}
+#ifdef CONFIG_SSI
+#ifdef IPC_SHM_RACE_FIX
+	ids->lock_dest_id = -1;
+#endif
+#endif
 
 	ids->entries = ipc_rcu_alloc(sizeof(struct kern_ipc_perm *)*size +
 				     sizeof(struct ipc_id_ary));
@@ -229,7 +234,16 @@
 	if (table)
 		up(&ids->sem);
 }
+
+#ifdef IPC_SHM_LOCK_DEST_FIX
+/* Called with ids.sem held */
+void
+ipc_lock_dest_id(struct ipc_ids *ids, int id)
+{
+	ids->lock_dest_id = id; /* SHM_LOCK_DEST */
+}
 #endif
+#endif /* CONFIG_SSI */
 
 /**
  *	ipc_addid 	-	add an IPC identifier

Index: msg.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/ipc/msg.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- msg.c	28 Mar 2009 23:44:16 -0000	1.13
+++ msg.c	27 Oct 2009 03:18:30 -0000	1.14
@@ -626,6 +626,7 @@
 						}
 					}
 				}
+				NSC_NODELIST_FREE(nl);
 				msginfo.msgpool += msg_ids.in_use;
 				msginfo.msgmap += atomic_read(&msg_hdrs);
 				msginfo.msgtql += atomic_read(&msg_bytes);
@@ -1472,7 +1473,7 @@
 #ifdef CONFIG_SSI
 	struct ssi_nodeid_pair *node_id_pairs = NULL;
 	int local_view = ssi_get_localview();
-	int id_count, size;
+	int id_count, size = 0;
 	char viewstr[10];
 #endif
 	down(&msg_ids.sem);
@@ -1481,19 +1482,15 @@
 
 	bzero(viewstr, 10);
 	if (!local_view) {
-		size = 30; /* Random guess */
-		for (;;) {
-			int allocated = size;
+		(void) cli_ipcname_gettotal(NAME_SERVICE_MSG, &node_id_pairs, &size);
+		if (!size)
+			goto done;
 
-			node_id_pairs = kmalloc(size * sizeof(*node_id_pairs), GFP_KERNEL);
-			if (node_id_pairs == NULL)
-				goto done;
+		node_id_pairs = kmalloc(size * sizeof(*node_id_pairs), GFP_KERNEL);
+		if (node_id_pairs == NULL)
+			goto done;
 
-			(void) cli_ipcname_gettotal(NAME_SERVICE_MSG, &node_id_pairs, &size);
-			if (allocated >= size)
-				break;
-			kfree (node_id_pairs);
-		}
+		(void) cli_ipcname_gettotal(NAME_SERVICE_MSG, &node_id_pairs, &size);
 	}
 	id_count = MSG_MAX_ID;
 
@@ -1504,26 +1501,26 @@
 		struct msg_queue * msq;
 #ifdef CONFIG_SSI
 		int node_num;
-		key_t ipc_id = i;
-		msq = NULL;
+		key_t ipc_id;
 
 		if (!local_view) {
 			ipc_id = node_id_pairs[i].ipc_id;
 			node_num = node_id_pairs[i].node_num;
 
 			if (node_num == CLUSTERNODE_INVAL) {
-				node_num = -1;
+				/* node_num = -1; */
+				continue;
 			} else if (node_num == this_node) {
 				msq = msg_lock(ipc_id);
-			} else {
-				msq = kzmalloc(sizeof(struct msg_queue), GFP_KERNEL);
-				if (msq == NULL)
-					break;
-				ssi_msg_get_msg_queue(node_num, ipc_id, (char **)&msq);
+			} else if (!(msq = kmalloc(sizeof(*msq), GFP_KERNEL)) ||
+				   ssi_msg_get_msg_queue(node_num, ipc_id,
+							 (char **)&msq) != 0) {
+				continue;
 			}
 		} else {
 			node_num = this_node;
 			msq = msg_lock(i);
+			ipc_id = i; /* satisfy compiler */
 		}
 #else
 		msq = msg_lock(i);
@@ -1554,14 +1551,10 @@
 			else
 				strcpy(viewstr, "default");
 			len += sprintf(buffer + len, " %10s %10d\n", viewstr, node_num);
-			if (local_view)
+			if (node_num == this_node)
 				msg_unlock(msq);
-			else {
-				if (node_num == this_node)
-					msg_unlock(msq);
-				else
-					kfree(msq);
-			}
+			else
+				kfree(msq);
 #else
 			msg_unlock(msq);
 #endif
@@ -1578,7 +1571,8 @@
 done:
 	up(&msg_ids.sem);
 #ifdef	CONFIG_SSI
-	kfree (node_id_pairs);
+	if (!local_view)
+		kfree(node_id_pairs);
 #endif
 	*start = buffer + (offset - begin);
 	len -= (offset - begin);


------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay 
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference