[SSI] openssi/kernel/cluster/ssi/vproc procfs_subr.c,1.19,1.20

Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/vproc
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv6902/cluster/ssi/vproc

Modified Files:
      Tag: OPENSSI-FC
	procfs_subr.c 
Log Message:
VPROC (#ifdef PROC_LARGE_MAXPIDS):
- Refactor ssi_get_tgid_list() to do concurrent async requests and buffer the PIDs list for each node in the cluster.
- Revert PROC_MAXPIDS value to 20 (the default in Linux base). filldir() in proc_pid_readdir() stops at around 20 entries.
- Reduce VPROC_MAXPIDS value so that internal pid_list[] cache and related proc_readdir_cookie structure fit within one memory page. Reduce memory pressure. Remove large kmalloc() call in proc_pid_readdir().
- Fix ssi_get_tgid_list() skips remaining nodes in the cluster if "start_node" in struct proc_root_readdir_cookie is not found in CLMS node list. This can happen when the next node ("start_node") goes down while the system is reading the list of PIDs. Affects userspace accessing procfs and is not in localview mode.

 cluster/ssi/vproc/procfs_subr.c |  285 ++++++++++++++++++++++++++++----
 fs/proc/base.c                  |   31 +--
 2 files changed, 267 insertions(+), 49 deletions(-)


Index: procfs_subr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/vproc/procfs_subr.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- procfs_subr.c	25 Oct 2010 05:08:56 -0000	1.19
+++ procfs_subr.c	25 Oct 2010 05:24:43 -0000	1.20
@@ -4,6 +4,7 @@
 #include <linux/fs.h>
 #include <linux/tty.h>
 
+#include <cluster/async.h>
 #include <cluster/clms.h>
 #include <cluster/config.h>
 #include <cluster/nodelist.h>
@@ -12,19 +13,39 @@
 #include <cluster/gen/reopen.h>
 
 #ifdef PROC_LARGE_MAXPIDS
-#define PROC_MAXPIDS VPROC_MAXPIDS
-#else
-#define PROC_MAXPIDS 20
-#endif
+struct proc_readdir_cookie {
+	struct list_head d_list;
+	int pids_done;
+	int pids_size; /* size of pid_list[] */
+	int pids_read; /* # of pids read into pids[] */
+	clusternode_t node;
+	int h_index; /* vproc hash */
+	int v_index; /* vproc list */
+	unsigned int *pid_list; /* pid_list[] */
+};
+#define MAX_PROC_READDIR_COOKIES 2 /* default: 2 */
+
+#define VPROC_MAXPIDS \
+	((PAGE_SIZE-sizeof(struct proc_readdir_cookie)) / sizeof(unsigned int))
 
 struct proc_root_readdir_cookie {
 	unsigned int start_index;
 	unsigned int cur_index;
 	clusternode_t start_node;
+	struct proc_readdir_cookie *cur_cookie;
+	struct list_head cp_list; /* proc_readdir_cookie's */
+	unsigned int pid_list[PROC_MAXPIDS]; /* cache for short read */
+};
+#else /* PROC_LARGE_MAXPIDS */
+struct proc_root_readdir_cookie {
+	unsigned int start_index;
+	unsigned int cur_index;
+	clusternode_t start_node;
 	int h_index;
 	int v_index;
 	unsigned int pid_list[PROC_MAXPIDS];
 };
+#endif /* !PROC_LARGE_MAXPIDS */
 
 int ssi_proc_root_open(struct inode *inode, struct file *file)
 {
@@ -37,17 +58,17 @@
 		return 0;
 #ifdef PROC_LARGE_MAXPIDS
 	cp = kmalloc(sizeof(*cp), GFP_KERNEL);
-#else
-	cp = kzmalloc(sizeof(*cp), GFP_KERNEL);
-#endif
 	if (cp == NULL)
 		return -ENOMEM;
-#ifdef PROC_LARGE_MAXPIDS
 	cp->start_index = 0;
 	cp->cur_index = 0;
 	cp->start_node = 0;
-	cp->h_index = 0;
-	cp->v_index = 0;
+	cp->cur_cookie = NULL;
+	INIT_LIST_HEAD(&cp->cp_list);
+#else
+	cp = kzmalloc(sizeof(*cp), GFP_KERNEL);
+	if (cp == NULL)
+		return -ENOMEM;
 #endif
 	file->private_data = cp;
 	return 0;
@@ -55,14 +76,53 @@
 
 int ssi_proc_root_release(struct inode *inode, struct file *file)
 {
+#ifdef PROC_LARGE_MAXPIDS
+	struct proc_root_readdir_cookie *cp;
+	struct proc_readdir_cookie *data, *tmp;
+
+	(void)inode;
+	if (PVP(current->p_vproc)->pvp_localview)
+		return 0;
+
+	cp = file->private_data;
+retry:
+	list_for_each_entry_safe(data, tmp, &cp->cp_list, d_list) {
+		if (!data->pids_done)
+			continue;
+		list_del(&data->d_list);
+		free_page((unsigned long)data);
+	}
+	if (unlikely(!list_empty(&cp->cp_list))) {
+		idelay((HZ+99)/100);
+		goto retry;
+	}
+#else
 	(void)inode;
 	if (PVP(current->p_vproc)->pvp_localview)
 		return 0;
+#endif /* !PROC_LARGE_MAXPIDS */
 	kfree(file->private_data);
 	file->private_data = NULL;
 	return 0;
 }
 
+#ifdef PROC_LARGE_MAXPIDS
+void
+ssi_get_tgid_list_async_handler(void *data)
+{
+	struct proc_readdir_cookie *cp = (struct proc_readdir_cookie *) data;
+	u_int *pidsp = cp->pid_list;
+
+	if (PVPSOP_GET_TGID_LIST(cp->node, &pidsp,
+			&cp->pids_size, &cp->h_index, &cp->v_index)) {
+		cp->pids_size = 0;
+	}
+	cp->pids_done = 1;
+	smp_mb();
+}
+#endif
+
+/* Caller holds inode->i_sem */
 int
 ssi_get_tgid_list(struct file *file, unsigned int index, unsigned int *pids)
 {
@@ -70,25 +130,193 @@
 	nsc_nodelist_t *nodelist;
 	nsc_nlcookie_t cookie;
 	clusternode_t node;
-	u_int *pidsp;
-	int pids_size, nr_pids = 0;
-	unsigned int out_index;
-	int pids_off;
+#ifdef PROC_LARGE_MAXPIDS
+	struct proc_readdir_cookie *data;
+	struct list_head *pos;
+	int count, nr_pids = 0;
 
+	/* Use the saved starting position. */
+	if (cp->start_index && index >= cp->start_index) {
+		/* short read, fill pids[] from cache */
+		if (index < cp->cur_index) {
+			nr_pids = cp->cur_index - index;
+			memcpy(pids, cp->pid_list + index - cp->start_index,
+			       nr_pids * sizeof(unsigned int));
+			index = cp->cur_index - nr_pids;
+			if (unlikely(nr_pids == PROC_MAXPIDS)) {
+				cp->start_index = index;
+				return nr_pids;
+			}
+		}
+		if (cp->start_node == CLUSTERNODE_INVAL) {
+			node = cp->start_node;
+			goto wait_for_pids;
+		}
+	} else
+ 		BUG_ON(index < cp->start_index);
+
+	/* Limit number of queued requests */
+	count = 0;
+	list_for_each(pos, &cp->cp_list) {
+		if (++count == MAX_PROC_READDIR_COOKIES) {
+			node = cp->start_node;
+			goto wait_for_pids;
+		}
+	}
+
+queue_requests:
+	/* get node list */
+	nodelist = clms_get_nsc_nodelist(CLMS_NODE_UP
+					 |CLMS_NODE_HALF_UP
+					 |CLMS_NODE_HALF_DOWN);
+	NSC_NLCOOKIE_INIT(&cookie);
+	if (likely(NSC_NODELIST_TEST1(nodelist, this_node))) {
+		do {
+			node = NSC_NODELIST_GET_NEXT(&cookie, nodelist);
+			if (node == CLUSTERNODE_INVAL) {
+				NSC_NODELIST_FREE(nodelist);
+				goto wait_for_pids;
+			}
+		} while (node < cp->start_node);
+	} else {
+		/* If this node not in the list, then it must be coming
+		 * up or going down, so clear the list and only process
+		 * this node.
+		 */
+		NSC_NODELIST_CLRALL(nodelist);
+		node = this_node;
+	}
+
+	for (;;) {
+		data = (void *) __get_free_page(GFP_KERNEL);
+		if (!data)
+			break;
+
+		INIT_LIST_HEAD(&data->d_list);
+		data->pids_done = 0;
+		data->pids_size = VPROC_MAXPIDS;
+		data->pids_read = 0;
+		data->node = node;
+		data->h_index = 0;
+		data->v_index = 0;
+		data->pid_list = (void *)((char *)data + sizeof(*data));
+
+		list_add_tail(&data->d_list, &cp->cp_list);
+
+		if (nsc_async_queue(nsc_generic_async_queue,
+				ssi_get_tgid_list_async_handler,
+				(void *)data, sizeof(*data), 0)) {
+			list_del(&data->d_list);
+			free_page((unsigned long)data);
+			break;
+		}
+
+		node = NSC_NODELIST_GET_NEXT(&cookie, nodelist);
+		if (node == CLUSTERNODE_INVAL)
+			break;
+		if (++count == MAX_PROC_READDIR_COOKIES)
+			break;
+	}
+	NSC_NODELIST_FREE(nodelist);
+
+wait_for_pids:
+	do {
+		struct proc_readdir_cookie *tmp;
+
+		count = 0;
+		if (cp->cur_cookie && cp->cur_cookie->pids_done) {
+			data = cp->cur_cookie;
+			cp->cur_cookie = NULL;
+			goto resume;
+		}
+		list_for_each_entry(data, &cp->cp_list, d_list) {
+			int __nr_pids, avail_pids;
+resume:
+			count++;
+			if (!data->pids_done || !data->pids_size)
+				continue;
+			cp->cur_cookie = data;
+
+			/* Populate pids[] */
+			__nr_pids = PROC_MAXPIDS - nr_pids;
+			avail_pids = data->pids_size - data->pids_read;
+
+			if (avail_pids < __nr_pids)
+				__nr_pids = avail_pids;
+			memcpy(pids + nr_pids,
+				data->pid_list + data->pids_read,
+				__nr_pids * sizeof(unsigned int));
+
+			nr_pids += __nr_pids;
+			data->pids_read += __nr_pids;
+
+			if (data->pids_size != data->pids_read) {
+				SSI_ASSERT(nr_pids == PROC_MAXPIDS);
+				break;
+			}
+
+			/* Reached end of vproc list */
+			if (!data->v_index) {
+				data->pids_size = 0;
+				cp->cur_cookie = NULL;
+				goto next;
+			}
+
+			/* More vproc pids to read */
+			data->pids_done = 0;
+			data->pids_size = VPROC_MAXPIDS;
+			data->pids_read = 0;
+			if (nsc_async_queue(nsc_generic_async_queue,
+					ssi_get_tgid_list_async_handler,
+					(void *)data, sizeof(*data), 0)) {
+				data->pids_size = 0;
+				break;
+			}
+next:
+			if (nr_pids == PROC_MAXPIDS)
+				break;
+		}
+		if (!count)
+			break;
+
+		list_for_each_entry_safe(data, tmp, &cp->cp_list, d_list) {
+			if (!data->pids_done || data->pids_size)
+				continue;
+			list_del(&data->d_list);
+			free_page((unsigned long)data);
+		}
+
+		if (nr_pids == 0)
+			idelay((HZ+99)/100);
+	} while (nr_pids == 0);
+
+	/* Try harder to fill pids[] */
+	if (nr_pids != PROC_MAXPIDS && node != CLUSTERNODE_INVAL &&
+	    count < MAX_PROC_READDIR_COOKIES)
+		goto queue_requests;
+
+	cp->start_index = index;
+	cp->cur_index = index + nr_pids;
+	cp->start_node = node;
+	if (count)
+		memcpy(cp->pid_list, pids, nr_pids * sizeof(unsigned int));
+
+	return nr_pids;
+#else /* PROC_LARGE_MAXPIDS */
+	unsigned int out_index, start_index;
+	int pids_off, nr_pids = 0;
+
+	start_index = index;
 	if (cp->start_index && index >= cp->start_index) {
 		/* Use the saved starting position. */
 		out_index = cp->cur_index;
-		/* If caller truncated read, fill-in pids from previous read */
-		if (unlikely(index < cp->cur_index)) {
+		if (index < cp->cur_index) {
 			nr_pids = cp->cur_index - index;
-			BUG_ON(nr_pids > PROC_MAXPIDS);
 			memcpy(pids, cp->pid_list + index - cp->start_index,
 			       nr_pids * sizeof(unsigned int));
 			index = cp->cur_index;
 		}
-		BUG_ON(index > cp->cur_index);
 	} else {
-		/* Reset the saved starting position. */
 		cp->start_node = 0;
 		cp->h_index = 0;
 		cp->v_index = 0;
@@ -102,12 +330,10 @@
 	NSC_NLCOOKIE_INIT(&cookie);
 	if (NSC_NODELIST_TEST1(nodelist, this_node)) {
 		node = NSC_NODELIST_GET_NEXT(&cookie,nodelist);
-		if (cp->start_node) {
-			while (node != cp->start_node) {
-				node = NSC_NODELIST_GET_NEXT(&cookie,nodelist);
-				if (node == CLUSTERNODE_INVAL)
-					goto out;
-			}
+		while (cp->start_node && node != cp->start_node) {
+			node = NSC_NODELIST_GET_NEXT(&cookie,nodelist);
+			if (node == CLUSTERNODE_INVAL)
+				goto out;
 		}
 	}
 	else {
@@ -119,7 +345,11 @@
 		NSC_NODELIST_CLRALL(nodelist);
 	}
 
+
 	for (;;) {
+		u_int *pidsp;
+		int pids_size;
+
 		pids_size = PROC_MAXPIDS - nr_pids;
 		pidsp = pids + nr_pids;
 		if (PVPSOP_GET_TGID_LIST(node, &pidsp,
@@ -154,12 +384,13 @@
 	}
 out:
 	NSC_NODELIST_FREE(nodelist);
-	cp->start_index = index;
-	cp->cur_index = index + nr_pids;
+	cp->start_index = start_index;
+	cp->cur_index = start_index + nr_pids;
 	cp->start_node = node;
 	memcpy(cp->pid_list, pids, nr_pids * sizeof(unsigned int));
 
 	return nr_pids;
+#endif /* !PROC_LARGE_MAXPIDS */
 }
 
 


------------------------------------------------------------------------------
Nokia and AT&T present the 2010 Calling All Innovators-North America contest
Create new apps & games for the Nokia N8 for consumers in  U.S. and Canada
$10 million total in prizes - $4M cash, 500 devices, nearly $6M in marketing
Develop with Nokia Qt SDK, Web Runtime, or Java and Publish to Ovi Store 
http://p.sf.net/sfu/nokia-dev2dev