[SSI] openssi/kernel/cluster/ssi/cfs inode.c, 1.65, 1.66 write.c, 1.44, 1.45
Roger Tsang <[email protected]> Fri, 02 Apr 2010 20:53:22 +0000
| Newsgroups | gmane.linux.cluster.ssic.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/cfs
In directory sfp-cvsdas-3.v30.ch3.sourceforge.com:/tmp/cvs-serv32470/kernel/cluster/ssi/cfs
Modified Files:
Tag: OPENSSI-FC
inode.c write.c
Log Message:
CFS:
- Fix OOPs at tok_destroy() when get_new_inode() lost the race and calls destroy_inode() on partially initialized CFS inode. cfs_alloc_inode() did not initialize cnode HOLD lists.
- Optimize away large memset operation in CFS init_once(). Much of cnode structure cannot be initialized by zero'ing anyway.
CFS (#ifdef CFS_CHARD_SYNC_FIX):
- Fix "desynchronized value of cfs_i.ndirty" messages after failover. cfs_commit_result() to not mark requests dirty on ECFS_TRYAGAIN because these requests will be re-flushed directly without going through cfs_scan_dirty() again. Bug introduced in OPENSSI-FC-1-9-6-PRE13 and affects failover.
- Rename cfs_prepare_reflush() to cfs_reflush_inode() and do the re-flush directly instead of having the caller do cfs_flush_inode(). Do the flush directly to avoid having to clear PG_BUSY flag which may cause existing threads in cfs_wait_on_requests() to return too soon. Bug introduced in OPENSSI-FC-1-9-6-PRE13 and affects failover.
CFS (#ifdef CFS_EXPORT_OPS):
- cfs_init_locked() no longer calls cfstok_init(); done in CFS init_once() as part of the tok_destroy() OOPs fix.
cluster/ssi/cfs/inode.c | 49 +++++++++++++++----
cluster/ssi/cfs/write.c | 99 +++++++++++++++++++++-------------------
2 files changed, 93 insertions(+), 55 deletions(-)
Index: inode.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/cfs/inode.c,v
retrieving revision 1.65
retrieving revision 1.66
diff -u -d -r1.65 -r1.66
--- inode.c 5 Mar 2010 06:15:04 -0000 1.65
+++ inode.c 2 Apr 2010 20:53:20 -0000 1.66
@@ -655,7 +655,9 @@
{
struct cfs_find_desc *desc = (struct cfs_find_desc *)opaque;
+#ifndef CFS_EXPORT_OPS
cfstok_init(inode);
+#endif
/* SSI_XXX: Load from super block or cna_blocksize? */
inode->i_blksize = inode->i_sb->s_blocksize;
@@ -1715,9 +1717,15 @@
if ((flags & (SLAB_CTOR_VERIFY|SLAB_CTOR_CONSTRUCTOR)) ==
SLAB_CTOR_CONSTRUCTOR) {
- memset(cp, 0, sizeof(struct cnode));
-
- inode_init_once(&cp->vfs_inode);
+ /* memset(cp, 0, sizeof(struct cnode)); */
+ memset(&cp->c_fh, 0, sizeof(cp->c_fh));
+ cp->c_flags = 0;
+ cp->c_hp = NULL;
+#if defined(CFS_ZAP_CACHES) && defined(CFS_FILE_RW_NOLOCK)
+ cp->read_cache_jiffies = 0;
+ cp->attrtimeo = 0;
+ cp->attrtimeo_timestamp = 0;
+#endif
spin_lock_init(&cp->req_lock);
INIT_LIST_HEAD(&cp->dirty);
INIT_LIST_HEAD(&cp->commit);
@@ -1727,20 +1735,37 @@
INIT_LIST_HEAD(&cp->open_files);
cp->nread = 0;
#endif
- /* cp->ndirty = 0; */
- /* cp->ncommit = 0; */
- /* cp->npages = 0; */
-#ifdef CFS_ZAP_CACHES
- init_waitqueue_head(&cp->c_i_wait);
-#endif
- /* cp->c_hp = NULL; */
+ cp->ndirty = 0;
+ cp->ncommit = 0;
+ cp->npages = 0;
#ifdef SSI_CFS_SKIP
INIT_CONDITION(&(cp->c_cxlock));
#endif
+#ifdef SSI_CFS_FULLTIMES
+ init_XXX(&cp->c_attr);
+#endif
+ cfstok_init(&cp->vfs_inode);
+ cp->c_locknumber = NULL;
+#ifdef SSI_RANGE_TOKENS
+ init_XXX(c_rtokens[CFSTOK_NRANGE]);
+#endif
#ifdef CFSTOKHOLD_LINUX_LIST
for(idx = 0; idx < CFSTOK_NTOKS; idx++)
INIT_LIST_HEAD(&cp->c_hlist[idx]);
#endif
+ cp->c_vn = 0;
+#ifdef SSI_CFS_POLICY
+ cp->c_toktime = 0;
+ cp->c_wmapcnt = 0;
+ cp->c_rmapcnt = 0;
+ cp->c_wopncnt = 0;
+ cp->c_ropncnt = 0;
+#endif
+#ifdef SSI_CFS_ACLS
+ cp->c_aclcnt = 0;
+ cp->c_daclcnt = 0;
+ cp->c_aclbufp = NULL;
+#endif
INIT_COND_LOCK(&(cp->c_statelock));
INIT_LOCK(&(cp->c_tcblock));
for (idx = 0 ; idx < CFS_MAX_LISTS ; idx++)
@@ -1750,6 +1775,10 @@
INIT_LIST_HEAD(&cp->c_dirty_down);
INIT_LIST_HEAD(&cp->c_read_down);
#endif
+#ifdef CFS_ZAP_CACHES
+ init_waitqueue_head(&cp->c_i_wait);
+#endif
+ inode_init_once(&cp->vfs_inode);
}
}
Index: write.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/cfs/write.c,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- write.c 23 Mar 2010 01:39:14 -0000 1.44
+++ write.c 2 Apr 2010 20:53:20 -0000 1.45
@@ -121,8 +121,7 @@
static int cfs_wait_on_write_congestion(struct address_space *, int);
static int cfs_wait_on_requests(struct inode *, unsigned long, unsigned int);
#ifdef CFS_CHARD_SYNC_FIX
-static int cfs_prepare_reflush(struct inode *, unsigned long, unsigned int,
- struct writeback_control *);
+static int cfs_reflush_inode(struct inode *, unsigned long, unsigned int, int);
#endif
static kmem_cache_t *cfs_wdata_cachep;
@@ -517,7 +516,6 @@
}
#ifdef CFS_CHARD_SYNC_FIX
-flush_again:
node = itocmi(inode)->mi_server;
#endif
err = cfs_flush_inode(inode, index, npages, wb_priority(wbc));
@@ -527,6 +525,9 @@
if (err < 0)
goto out;
wbc->nr_to_write -= err;
+#ifdef CFS_CHARD_SYNC_FIX
+wait_on_requests:
+#endif
if (!wbc->nonblocking && wbc->sync_mode == WB_SYNC_ALL) {
#ifdef CFS_COMMIT_FILE_RANGE
err = cfs_wait_on_requests(inode, index, npages);
@@ -539,10 +540,14 @@
#ifndef KERNEL2613
#ifdef CFS_COMMIT_FILE_RANGE
#ifdef CFS_CHARD_SYNC_FIX
- /* If server failed over, flush unstable data to new server. */
- if (node != itocmi(inode)->mi_server &&
- cfs_prepare_reflush(inode, index, npages, wbc))
- goto flush_again;
+ if (err > 0 && node != itocmi(inode)->mi_server) {
+ node = itocmi(inode)->mi_server;
+ /* Server failed over already so we need to
+ * re-flush unstable data to new server.
+ */
+ err = cfs_reflush_inode(inode, index, npages, wb_priority(wbc));
+ goto wait_on_requests;
+ }
#endif
err = cfs_commit_inode(inode, index, npages, wb_priority(wbc));
#else
@@ -1754,8 +1759,6 @@
req = cfs_list_entry(data->pages.next);
cfs_list_remove_request(req);
#ifdef CFS_CHARD_SYNC_FIX
- cfs_mark_request_dirty(req);
- cfs_list_remove_request(req);
cfs_list_add_request(req, &itoc(data->inode)->c_dirty_down);
res++;
#else
@@ -2110,6 +2113,28 @@
return -ENOMEM;
}
+static int
+__cfs_flush_list(
+ struct inode *inode,
+ struct list_head *head,
+ unsigned int res,
+ int how)
+{
+#if !defined(KERNEL2613) && !defined(KERNEL2616_53)
+ return cfs_flush_list(head, CFS_SERVER(inode)->wpages, how);
+#else
+ struct cfs_server *server = CFS_SERVER(inode);
+ cnode_t *cp = itoc(inode);
+
+ /* For single writes, FLUSH_STABLE is more efficient */
+ if (res == cp->npages && cp->npages <= server->wpages) {
+ if (res > 1 || cfs_list_entry(head->next)->wb_bytes <= server->wsize)
+ how |= FLUSH_STABLE;
+ }
+ return cfs_flush_list(head, server->wpages, how);
+#endif
+}
+
int cfs_flush_inode(struct inode *inode, unsigned long idx_start,
unsigned int npages, int how)
{
@@ -2121,21 +2146,8 @@
spin_lock(&cp->req_lock);
res = cfs_scan_dirty(inode, &head, idx_start, npages);
spin_unlock(&cp->req_lock);
-#if !defined(KERNEL2613) && !defined(KERNEL2616_53)
if (res)
- error = cfs_flush_list(&head, CFS_SERVER(inode)->wpages, how);
-#else
- if (res) {
- struct cfs_server *server = CFS_SERVER(inode);
-
- /* For single writes, FLUSH_STABLE is more efficient */
- if (res == cp->npages && cp->npages <= server->wpages) {
- if (res > 1 || cfs_list_entry(head.next)->wb_bytes <= server->wsize)
- how |= FLUSH_STABLE;
- }
- error = cfs_flush_list(&head, server->wpages, how);
- }
-#endif
+ error = __cfs_flush_list(inode, &head, res, how);
if (error < 0)
return error;
return res;
@@ -2143,19 +2155,25 @@
#ifdef CFS_CHARD_SYNC_FIX
/*
- * Move requests marked for commit to dirty queue.
+ * Re-flush requests marked for commit.
+ *
+ * This is only necessary on failover to ensure page cache at new server is
+ * re-populated with dirty pages not yet commited to stable storage.
+ * There is a window between the time of server failure and completion of
+ * failover prior cfs_commit_inode() while we slept in cfs_wait_on_requests().
+ * -Roger
*/
static int
-cfs_prepare_reflush(
+cfs_reflush_inode(
struct inode *inode,
unsigned long idx_start,
unsigned int npages,
- struct writeback_control *wbc)
+ int how)
{
- struct cfs_page *req;
cnode_t *cp = itoc(inode);
LIST_HEAD(head);
unsigned int res;
+ int error = 0;
spin_lock(&cp->req_lock);
res = cfs_scan_commit(inode, &head, idx_start, npages);
@@ -2166,20 +2184,9 @@
/* Undo nr_unstable in cfs_mark_request_commit() */
sub_page_state(nr_unstable,res);
- while (!list_empty(&head)) {
- req = cfs_list_entry((&head)->next);
- cfs_list_remove_request(req);
- cfs_mark_request_dirty(req);
-#ifdef KERNEL2616_53
- cfs_clear_page_writeback(req);
-#else
- cfs_unlock_request(req);
-#endif
- }
-
- if (wbc)
- wbc->nr_to_write += res;
-
+ error = __cfs_flush_list(inode, &head, res, how);
+ if (error < 0)
+ return error;
return res;
}
#endif /* CFS_CHARD_SYNC_FIX */
@@ -2236,16 +2243,18 @@
error = 0;
if (wait)
error = cfs_wait_on_requests(inode, idx_start, npages);
+ if (error == 0)
+ error = cfs_flush_inode(inode, idx_start, npages, how);
#ifdef CFS_CHARD_SYNC_FIX
- /* If server failed over, flush unstable data to new server. */
if (error >= 0 && node != itocmi(inode)->mi_server) {
node = itocmi(inode)->mi_server;
- cfs_prepare_reflush(inode, idx_start, npages, NULL);
+ /* Server failed over already so we need to
+ * re-flush unstable data to new server.
+ */
+ error = cfs_reflush_inode(inode, idx_start, npages, how);
}
#endif
if (error == 0)
- error = cfs_flush_inode(inode, idx_start, npages, how);
- if (error == 0)
#ifndef KERNEL2613
error = cfs_commit_inode(inode, idx_start, npages, how);
#else
------------------------------------------------------------------------------
Download Intel® Parallel Studio Eval
Try the new software tools for yourself. Speed compiling, find bugs
proactively, and fine-tune applications for parallel performance.
See why Intel Parallel Studio got high marks during beta.
http://p.sf.net/sfu/intel-sw-dev