Re: [RFC] ext3 multi-mount protection

Qi Yong <[email protected]>
Newsgroups gmane.comp.file-systems.ext2.devel
Organization CFS
Message-ID <[email protected]> 
Andreas Dilger wrote:
> Is there any ext3/ext4 interest in a mechanism for protecting ext3/ext4
> filesystems from being mounted concurrently by multiple nodes on a SAN
> at the same time?
> 
> We often run the ext3 server filesystems in an HA configuration so that
> there are two servers on a FC switch that can access the same shared
> disk.  Sadly, HA software is far from reliable and this can sometimes
> lead to both servers trying to access the same device at one time.
> This is true if you are using ext3 for HA NFS serving, or other non-HA
> setups where the same disks are accessible by multiple nodes (regardless
> of whether they are _supposed_ to touch those disks or not), because the
> ext3 journal recovery can corrupt the filesystem even if mounted read-only,
> or if some clueless sysadmin tries to mount or use "those unused disks".
> 
> We've developed a small patch for ext3 to prevent the filesystem from
> being mounted concurrently on two nodes.  Is there general interest in
> this becoming part of ext3/ext4?
> 
> Cheers, Andreas
> --
> Andreas Dilger
> Principal Software Engineer
> Cluster File Systems, Inc.


Hello,

The attached is our mmp v3.0 kernel patch against the current vanilla kernel in git.
(mmp: multi-mount protection)

tytso, could you possibly reserve this flag? 
#define EXT3_FEATURE_INCOMPAT_ALIVE            0x0080


Rationale
---------

An incompatible feature "alive" is introduced to enable mmp.
With feature alive set:

On an active fs, kalived updates "sequence number" in "alive block" at 5-second intervals.
At mount time, the alive block is read and tested to detect if the mounting fs is actively used
by other nodes or not. If the fs is active, mount fails; otherwise start kalived and continue.


Algorithm
---------

1    read alive_struct
2    check sequence number == 0? yes, goto 6
3    wait 8s
4    reread alive_struct
5    seq change? yes = exit
6    write new random sequence number
7    wait 6s
8    reread alive_struct
9    seq change? yes = exit
10   start kalived


alive block
-----------

#define	ALIVE_MAGIC	0xA1153C29
struct alive_struct {
	__le32	al_magic;
	__le32	al_seq;
	__le32	al_time;
	char	al_nodename[65];
};

In v1, the alive block resides in the unused sector #0.
In v2, the alive block resides in a hidden inode.
In v3, the alive block resides in a regular file "/.alive".

-- 
Qi Yong
System Software Engineer
Cluster File Systems, Inc.


-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642

_______________________________________________
Ext2-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/ext2-devel
mmp-v3.0.patch (text/x-patch, 10.1 KB) 
Signed-off-by: Qi Yong <[email protected]>
---

 fs/ext3/al.h               |   11 +
 fs/ext3/namei.c            |    2 
 fs/ext3/super.c            |  262 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/ext3_fs.h    |    4 
 include/linux/ext3_fs_sb.h |    1 
 5 files changed, 278 insertions(+), 2 deletions(-)

diff --git a/fs/ext3/al.h b/fs/ext3/al.h
new file mode 100644
index 0000000..88b9598
--- /dev/null
+++ b/fs/ext3/al.h
@@ -0,0 +1,11 @@
+/*
+ * (C) 2006  Qi Yong <[email protected]>
+ */
+
+#define	ALIVE_MAGIC	0xA1153C29
+struct alive_struct {
+	__le32	al_magic;
+	__le32	al_seq;
+	__le32	al_time;
+	char	al_nodename[65];
+};
diff --git a/fs/ext3/namei.c b/fs/ext3/namei.c
index d9176db..4201a8f 100644
--- a/fs/ext3/namei.c
+++ b/fs/ext3/namei.c
@@ -804,7 +804,7 @@ static inline int search_dirblock(struct
  * The returned buffer_head has ->b_count elevated.  The caller is expected
  * to brelse() it when appropriate.
  */
-static struct buffer_head * ext3_find_entry (struct dentry *dentry,
+struct buffer_head * ext3_find_entry (struct dentry *dentry,
 					struct ext3_dir_entry_2 ** res_dir)
 {
 	struct super_block * sb;
diff --git a/fs/ext3/super.c b/fs/ext3/super.c
index 813d589..042b5ab 100644
--- a/fs/ext3/super.c
+++ b/fs/ext3/super.c
@@ -16,6 +16,7 @@
  *        David S. Miller ([email protected]), 1995
  */
 
+#define DEBUG
 #include <linux/module.h>
 #include <linux/string.h>
 #include <linux/fs.h>
@@ -35,12 +36,14 @@ #include <linux/mount.h>
 #include <linux/namei.h>
 #include <linux/quotaops.h>
 #include <linux/seq_file.h>
+#include <linux/kthread.h>
 
 #include <asm/uaccess.h>
 
 #include "xattr.h"
 #include "acl.h"
 #include "namei.h"
+#include "al.h"
 
 static int ext3_load_journal(struct super_block *, struct ext3_super_block *,
 			     unsigned long journal_devnum);
@@ -61,6 +64,8 @@ static int ext3_statfs (struct dentry * 
 static void ext3_unlockfs(struct super_block *sb);
 static void ext3_write_super (struct super_block * sb);
 static void ext3_write_super_lockfs(struct super_block *sb);
+struct buffer_head * ext3_find_entry (struct dentry *dentry,
+					struct ext3_dir_entry_2 ** res_dir);
 
 /* 
  * Wrappers for journal_start/end.
@@ -430,6 +435,10 @@ #endif
 		invalidate_bdev(sbi->journal_bdev, 0);
 		ext3_blkdev_remove(sbi);
 	}
+	if (EXT3_HAS_INCOMPAT_FEATURE(sb, EXT3_FEATURE_INCOMPAT_ALIVE)) {
+		BUG_ON(!sbi->s_alive_tsk);
+		kthread_stop(sbi->s_alive_tsk);
+	}
 	sb->s_fs_info = NULL;
 	kfree(sbi);
 	return;
@@ -1338,6 +1347,255 @@ static ext3_fsblk_t descriptor_loc(struc
 	return (has_super + ext3_group_first_block_no(sb, bg));
 }
 
+static int write_alive(struct buffer_head * bh)
+{
+	lock_buffer(bh);
+	bh->b_end_io = end_buffer_write_sync;
+	get_bh(bh);
+	submit_bh(WRITE, bh);
+	wait_on_buffer(bh);
+	if (unlikely(!buffer_uptodate(bh)))
+		return 1;
+	return 0;
+}
+
+static int read_alive_again(struct buffer_head * bh)
+{
+	lock_buffer(bh);
+	bh->b_end_io = end_buffer_read_sync;
+	get_bh(bh);
+	submit_bh(READ, bh);
+	wait_on_buffer(bh);
+	if (!buffer_uptodate(bh)) {
+		brelse(bh);
+		return 1;
+	}
+	return 0;
+}
+
+/*
+ * The caller must have a ref on the buffer_head.
+ */
+static int kalived(void *data)
+{
+	struct buffer_head * bh;
+	struct alive_struct * alive;
+	char b[BDEVNAME_SIZE];
+	u32 seq = 0;
+
+	bh = (struct buffer_head *)data;
+	bdevname(bh->b_bdev, b);
+
+	alive = (struct alive_struct *)(bh->b_data);
+	alive->al_magic = cpu_to_le32(ALIVE_MAGIC);
+	alive->al_time = cpu_to_le32(get_seconds());
+
+	down_read(&uts_sem);
+	memcpy(alive->al_nodename, system_utsname.nodename, 65);
+	up_read(&uts_sem);
+
+	while (!kthread_should_stop()) {
+		if (++seq == 0)
+			++seq;
+
+		alive->al_seq = cpu_to_le32(seq);
+		alive->al_time = cpu_to_le32(get_seconds());
+
+		if (unlikely(write_alive(bh))) {
+			/* panic here? */
+			printk(KERN_ERR "Alive (device %s): "
+				"can't write alive block\n", b);
+			continue;
+		}
+
+		schedule_timeout_interruptible(5 * HZ);
+	}
+
+	alive->al_seq = 0;
+	alive->al_time = cpu_to_le32(get_seconds());
+
+	if (unlikely(write_alive(bh)))
+		printk(KERN_ERR "Alive (device %s): "
+			"can't reset alive block\n", b);
+	brelse(bh);
+	return 0;
+}
+
+static unsigned long get_alive_ino(struct super_block *sb)
+{
+	unsigned long	ino = 0;
+	struct dentry	alive;
+	struct dentry	* root;
+	struct inode	* root_inode;
+	struct ext3_dir_entry_2 * de;
+	struct buffer_head	* bh;
+
+	root_inode = iget(sb, EXT3_ROOT_INO);
+	root = d_alloc_root(root_inode);
+	if (!root) {
+		printk(KERN_ERR "Alive (device %s): get root inode failed\n",
+			sb->s_id);
+		iput(root_inode);
+		goto out;
+	}
+
+	alive.d_name.name = ".alive";
+	alive.d_name.len = 6;
+	alive.d_parent = root;
+
+	bh = ext3_find_entry(&alive, &de);
+	dput(root);
+
+	if (!bh) {
+		printk(KERN_WARNING "Alive (device %s): alive lookup failed\n",
+			sb->s_id);
+		goto out;
+	}
+
+	ino = le32_to_cpu(de->inode);
+	brelse (bh);
+	pr_debug("Alive (device %s): alive_ino=%lu\n", sb->s_id, ino);
+out:
+	return ino;
+}
+
+/* check alive file */
+static int check_alive(struct super_block *sb, struct ext3_sb_info *sbi)
+{
+	unsigned long		ino;
+	struct buffer_head	* bh;
+	struct ext3_inode_info	* ei;
+	struct inode		* alive_inode;
+	struct alive_struct	* alive;
+	u32 alive_block;
+	u32 seq;
+
+	ino = get_alive_ino(sb);
+	if (!ino)
+		goto failed;
+
+	alive_inode = iget(sb, ino);
+	if (!alive_inode) {
+		iput(alive_inode);
+		printk(KERN_ERR "Alive (device %s): get alive inode failed\n",
+			sb->s_id);
+		goto failed;
+	}
+	if (!alive_inode->i_nlink) {
+		make_bad_inode(alive_inode);
+		iput(alive_inode);
+		printk(KERN_ERR "Alive (device %s): alive inode is deleted\n",
+			sb->s_id);
+		goto failed;
+	}
+	if (!S_ISREG(alive_inode->i_mode)) {
+		iput(alive_inode);
+		printk(KERN_ERR "Alive (device %s): invalid alive inode\n",
+			sb->s_id);
+		goto failed;
+	}
+
+	ei = EXT3_I(alive_inode);
+	alive_block = ei->i_data[0];
+	iput(alive_inode);
+
+	pr_debug("Alive (device %s): read in alive block #%u\n",
+			sb->s_id, alive_block);
+
+	/* first read */
+	bh = sb_bread(sb, alive_block);
+	if (!bh) {
+		printk(KERN_ERR "Alive (device %s): "
+			"can't read alive block #%u\n", sb->s_id, alive_block);
+		goto failed;
+	}
+
+	alive = (struct alive_struct *)(bh->b_data);
+	if (le32_to_cpu(alive->al_magic) != ALIVE_MAGIC) {
+		printk(KERN_ERR "Alive (device %s): "
+			"magic mismatch\n", sb->s_id);
+		brelse(bh);
+		goto failed;
+	}
+
+	seq = le32_to_cpu(alive->al_seq);
+	pr_debug("Alive (device %s): seq=%u\n", sb->s_id, seq);
+	pr_info ("Alive (device %s): last touched by node: %s, "
+		"%li seconds ago\n", sb->s_id, alive->al_nodename,
+		get_seconds() - le32_to_cpu(alive->al_time));
+
+	if (seq == 0)
+		goto skip;
+
+	/* wait 8s */
+	pr_info("Alive (device %s): wait for 8 seconds...\n", sb->s_id);
+	schedule_timeout_uninterruptible(HZ * 8);
+
+	/* read again */
+	if (read_alive_again(bh)) {
+		printk(KERN_ERR "Alive (device %s): "
+			"can't read alive block #%u\n",
+			sb->s_id, alive_block);
+		goto failed;
+	}
+
+	alive = (struct alive_struct *)(bh->b_data);
+	pr_debug("Alive (device %s): seq=%u\n",
+		sb->s_id, le32_to_cpu(alive->al_seq));
+
+	if (seq != le32_to_cpu(alive->al_seq)) {
+		printk(KERN_WARNING "Alive (device %s): "
+			"still active on node %s\n",
+			sb->s_id, alive->al_nodename);
+		brelse(bh);
+		goto failed;
+	}
+skip:
+	/* write a new random seq */
+	get_random_bytes(&seq, sizeof(u32));
+	alive->al_seq = cpu_to_le32(seq);
+	if (unlikely(write_alive(bh))) {
+		printk(KERN_ERR "Alive (device %s): "
+			"can't write alive block\n", sb->s_id);
+		goto failed;
+	}
+	pr_debug("Alive (device %s): write random seq=%u\n", sb->s_id, seq);
+
+	/* wait 6s */
+	pr_info("Alive (device %s): wait for 6 seconds...\n", sb->s_id);
+	schedule_timeout_uninterruptible(HZ * 6);
+
+	/* read again */
+	if (read_alive_again(bh)) {
+		printk(KERN_ERR "Alive (device %s): "
+			"can't read alive block #%u\n",
+			sb->s_id, alive_block);
+		goto failed;
+	}
+
+	alive = (struct alive_struct *)(bh->b_data);
+	pr_debug("Alive (device %s): seq=%u\n",
+		sb->s_id, le32_to_cpu(alive->al_seq));
+
+	if (seq != le32_to_cpu(alive->al_seq)) {
+		printk(KERN_WARNING "Alive (device %s): "
+			"still active on node %s\n",
+			sb->s_id, alive->al_nodename);
+		brelse(bh);
+		goto failed;
+	}
+
+	/* succeed */
+	pr_info("Alive (device %s): alive check passed!\n", sb->s_id);
+	sbi->s_alive_tsk = kthread_run(kalived, bh, "kalived");
+	return 0;
+
+failed:
+	printk(KERN_WARNING "Alive (device %s): alive check failed!\n",
+		sb->s_id);
+	return 1;
+}
+
 
 static int ext3_fill_super (struct super_block *sb, void *data, int silent)
 {
@@ -1651,6 +1909,10 @@ #endif
 			  EXT3_HAS_INCOMPAT_FEATURE(sb,
 				    EXT3_FEATURE_INCOMPAT_RECOVER));
 
+	if (EXT3_HAS_INCOMPAT_FEATURE(sb, EXT3_FEATURE_INCOMPAT_ALIVE))
+		if (check_alive(sb, sbi))
+			goto failed_mount2;
+
 	/*
 	 * The first inode we look at is the journal inode.  Don't try
 	 * root first: it may be modified in the journal!
diff --git a/include/linux/ext3_fs.h b/include/linux/ext3_fs.h
index 5607e64..e2293d0 100644
--- a/include/linux/ext3_fs.h
+++ b/include/linux/ext3_fs.h
@@ -560,11 +560,13 @@ #define EXT3_FEATURE_INCOMPAT_FILETYPE		
 #define EXT3_FEATURE_INCOMPAT_RECOVER		0x0004 /* Needs recovery */
 #define EXT3_FEATURE_INCOMPAT_JOURNAL_DEV	0x0008 /* Journal device */
 #define EXT3_FEATURE_INCOMPAT_META_BG		0x0010
+#define EXT3_FEATURE_INCOMPAT_ALIVE		0x0080
 
 #define EXT3_FEATURE_COMPAT_SUPP	EXT2_FEATURE_COMPAT_EXT_ATTR
 #define EXT3_FEATURE_INCOMPAT_SUPP	(EXT3_FEATURE_INCOMPAT_FILETYPE| \
 					 EXT3_FEATURE_INCOMPAT_RECOVER| \
-					 EXT3_FEATURE_INCOMPAT_META_BG)
+					 EXT3_FEATURE_INCOMPAT_META_BG| \
+					 EXT3_FEATURE_INCOMPAT_ALIVE)
 #define EXT3_FEATURE_RO_COMPAT_SUPP	(EXT3_FEATURE_RO_COMPAT_SPARSE_SUPER| \
 					 EXT3_FEATURE_RO_COMPAT_LARGE_FILE| \
 					 EXT3_FEATURE_RO_COMPAT_BTREE_DIR)
diff --git a/include/linux/ext3_fs_sb.h b/include/linux/ext3_fs_sb.h
index f61309c..43627b3 100644
--- a/include/linux/ext3_fs_sb.h
+++ b/include/linux/ext3_fs_sb.h
@@ -78,6 +78,7 @@ #ifdef CONFIG_QUOTA
 	char *s_qf_names[MAXQUOTAS];		/* Names of quota files with journalled quota */
 	int s_jquota_fmt;			/* Format of quota to use */
 #endif
+	struct task_struct * s_alive_tsk;
 };
 
 #endif	/* _LINUX_EXT3_FS_SB */
lmpx.com only provides a reader for public news (NNTP) servers. It is not affiliated with the servers or forums shown here and is not responsible for the content of articles, which is written by their respective authors.