Re: How to automatically add a nbd to a raid1?
Peter Breuer <[email protected]> Sun, 28 Dec 2008 03:43:27 +0100 (CET)
| Newsgroups | gmane.linux.enbd.general |
|---|---|
| Message-ID | <[email protected]> |
My udev patch wasn't enough. It made the 2.4.35 enbd device signal udev
when it signaled at all .. but it wasn't signalling all the times it
should have.
Now it does (I sneaked both this and the other patch into the existing
enbd-2.4.35 archive without telling). The changes don't affect anything
except this particular very limited subsystem.
Debugging udev now shows online/offline messages being passed (one needs
to set show_errs=3D1 on the enbd device to make it happen).
Dec 28 03:05:13 barney udevd[1053]:
msg_queue_insert: seq 2283 queued, 'online' 'block'
Dec 28 03:05:13 barney udevd-event[6068]:
pass_env_to_socket: passed 135 bytes to socket '/org/freedesktop/hal/=
udev_event',=20
Dec 28 03:05:13 barney udevd-event[6068]:
pass_env_to_socket: passed -1 bytes to socket '/org/kernel/udev/monit=
or',=20
Dec 28 03:05:13 barney udevd-event[6068]:
udev_event_run: seq 2283 finished with 0
Dec 28 03:05:13 barney udevd[1053]:
udev_event_run: seq 2283 forked, pid [6068], =F3nline' 'block', 0 sec=
onds old
Dec 28 03:05:13 barney udevd[1053]:
udev_done: seq 2283, pid [6068] exit with 0, 0 seconds old
I've been able to match these messages in the enbd udev rules file like t=
his:
KERNEL=3D=3D"nd[a-z]", \
SUBSYSTEM=3D=3D"block", \
ACTION=3D=3D"online", \
PROGRAM=3D"/usr/bin/logger -p daemon.notice -t udev says $kernel is onl=
ine"
KERNEL=3D=3D"nd[a-z]", \
SUBSYSTEM=3D=3D"block", \
ACTION=3D=3D"offline", \
PROGRAM=3D"/usr/bin/logger -p daemon.notice -t udev says $kernel is off=
line"
Giving rise to log messages like this:
Dec 28 03:28:00 barney enbd-server[7632]:
enbd-server #2052 newproto[0]: (ERROR) net errored (header 64 got 0) =
on req. Breaking off.=20
Dec 28 03:28:00 barney udev:
says ndb is offline
The extra patch for more signalling was:
diff -urbN enbd-2.4.35/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c =
enbd-2.4.35a/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c
--- enbd-2.4.35/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c 2008-06=
-01 10:43:05.000000000 +0200
+++ enbd-2.4.35a/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c 2008-0=
7-22 12:15:32.000000000 +0200
@@ -3406,6 +3526,7 @@
if (atomic_test_and_clear_mask (&lo->flags, ENBD_VALIDATED))
set_invalid =3D 1;
lo->lives++;
- did_enabled =3D 1;
+ if (!(lo->flags & ENBD_REMOTE_INVALID))
+ did_enabled =3D 1;
}=20
write_unlock_irqrestore (&lo->meta_lock, flags);
@@ -3956,16 +4069,25 @@
=20
unsigned long flags;
int cleared_show_errs =3D 0;
+ int did_enabled =3D 0;
=20
write_lock_irqsave (&lo->meta_lock, flags);
if (atomic_test_and_clear_mask(&lo->flags, ENBD_REMOTE_INVALID)
&& atomic_test_and_clear_mask (&lo->flags, ENBD_SET_SHOW_ERRS)
- && atomic_test_and_clear_mask (&lo->flags, ENBD_SHOW_ERRS))=20
+ && atomic_test_and_clear_mask (&lo->flags, ENBD_SHOW_ERRS)) {
cleared_show_errs =3D 1;
+ if (lo->flags & ENBD_ENABLED)
+ did_enabled =3D 1;
+ }
write_unlock_irqrestore (&lo->meta_lock, flags);
=20
if (cleared_show_errs)
ENBD_INFO ( "cleared show_errs on nd%s\n", lo->devnam);
+ if (did_enabled) {
+ ENBD_ALERT("ENABLED nd%s\n", lo->devnam);
+ //__invalidate_device(lo->inode->i_bdev, 0);
+ enbd_notify_md_devices(lo, HOT_ADD_DISK);
+ }
return 0;
}
=20
@@ -3987,6 +4109,7 @@
int do_invalidate =3D 0;
int set_invalid =3D 0;
int set_show_errs =3D 0;
+ int did_disabled =3D 0;
=20
write_lock_irqsave (&lo->meta_lock, flags);
if (!(atomic_test_and_set_mask (&lo->flags, ENBD_REMOTE_INVALID))) {
@@ -4004,6 +4127,8 @@
}
// PTB test removing partitions
do_invalidate =3D 1;
+ if (lo->flags & ENBD_ENABLED)
+ did_disabled =3D 1;
}
write_unlock_irqrestore (&lo->meta_lock, flags);
=20
@@ -4026,6 +4151,10 @@
// PTB will cause requests to start being errored
//invalidate_partition(lo->disk, 0);
}=20
+ if (did_disabled) {
+ ENBD_ALERT("DISABLED nd%s\n", lo->devnam);
+ enbd_notify_md_devices(lo, SET_DISK_FAULTY);
+ }
=20
return 0;
}
@@ -4357,9 +4488,9 @@
=20
if (atomic_test_and_clear_mask (&lo->flags, ENBD_ENABLED)) {
ENBD_ALERT ("disabled device nd%s\n", lo->devnam);
- }
-
- enbd_notify_md_devices(lo, SET_DISK_FAULTY);
+ if (!(lo->flags & ENBD_REMOTE_INVALID))
+ enbd_notify_md_devices(lo, SET_DISK_FAULTY);
+ }
=20
// PTB have to recheck partitions on next open
if (atomic_test_and_clear_mask (&lo->flags, ENBD_VALIDATED)) {
@@ -6037,6 +6204,8 @@
ENBD_ALERT("REMOTE CHECK set REMOTE INVALID on n=
d%s\n",
lo->devnam);
//__invalidate_device(lo->bdev, 0);
+ if (lo->flags & ENBD_ENABLED)
+ enbd_notify_md_devices(lo, SET_DISK_FAUL=
TY);
return 1;
} else {
ENBD_ALERT("REMOTE CHECK still has REMOTE INVALI=
D on nd%s\n",
@@ -6049,6 +6218,8 @@
if (atomic_test_and_clear_mask (&lo->flags, ENBD_REMOTE_INVALID)=
) {
ENBD_ALERT("REMOTE CHECK set REMOTE VALID on nd%s\n",
lo->devnam);
+ if (lo->flags & ENBD_ENABLED)
+ enbd_notify_md_devices(lo, HOT_ADD_DISK);
return 1;
}=20
// PTB this is called by check_disk_change
@@ -6257,14 +6415,19 @@
enbd_reset_and_sync (struct enbd_device *lo)
{
int set_invalid =3D 0;
+ int did_disabled =3D 0;
=20
write_lock(&lo->meta_lock);
if (atomic_read(&lo->refcnt) > 0) {
write_unlock(&lo->meta_lock);
ENBD_ALERT("enbd_reset called on open device\n");
return;
}
- enbd_atomic_clear_mask(ENBD_ENABLED, &lo->flags);
+ if (atomic_test_and_clear_mask (&lo->flags, ENBD_ENABLED)) {
+ ENBD_ALERT ("disabled device nd%s\n", lo->devnam);
+ if (!(lo->flags & ENBD_REMOTE_INVALID))
+ did_disabled =3D 1;
+ }
if (atomic_test_and_clear_mask (&lo->flags, ENBD_VALIDATED))=20
set_invalid =3D 1;
=20
@@ -6285,6 +6439,9 @@
if (set_invalid)
ENBD_INFO("set INVALID on nd%s\n", lo->devnam);
=20
+ if (did_disabled)
+ enbd_notify_md_devices(lo, SET_DISK_FAULTY);
+
enbd_sync_sync(lo); // PTB takes no lock
//__invalidate_device(lo->bdev, 0);
ENBD_INFO ("reset device nd%s\n", lo->devnam);
@@ -6867,7 +7261,10 @@
struct gendisk *disk =3D lo->disk;
int i;
=20
- enbd_atomic_clear_mask (ENBD_ENABLED, &lo->flags);
+ if (atomic_test_and_clear_mask (&lo->flags, ENBD_ENABLED)) {
+ ENBD_ALERT ("disabled device nd%s\n", lo->devnam);
+ enbd_notify_md_devices(lo, SET_DISK_FAULTY);
+ }
=20
// PTB flush seems to cause sporadic oopses
//enbd_sync_sync (lo);
The other patch I sent seemed to just about apply (I had one line
wrong by eye, so I did it by hand when I applied it). But it also
needed to change the way kobj is referenced for very recent kernels.
This patch is fixed up to do that too (it's an ifdef) .. anyway, it's
what I am running on top of 2.4.35 to make this report.
Apologies for seeming vague .. 2.4.35 is nominally stable, which means
it is behind the times! (about a year). I have to guess when dealing
with it .. and then later I go and really really check and see what
is exactly up. It probably could do with emitting those messages
without prompting from show_errs=3D1 too. But show_errs=3D1 is correct fo=
r=20
raid use.
(This patch supports talking to udev about connection or disconnection, u=
sing
kobjs).
diff -urbN enbd-2.4.35/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c =
enbd-2.4.35a/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c
--- enbd-2.4.35/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c 2008-06=
-01 10:43:05.000000000 +0200
+++ enbd-2.4.35a/kernel/linux-2.6.x/drivers/block/enbd/enbd_base.c 2008-0=
7-22 12:15:32.000000000 +0200
@@ -270,6 +274,7 @@
#if LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,20)
# include <linux/freezer.h>
#endif /* LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,20) */
+#include <linux/kobject.h>
=20
#if defined(MODULE)
int linux_version_code =3D LINUX_VERSION_CODE;
@@ -3354,6 +3397,46 @@
return;
}
=20
+/*
+ * PTB - For use in set_disk and enbd_notify_md_devices nowhere else.
+ * Needs lo->disk.
+ *
+ * The problem with using get_disk here is that it raises the refcount b=
y
+ * one and then prevents use of rmmod, since the count is 1. Running
+ * rmmod would lower the count to 0, but we can't, since the count is
+ * one (so use rmmod -f). The alternative is to let close lower the coun=
t.
+ */
+static struct kobject *
+enbd_find_kobj (dev_t dev, int *part, void *data)
+{
+ struct enbd_device *lo =3D data;
+ if (!lo) {
+ ENBD_ERROR("null enbd device given\n");
+ return ERR_PTR(-ENOMEM);
+ }
+ if (lo->magic !=3D ENBD_DEV_MAGIC) {
+ ENBD_ERROR("enbd device has bad magic\n");
+ return ERR_PTR(-EINVAL);
+ }
+ if (!lo->disk) {
+ ENBD_ERROR("no disk yet set for nd%s\n", lo->devnam);
+ return ERR_PTR(-EINVAL);
+ }
+ if (part && (*part < 0 || *part >=3D ENBD_MAXCONN)) {
+ ENBD_ERROR("invalid partition %d for nd%s\n",
+ part ? *part : -2, lo->devnam);
+ return ERR_PTR(-EINVAL);
+ }
+ if (!lo->kobj)
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+ lo->kobj =3D &lo->disk->kobj; // PTB avoids incrementing owners
+#else /* LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,25) */
+ lo->kobj =3D &lo->disk->dev.kobj; // PTB avoids incrementing ow=
ners
+#endif /* LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,25) */
+ return lo->kobj;
+}
+
+
#ifndef HOT_ADD_DISK
#define HOT_ADD_DISK _IO (MD_MAJOR, 0x28)
#endif
@@ -3370,9 +3453,11 @@
{
int j;
struct enbd_md *md =3D &enbd_md;
+ dev_t enbd_dev;
+ struct kobject *kobj;
=20
for (j =3D 0; j - 1 < lo->nslot; j++) {
- dev_t enbd_dev =3D MKDEV (major, j + (lo->nbd << ENBD_SHIFT));
+ enbd_dev =3D MKDEV (major, j + (lo->nbd << ENBD_SHIFT));
if (j =3D=3D 0) {
if (atomic_read (&lo->md_count) <=3D 0)
continue;
@@ -3381,8 +3466,34 @@
if (slot->md_count <=3D 0)
continue;
}
md->notify(md, enbd_dev, cmd);
}
+ if (!lo->disk) {
+ ENBD_ALERT ("no disk found, can't notify udev nd%s is %s\n",
+ lo->devnam, cmd=3D=3DHOT_ADD_DISK?"up":"down");
+ // PTB if no disk, lo->kobj can't equal &disk->kobj
+ return 0;
+ }
+ if (!lo->kobj) {
+ ENBD_ALERT ("no kobj found, can't notify udev nd%s is %s\n",
+ lo->devnam, cmd=3D=3DHOT_ADD_DISK?"up":"down");
+ return 0;
+ }
+ kobj =3D lo->kobj;
+
+ switch (cmd) {
+ case HOT_ADD_DISK:
+ kobject_uevent(kobj, KOBJ_ONLINE);
+ ENBD_ALERT ("notified udev nd%s is up\n", lo->devnam);
+ break;
+ case SET_DISK_FAULTY:
+ kobject_uevent(kobj, KOBJ_OFFLINE);
+ ENBD_ALERT ("notified udev nd%s is down\n", lo->devnam);
+ break;
+ default:
+ ENBD_ERROR ("unknown cmd %d on nd%s\n", cmd, lo->devnam);
+ return -EINVAL;
+ }
return 0;
}
=20
@@ -6190,25 +6367,6 @@
=20
=20
/*
- * PTB - For use in set_disk and nowhere else. Needs lo->disk.
- */
-static struct kobject *
-enbd_find (dev_t dev, int *part, void *data)
-{
- struct enbd_device *lo =3D data;
- if (!lo)
- return NULL;
- if (lo->magic !=3D ENBD_DEV_MAGIC)
- return NULL;
- if (!lo->disk)
- return NULL;
- if (part && (*part < 0 || *part >=3D ENBD_MAXCONN))
- return NULL;
- return get_disk (lo->disk);
-}
-
-
-/*
* PTB - used in setup to fix the gendisk data, chain the gendisk,
* claim the blockdev region, etc.
*/
@@ -6234,7 +6392,7 @@
// PTB mark disk removable! FIXME
disk->flags |=3D GENHD_FL_REMOVABLE;
blk_register_region (MKDEV (major, first_minor),
- npart, THIS_MODULE, enbd_find, NULL, lo);
+ npart, THIS_MODULE, enbd_find_kobj, NULL, lo);
set_capacity (disk, lo->sectors);
// we should rescan later. From userland?
add_disk (disk);
@@ -6748,6 +7139,12 @@
}
=20
lo->disk =3D disk;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,25)
+ lo->kobj =3D &disk->kobj;
+#else /* LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,25) */
+ lo->kobj =3D &disk->dev.kobj;
+#endif /* LINUX_VERSION_CODE >=3D KERNEL_VERSION(2,6,25) */
+
enbd_init_queue(lo, disk->queue);
enbd_init_ioctl_stub(&lo->remote_ioctl);
=20
@@ -6857,6 +7250,7 @@
}
put_disk(disk);
lo->disk =3D NULL;
+ lo->kobj =3D NULL;
}
return err < 0 ? err : -EIO;
}
@@ -6889,10 +7286,6 @@
del_gendisk (disk);
put_disk (disk);
lo->disk =3D NULL;
+ lo->kobj =3D NULL;
}
- if (lo->blockmap) {
- kfree (lo->blockmap);
- lo->blockmap =3D NULL;
- lo->kobj =3D NULL;
- }
// PTB this should be under lock, but do we really care now?
spin_lock(&lo->lock);
diff -urbN enbd-2.4.35/kernel/linux-2.6.x/include/linux/enbd.h enbd-2.4.3=
5a/kernel/linux-2.6.x/include/linux/enbd.h
--- enbd-2.4.35/kernel/linux-2.6.x/include/linux/enbd.h 2008-04-19 15:40:=
49.000000000 +0200
+++ enbd-2.4.35a/kernel/linux-2.6.x/include/linux/enbd.h 2008-05-30 12:24=
:15.000000000 +0200
@@ -356,7 +380,7 @@
struct inode * inode; /* PTB add - for ref */
int bufsiz; /* PTB add - userspace buffer=
size */
atomic_t kmax; /* PTB add - max kernel threa=
ds */
- char *blockmap; /* PTB add - map of block sta=
tes */
+ struct kobject *kobj;
unsigned long disabled; /* PTB add - when was it disa=
bled */
int req_timeo; /* PTB add - net timeout */
int pulse_intvl; /* PTB add - inactivity intvl=
*/
Peter