[SSI] openssi/kernel/cluster/ssi/net ipvs_ssisys.c, 1.11, 1.12 ipvs_svr.c, 1.25, 1.26
Roger Tsang <[email protected]>
| Newsgroups | gmane.linux.cluster.ssic.cvs |
|---|---|
| Message-ID | <[email protected]> |
Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/net
In directory fdv4jf1.ch3.sourceforge.com:/tmp/cvs-serv16224/kernel/cluster/ssi/net
Modified Files:
Tag: OPENSSI-FC
ipvs_ssisys.c ipvs_svr.c
Log Message:
Bug fixes and enhancements. See ChangeLog.
Index: ipvs_ssisys.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/net/ipvs_ssisys.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- ipvs_ssisys.c 3 Feb 2009 06:18:12 -0000 1.11
+++ ipvs_ssisys.c 27 Oct 2009 03:18:29 -0000 1.12
@@ -23,6 +23,7 @@
#include <cluster/nsc.h>
#include <cluster/nodelist.h>
#include <cluster/clms.h>
+#include <cluster/ssi/net.h>
#include <cluster/gen/ics_clip_protos_gen.h>
#include <cluster/gen/ics_clip_macros_gen.h>
@@ -62,16 +63,19 @@
continue;
(void) RIPVS_SETDIRECTOR_MSG(node, dipaddr, dnode);
}
+ NSC_NODELIST_FREE(nl);
return 0;
}
-int ipvs_add_entry(clusternode_t node, u_int cvip, u_short port,
+int
+_ipvs_add_entry(clusternode_t node, u_int cvip, u_short port,
u_int rip, int weight, u_short protocol, char *lvs_sched)
{
-
int retval;
+ BUG_ON(node == this_node);
+
if (node == this_node) {
/* This node is the director node for the CVIP */
/* Should we handle the return of this function */
@@ -89,11 +93,32 @@
return retval;
}
-int ipvs_del_entry(clusternode_t node, u_int cvip, u_short port,
+int
+ipvs_add_entry(struct ipvs_dirinfo *info, u_short port,
+ u_int rip, int weight, u_short protocol, char *lvs_sched)
+{
+ int retval;
+ extern void local_ipvs_add_entry(struct ipvs_dirinfo *, int *, u_short,
+ u_int, int, u_short, char *);
+
+ if (info->dnode == this_node)
+ local_ipvs_add_entry(info, &retval, port, rip, weight,
+ protocol, lvs_sched);
+ else
+ (void) RIPVS_ADD_ENTRY(info->dnode, &retval, info->daddr, port, rip,
+ weight, protocol, lvs_sched,
+ strlen(lvs_sched));
+ return retval;
+}
+
+int
+_ipvs_del_entry(clusternode_t node, u_int cvip, u_short port,
u_int rip, u_short protocol)
{
int retval;
+ BUG_ON(node == this_node);
+
if (node == this_node) {
/* This node is the director node for the CVIP */
/* Should we handle the return of this function */
@@ -106,3 +131,20 @@
return retval;
}
+
+int
+ipvs_del_entry(struct ipvs_dirinfo *info, u_short port,
+ u_int rip, u_short protocol)
+{
+ int retval;
+ extern void local_ipvs_del_entry(struct ipvs_dirinfo *, int *, u_short,
+ u_int, u_short);
+
+ if (info->dnode == this_node)
+ local_ipvs_del_entry(info, &retval, port, rip, protocol);
+ else
+ (void) RIPVS_DEL_ENTRY(info->dnode, &retval, info->daddr,
+ port, rip, protocol);
+
+ return retval;
+}
Index: ipvs_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/net/ipvs_svr.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- ipvs_svr.c 19 Feb 2009 08:01:02 -0000 1.25
+++ ipvs_svr.c 27 Oct 2009 03:18:29 -0000 1.26
@@ -51,24 +51,19 @@
extern int service_add(u_int32_t, u_int16_t, u_int32_t, int, u_int16_t, char *);
extern int service_del(u_int32_t, u_int16_t, u_int32_t, u_int16_t);
-extern int ipvs_add_entry(clusternode_t, u_int, u_short, u_int, int, u_short, char *);
-extern int ipvs_del_entry(clusternode_t, u_int, u_short, u_int, u_short);
+extern int _ipvs_add_entry(clusternode_t, u_int, u_short, u_int, int, u_short, char *);
+extern int _ipvs_del_entry(clusternode_t, u_int, u_short, u_int, u_short);
+extern int ipvs_add_entry(struct ipvs_dirinfo *, u_short, u_int, int, u_short, char *);
+extern int ipvs_del_entry(struct ipvs_dirinfo *, u_short, u_int, u_short);
static LIST_HEAD(ipvs_directors);
-static DECLARE_RWSEM(cvip_sem);
+static DECLARE_RWSEM(ipvs_directors_sem);
static LIST_HEAD(h_tcppwlist);
static LIST_HEAD(h_udppwlist);
/* port weight list semaphore. One to protect both the above list */
static DECLARE_RWSEM(pwlist_sem);
-struct ipvs_dirinfo {
- struct list_head list;
- clusternode_t dnode;
- __u32 daddr;
- clusternode_t *p_directornode;
-};
-
struct portweight_list {
struct list_head list;
u_short start_port;
@@ -90,16 +85,20 @@
{
struct ipvs_dirinfo *info;
- info = (struct ipvs_dirinfo *)kzmalloc(sizeof(*info), GFP_KERNEL);
+ info = kmalloc(sizeof(*info), GFP_KERNEL);
if (!info)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&info->list);
+ atomic_set(&info->ipvs_refcnt, 1);
info->dnode = node;
info->daddr = ip;
- down_write(&cvip_sem);
- list_add(&(info->list), &(ipvs_directors));
- up_write(&cvip_sem);
+ info->p_directornode = NULL;
+ spin_lock_init(&info->ipvs_lock);
+
+ down_write(&ipvs_directors_sem);
+ list_add(&info->list, &ipvs_directors);
+ up_write(&ipvs_directors_sem);
return info;
}
@@ -109,31 +108,43 @@
{
struct ipvs_dirinfo *info;
- down_read(&cvip_sem);
+ down_read(&ipvs_directors_sem);
list_for_each_entry(info, &ipvs_directors, list) {
- if (info->daddr == ip) {
- up_read(&cvip_sem);
- return info;
- }
+ if (info->daddr != ip)
+ continue;
+ atomic_inc(&info->ipvs_refcnt);
+ up_read(&ipvs_directors_sem);
+ return info;
}
- up_read(&cvip_sem);
+ up_read(&ipvs_directors_sem);
return NULL;
}
-static inline struct ipvs_dirinfo *
-modify_director(
- __u32 ip,
- clusternode_t node)
+/*
+ * Delete the possible director node array w.r.t to this CVIP
+ */
+static inline void
+__del_directors(struct ipvs_dirinfo *info)
{
- struct ipvs_dirinfo *info;
-
- if ((info = get_director(ip)))
- info->dnode = node;
- else
- info = add_director(ip, node);
+ spin_lock(&info->ipvs_lock);
+ if (info->p_directornode == NULL) {
+ spin_unlock(&info->ipvs_lock);
+ return;
+ }
+ kfree(info->p_directornode);
+ info->p_directornode = NULL;
+ spin_unlock(&info->ipvs_lock);
+}
- return info;
+static void
+put_director(struct ipvs_dirinfo *info)
+{
+ if (!atomic_dec_and_test(&info->ipvs_refcnt))
+ return;
+ SSI_ASSERT(list_empty(&info->list));
+ __del_directors(info);
+ kfree(info);
}
static inline void
@@ -141,22 +152,42 @@
{
struct ipvs_dirinfo *info;
- /*
- * First delete the possible director node array w.r.t to this
- * CVIP
- */
- del_pos_directors(ip);
+ info = get_director(ip);
+ if (!info)
+ return;
+ down_write(&ipvs_directors_sem);
+ list_del_init(&(info->list));
+ up_write(&ipvs_directors_sem);
+ put_director(info);
+ put_director(info); /* active */
+}
- down_write(&cvip_sem);
- list_for_each_entry(info, &ipvs_directors, list) {
- if (info->daddr == ip) {
- list_del(&(info->list));
- kfree(info);
- break;
- }
+static void
+modify_director(
+ __u32 ip,
+ clusternode_t node)
+{
+ struct ipvs_dirinfo *info;
+#ifdef IPVS_SERVICE_RACE_FIX
+ static DECLARE_MUTEX(mutex);
+
+ down(&mutex);
+#endif
+ if ((info = get_director(ip))) {
+ info->dnode = node;
+ put_director(info);
+ } else {
+ /* Skip put_director() here.
+ * _del_cvip() does final put_director().
+ */
+ info = add_director(ip, node);
+ if (IS_ERR(info))
+ printk(KERN_ALERT "%s: adding director failed %ld\n",
+ __FUNCTION__, PTR_ERR(info));
}
- up_write(&cvip_sem);
- return;
+#ifdef IPVS_SERVICE_RACE_FIX
+ up(&mutex);
+#endif
}
void
@@ -165,17 +196,11 @@
u_int daddr,
clusternode_t dnode)
{
- struct ipvs_dirinfo *info;
-
if (dnode == CLUSTERNODE_INVAL) {
_del_cvip(daddr);
return;
}
-
- info = modify_director(daddr, dnode);
- if (IS_ERR(info))
- printk(KERN_ALERT "%s: adding director failed %ld\n",
- __FUNCTION__, PTR_ERR(info));
+ modify_director(daddr, dnode);
}
/*
@@ -184,7 +209,7 @@
*/
int
pdirector_update(
- u_int32_t cvip,
+ struct ipvs_dirinfo *info,
u_int16_t port,
u_int32_t rip,
int weight,
@@ -192,42 +217,39 @@
char *lvs_sched,
int action)
{
- struct ipvs_dirinfo *info;
int i, ret = 0;
clusternode_t dnode;
- down_read(&cvip_sem);
- list_for_each_entry(info, &ipvs_directors, list) {
- if (info->daddr != cvip)
- continue;
+ spin_lock(&info->ipvs_lock);
- for (i = 0; i < MAX_DIRECTORS; i++) {
- /* No possible director? */
- if (!info->p_directornode)
- break;
+ for (i = 0; i < MAX_DIRECTORS; i++) {
+ /* No possible director? */
+ /* Or lost race with del_pos_directors() */
+ if (!info->p_directornode)
+ break;
- dnode = info->p_directornode[i];
- if (!dnode || (dnode == this_node))
- continue;
+ dnode = info->p_directornode[i];
+ if (!dnode || (dnode == this_node))
+ continue;
- /* check if the node is up. Send only to
- * UP nodes. During secondary node boot
- * up node will be half up
- */
- if (clms_isnodedown(dnode))
- continue;
+ /* check if the node is up. Send only to
+ * UP nodes. During secondary node boot
+ * up node will be half up
+ */
+ if (clms_isnodedown(dnode))
+ continue;
- if (action) {
- ret = ipvs_add_entry(dnode, cvip, port, rip,
- weight, protocol, lvs_sched);
- } else {
- ret = ipvs_del_entry(dnode, cvip, port, rip,
- protocol);
- }
- }
- break;
+ spin_unlock(&info->ipvs_lock);
+
+ if (action) {
+ ret = _ipvs_add_entry(dnode, info->daddr, port, rip,
+ weight, protocol, lvs_sched);
+ } else
+ ret = _ipvs_del_entry(dnode, info->daddr, port, rip, protocol);
+ /* SSI_XXX: ignoring return value */
+ spin_lock(&info->ipvs_lock);
}
- up_read(&cvip_sem);
+ spin_unlock(&info->ipvs_lock);
return ret;
}
@@ -247,12 +269,12 @@
char lvs_sched[IP_VS_SCHEDNAME_MAXLEN];
#ifdef IPVS_SERVICE_RACE_FIX
- SSI_ASSERT(sched_len <= IP_VS_SCHEDNAME_MAXLEN);
-
+ SSI_ASSERT(sched_len < IP_VS_SCHEDNAME_MAXLEN);
+ if (sched_len >= IP_VS_SCHEDNAME_MAXLEN)
+ sched_len = IP_VS_SCHEDNAME_MAXLEN-1;
strlcpy(lvs_sched, sched, sched_len+1);
*rval = service_add(cvip, port, rip, weight, protocol, lvs_sched);
-
/* Only if there is an IPVS entry. */
if (!*rval || *rval == -EEXIST)
#else
@@ -263,15 +285,53 @@
/* XXX: Not sure if we need this. */
if (!rval)
#endif
- /* Inform all the possible director node regarding this */
- /* We should call this only from master only on master director node */
- if ((info = get_director(cvip)) && info->dnode == this_node)
- (void) pdirector_update(cvip, port, rip, weight, protocol,
- lvs_sched, 1);
+ /* Inform all the possible director nodes */
+ if ((info = get_director(cvip))) {
+ /* only from master director node */
+ if (info->dnode == this_node)
+ (void) pdirector_update(info, port, rip, weight,
+ protocol, lvs_sched, 1);
+ put_director(info);
+ }
return 0;
}
+/* ripvs_add_entry() without get_director() call */
+void
+local_ipvs_add_entry(
+ struct ipvs_dirinfo *info,
+ int *rval,
+ u_short port,
+ u_int rip,
+ int weight,
+ u_short protocol,
+ char *sched)
+{
+ char lvs_sched[IP_VS_SCHEDNAME_MAXLEN];
+
+#ifdef IPVS_SERVICE_RACE_FIX
+ SSI_ASSERT(strlen(sched) < IP_VS_SCHEDNAME_MAXLEN);
+ strlcpy(lvs_sched, sched, IP_VS_SCHEDNAME_MAXLEN);
+
+ *rval = service_add(info->daddr, port, rip, weight, protocol, lvs_sched);
+ /* Only if there is an IPVS entry. */
+ if (!*rval || *rval == -EEXIST)
+#else
+ strncpy(lvs_sched, sched, IP_VS_SCHEDNAME_MAXLEN);
+ lvs_sched[IP_VS_SCHEDNAME_MAXLEN-1] = '\0';
+
+ *rval = service_add(info->daddr, port, rip, weight, protocol, lvs_sched);
+ /* XXX: Not sure if we need this. */
+ if (!rval)
+#endif
+ /* Inform all the possible director nodes */
+ /* only from master director node */
+ if (info->dnode == this_node)
+ (void) pdirector_update(info, port, rip, weight,
+ protocol, lvs_sched, 1);
+}
+
int
ripvs_del_entry(
clusternode_t node,
@@ -289,14 +349,39 @@
#else
if (!rval)
#endif
- /* Inform all the possible director node regarding this */
- /* We should call this only from master only on master director node */
- if ((info = get_director(cvip)) && info->dnode == this_node)
- (void) pdirector_update(cvip, port, rip, 0, protocol, 0, 0);
+ /* Inform all the possible director nodes */
+ if ((info = get_director(cvip))) {
+ /* only from master director node */
+ if (info->dnode == this_node)
+ (void) pdirector_update(info, port, rip, 0, protocol, 0, 0);
+ put_director(info);
+ }
return 0;
}
+/* ripvs_del_entry() without get_director() call */
+void
+local_ipvs_del_entry(
+ struct ipvs_dirinfo *info,
+ int *rval,
+ u_short port,
+ u_int rip,
+ u_short protocol)
+{
+ *rval = service_del(info->daddr, port, rip, protocol);
+#ifdef IPVS_SERVICE_RACE_FIX
+ if (*rval != -EBADR)
+#else
+ if (!rval)
+#endif
+ /* Inform all the possible director nodes */
+ /* only from master director node */
+ if (info->dnode == this_node)
+ (void) pdirector_update(info, port, rip, 0, protocol, 0, 0);
+}
+
+
#ifdef CONFIG_PROC_FS
static int
lvsrouting_show(
@@ -362,13 +447,13 @@
seq_printf(m, "CVIP Address \tNode number\n");
- down_read(&cvip_sem);
+ down_read(&ipvs_directors_sem);
list_for_each_entry(info, &ipvs_directors, list) {
char tbuf[16];
sprintf(tbuf, "%u.%u.%u.%u", NIPQUAD(info->daddr));
seq_printf(m, "%-16s\t%u\n", tbuf, info->dnode);
}
- up_read(&cvip_sem);
+ up_read(&ipvs_directors_sem);
return 0;
}
@@ -497,20 +582,18 @@
{
struct ipvs_dirinfo *info;
- down_read(&cvip_sem);
+ down_read(&ipvs_directors_sem);
list_for_each_entry(info, &ipvs_directors, list) {
if (!is_configured(info->daddr))
continue;
-
if (is_addserv)
- (void) ipvs_add_entry(info->dnode, info->daddr, port,
- rip, weight, protocol, lvs_sched);
+ (void) ipvs_add_entry(info, port, rip, weight,
+ protocol, lvs_sched);
else
- (void) ipvs_del_entry(info->dnode, info->daddr, port,
- rip, protocol);
+ (void) ipvs_del_entry(info, port, rip, protocol);
+ /* SSI_XXX: ignoring return value */
}
- up_read(&cvip_sem);
-
+ up_read(&ipvs_directors_sem);
return 0;
}
@@ -538,14 +621,20 @@
*/
continue;
} else if (port >= tplist->start_port) {
+#ifndef IPVS_SERVICE_RACE_FIX
up_read(&pwlist_sem);
-#ifdef IPVS_SERVICE_RACE_FIX
- if (!pdesc)
+#else
+ if (!pdesc) {
+ up_read(&pwlist_sem);
return 0;
+ }
#endif
- pdesc->port = port;
pdesc->weight = tplist->weight;
strlcpy(pdesc->lvs_sched, tplist->lvs_sched, IP_VS_SCHEDNAME_MAXLEN);
+#ifdef IPVS_SERVICE_RACE_FIX
+ up_read(&pwlist_sem);
+#endif
+ pdesc->port = port;
return 0;
}
break;
@@ -563,6 +652,7 @@
{
struct ipvs_dirinfo *info;
struct port_desc pdesc;
+ int ret;
if (getport_desc(ntohs(port), protocol, &pdesc)) {
/*
@@ -587,9 +677,11 @@
/* listen on local address */
return 0;
}
-
- return ipvs_add_entry(info->dnode, bindip, port, rip,
+ ret = ipvs_add_entry(info, port, rip,
pdesc.weight, protocol, pdesc.lvs_sched);
+ put_director(info);
+
+ return ret;
}
int
@@ -600,6 +692,7 @@
u_short protocol)
{
struct ipvs_dirinfo *info;
+ int ret;
if (cvip == INADDR_ANY)
return inaddr_any_op(port, rip, 0, protocol, NULL, 0);
@@ -608,8 +701,10 @@
/* release on local address */
return 0;
}
+ ret = ipvs_del_entry(info, port, rip, protocol);
+ put_director(info);
- return ipvs_del_entry(info->dnode, cvip, port, rip, protocol);
+ return ret;
}
long
@@ -746,9 +841,6 @@
/* rearrange the values */
tplist3->end_port = tplist->end_port;
-#ifdef IPVS_SERVICE_RACE_FIX
- mb();
-#endif
tplist->end_port = tplist2->start_port-1;
tplist3->start_port = tplist2->end_port+1;
tplist3->weight = tplist->weight;
@@ -797,18 +889,12 @@
{
struct ipvs_dirinfo *info;
- down_write(&cvip_sem);
- list_for_each_entry(info, &ipvs_directors, list) {
- if (info->daddr == addr) {
- kfree(info->p_directornode);
- info->p_directornode = NULL;
- goto out;
- }
+ info = get_director(addr);
+ if (info) {
+ __del_directors(info);
+ put_director(info);
}
-out:
- up_write(&cvip_sem);
return 0;
-
}
long
@@ -816,58 +902,70 @@
__u32 addr,
clusternode_t dnode)
{
-
struct ipvs_dirinfo *info;
+ clusternode_t *dirnodes;
int i;
- down_write(&cvip_sem);
- list_for_each_entry(info, &ipvs_directors, list) {
- if (info->daddr != addr)
- continue;
- /*
- * No possible director node present yet.
- * Allocate the array and add
- */
- if (info->p_directornode == NULL) {
- info->p_directornode = (clusternode_t *)
- kzmalloc(MAX_DIRECTORS *
- sizeof(clusternode_t), GFP_KERNEL);
- if (!info->p_directornode) {
- up_write(&cvip_sem);
- return -ENOMEM;
- }
- }
+ info = get_director(addr);
+ if (!info)
+ return -EINVAL;
- for (i = 0; i < MAX_DIRECTORS; i++) {
- if (info->p_directornode[i] == dnode) {
- /* This is already added to the array */
- break;
- } else if (info->p_directornode[i] == 0) {
- /* Add it here */
- info->p_directornode[i] = dnode;
- break;
- }
- }
- up_write(&cvip_sem);
+ spin_lock(&info->ipvs_lock);
- /* Cannot add more that MAX_DIRECTORS */
- if (i == MAX_DIRECTORS)
+ /*
+ * No possible director node present yet.
+ * Allocate the array and add
+ */
+ if (!info->p_directornode) {
+ spin_unlock(&info->ipvs_lock);
+ dirnodes = kzmalloc(MAX_DIRECTORS * sizeof(*dirnodes), GFP_KERNEL);
+ if (!dirnodes)
return -ENOMEM;
+
+ spin_lock(&info->ipvs_lock);
+ if (info->p_directornode) /* lost race */
+ kfree(dirnodes);
else
- return 0;
+ info->p_directornode = dirnodes;
}
- up_write(&cvip_sem);
- return -EINVAL;
+
+ for (i = 0; i < MAX_DIRECTORS; i++) {
+ if (info->p_directornode[i] == dnode)
+ /* This is already added to the array */
+ break;
+ if (info->p_directornode[i] == 0) {
+ /* Add it here */
+ info->p_directornode[i] = dnode;
+ break;
+ }
+ }
+ spin_unlock(&info->ipvs_lock);
+
+ /* Cannot add more than MAX_DIRECTORS */
+ if (i == MAX_DIRECTORS)
+ return -ENOMEM;
+
+ return 0;
}
long
del_cvip(__u32 dipaddr)
{
+ struct ipvs_dirinfo *info;
+
/* Allow this to run only on cvip master director node */
- if (get_director(dipaddr)->dnode == this_node)
- return set_master_director(dipaddr, CLUSTERNODE_INVAL);
- else
- return -EINVAL;
+ if ((info = get_director(dipaddr))) {
+ if (info->dnode == this_node) {
+ put_director(info);
+ /* TODO: Could optimize away get_director() in path:
+ * set_master_director, ripvs_setdirector, modify_director,
+ * get_director.
+ */
+ return set_master_director(dipaddr, CLUSTERNODE_INVAL);
+ }
+ put_director(info);
+ }
+ return -EINVAL;
}
#ifdef IPVS_REGISTER_PORTS_FIX
@@ -986,24 +1084,23 @@
if (getport_desc(port, protocol, NULL))
continue;
- /* Unregister port for ha-lvs unconfigure_services() */
if (!weight) {
+ /* Unregister port for ha-lvs unconfigure_services() */
(void) inaddr_any_op(htons(port), rip, weight, protocol,
lvs_sched, 0);
continue;
}
- down_read(&cvip_sem);
+ down_read(&ipvs_directors_sem);
list_for_each_entry(info, &ipvs_directors, list) {
- if (!is_configured(info->daddr))
- continue;
- if (!is_bound(port, protocol, info->daddr))
+ if (!is_configured(info->daddr) ||
+ !is_bound(port, protocol, info->daddr))
continue;
- (void) ipvs_add_entry(info->dnode, info->daddr,
- htons(port), rip, weight, protocol,
- lvs_sched);
+
+ (void) ipvs_add_entry(info, htons(port), rip, weight,
+ protocol, lvs_sched);
}
- up_read(&cvip_sem);
+ up_read(&ipvs_directors_sem);
#else
if (!is_bound(port, protocol))
continue;
------------------------------------------------------------------------------
Come build with us! The BlackBerry(R) Developer Conference in SF, CA
is the only developer event you need to attend this year. Jumpstart your
developing skills, take BlackBerry mobile applications to market and stay
ahead of the curve. Join us from November 9 - 12, 2009. Register now!
http://p.sf.net/sfu/devconference