[SSI] openssi/kernel/cluster/ssi/net ipvs_svr.c,1.28,1.29

Update of /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/net
In directory sfp-cvs-1.v30.ch3.sourceforge.com:/tmp/cvs-serv7714/cluster/ssi/net

Modified Files:
      Tag: OPENSSI-FC
	ipvs_svr.c 
Log Message:
- ssi_inet_bind, ssi_inet_listen: remove extra inet_release() in error path.
- ssi_inet_bind, ssi_inet_listen, ssi_inet_release: fix debug messages printing IP port number in network byte order.
- implement lockless ipvs_directors list traversal.


Index: ipvs_svr.c
===================================================================
RCS file: /cvsroot/ssic-linux/openssi/kernel/cluster/ssi/net/ipvs_svr.c,v
retrieving revision 1.28
retrieving revision 1.29
diff -u -d -r1.28 -r1.29
--- ipvs_svr.c	15 Dec 2010 08:01:51 -0000	1.28
+++ ipvs_svr.c	18 Dec 2013 07:10:31 -0000	1.29
@@ -57,7 +57,7 @@
 extern int ipvs_del_entry(struct ipvs_dirinfo *, u_short, u_int, u_short);
 
 static LIST_HEAD(ipvs_directors);
-static DECLARE_RWSEM(ipvs_directors_sem);
+static DEFINE_SPINLOCK(ipvs_directors_lock);
 
 static LIST_HEAD(h_tcppwlist);
 static LIST_HEAD(h_udppwlist);
@@ -95,32 +95,32 @@
 	info->daddr = ip;
 	info->p_directornode = NULL;
 	spin_lock_init(&info->ipvs_lock);
+	INIT_RCU_HEAD(&info->ipvs_rcu);
 
-	down_write(&ipvs_directors_sem);
-	list_add(&info->list, &ipvs_directors);
-	up_write(&ipvs_directors_sem);
+	spin_lock(&ipvs_directors_lock);
+	list_add_rcu(&info->list, &ipvs_directors);
+	spin_unlock(&ipvs_directors_lock);
 
 	return info;
 }
 
-#define __get_director(info) atomic_inc(&info->ipvs_refcnt)
+#define __get_director(info) atomic_test_and_add(1, &info->ipvs_refcnt)
 
 struct ipvs_dirinfo *
 get_director(__u32 ip)
 {
 	struct ipvs_dirinfo *info;
 
-	down_read(&ipvs_directors_sem);
-	list_for_each_entry(info, &ipvs_directors, list) {
-		if (info->daddr != ip)
-			continue;
-		__get_director(info);
-		up_read(&ipvs_directors_sem);
-		return info;
+	rcu_read_lock();
+	list_for_each_entry_rcu(info, &ipvs_directors, list) {
+		if (info->daddr == ip &&
+		    __get_director(info))
+			goto found;
 	}
-	up_read(&ipvs_directors_sem);
-
-	return NULL;
+	info = NULL;
+found:
+	rcu_read_unlock();
+	return info;
 }
 
 /*
@@ -140,13 +140,23 @@
 }
 
 static void
+ipvs_dirinfo_rcu_free(struct rcu_head *head)
+{
+	struct ipvs_dirinfo *info =
+		container_of(head, struct ipvs_dirinfo, ipvs_rcu);
+	kfree(info);
+}
+
+static void
 put_director(struct ipvs_dirinfo *info)
 {
-	if (!atomic_dec_and_test(&info->ipvs_refcnt))
+	if (!atomic_dec_and_lock(&info->ipvs_refcnt, &ipvs_directors_lock))
 		return;
-	BUG_ON(!list_empty(&info->list));
+	list_del_rcu(&info->list);
+	spin_unlock(&ipvs_directors_lock);
+
 	__del_directors(info);
-	kfree(info);
+	call_rcu(&info->ipvs_rcu, &ipvs_dirinfo_rcu_free);
 }
 
 static inline void
@@ -157,9 +167,6 @@
 	info = get_director(ip);
 	if (!info)
 		return;
-	down_write(&ipvs_directors_sem);
-	list_del_init(&(info->list));
-	up_write(&ipvs_directors_sem);
 	put_director(info);
 	put_director(info); /* active */
 }
@@ -449,13 +456,13 @@
 
 	seq_printf(m, "CVIP Address    \tNode number\n");
 
-	down_read(&ipvs_directors_sem);
-	list_for_each_entry(info, &ipvs_directors, list) {
+	rcu_read_lock();
+	list_for_each_entry_rcu(info, &ipvs_directors, list) {
 		char tbuf[16];
 		sprintf(tbuf, "%u.%u.%u.%u", NIPQUAD(info->daddr));
 		seq_printf(m, "%-16s\t%u\n", tbuf, info->dnode);
 	}
-	up_read(&ipvs_directors_sem);
+	rcu_read_unlock();
 
 	return 0;
 }
@@ -582,16 +589,19 @@
 	char *lvs_sched,
 	int is_addserv)
 {
-	struct ipvs_dirinfo *info, *tmp = NULL;
+	struct ipvs_dirinfo *info;
+	struct list_head *pos = &ipvs_directors;
 
-	info = list_prepare_entry(tmp, &ipvs_directors, list);
+	rcu_read_lock();
+	list_for_each_continue_rcu(pos, &ipvs_directors) {
+		info = list_entry(pos, typeof(*info), list);
+		if (!__get_director(info))
+			continue;
+		rcu_read_unlock();
 
-	down_read(&ipvs_directors_sem);
-	list_for_each_entry_continue(info, &ipvs_directors, list) {
+		might_sleep();
 		if (!is_configured(info->daddr))
-			continue;
-		__get_director(info);
-		up_read(&ipvs_directors_sem);
+			goto next;
 
 		/* SSI_XXX: ignoring return value */
 		if (is_addserv) {
@@ -599,15 +609,11 @@
 							protocol, lvs_sched);
 		} else
 			(void) ipvs_del_entry(info, port, rip, protocol);
-
-		if (tmp)
-			put_director(tmp);
-		tmp = info;
-		down_read(&ipvs_directors_sem);
+next:
+		rcu_read_lock();
+		put_director(info);
 	}
-	up_read(&ipvs_directors_sem);
-	if (tmp)
-		put_director(tmp);
+	rcu_read_unlock();
 	return 0;
 }
 
@@ -1078,28 +1084,28 @@
 	u_short protocol,
 	char *lvs_sched)
 {
-	struct ipvs_dirinfo *info, *tmp = NULL;
+	struct ipvs_dirinfo *info;
+	struct list_head *pos = &ipvs_directors;
 
-	info = list_prepare_entry(tmp, &ipvs_directors, list);
+	rcu_read_lock();
+	list_for_each_continue_rcu(pos, &ipvs_directors) {
+		info = list_entry(pos, typeof(*info), list);
+		if (!__get_director(info))
+			continue;
+		rcu_read_unlock();
 
-	down_read(&ipvs_directors_sem);
-	list_for_each_entry_continue(info, &ipvs_directors, list) {
+		might_sleep();
 		if (!is_configured(info->daddr) ||
 		    !is_bound(port, protocol, info->daddr))
-			continue;
-		__get_director(info);
-		up_read(&ipvs_directors_sem);
+			goto next;
 
 		(void) ipvs_add_entry(info, port, rip, weight,
 					protocol, lvs_sched);
-		if (tmp)
-			put_director(tmp);
-		tmp = info;
-		down_read(&ipvs_directors_sem);
+next:
+		rcu_read_lock();
+		put_director(info);
 	}
-	up_read(&ipvs_directors_sem);
-	if (tmp)
-		put_director(tmp);
+	rcu_read_unlock();
 }
 #endif /* IPVS_REGISTER_PORTS_FIX */
 
@@ -1166,7 +1172,7 @@
 	u_int32_t cvip;  /* IPv4 cluster virutal IP */
 	u_int32_t rip;  /* real server  IP */
 	icsinfo_t nodeinfo;
-	int ret;
+	int ret, err;
 
 	/* ICS interface  address is the real server address*/
 	if (ics_geticsinfo(this_node, &nodeinfo))
@@ -1175,25 +1181,18 @@
 	rip = in_aton((char*)&nodeinfo);
 	cvip = inet->rcv_saddr;
 
-#ifdef IPVS_SERVICE_RACE_FIX
-	if ((ret = inet_listen(sock, backlog)))
-		return ret;
-#endif
-
 	ret = register_ipvs_service(cvip, inet->sport, rip, protocol);
 	if (ret) {
 		/*
 		 * Failed to register the service
 		 */
 #ifdef IPVS_SERVICE_RACE_FIX
-		(void) inet_release(sock);
 		printk(KERN_ALERT "%s: register IPVS real server %s:%d failed %d\n",
-				__FUNCTION__, (char *)&nodeinfo, inet->sport, ret);
+				__FUNCTION__, (char *)&nodeinfo, ntohs(inet->sport), ret);
 #endif
 		return ret;
 	}
 
-#ifndef IPVS_SERVICE_RACE_FIX
 	/*
 	 * Do the actual listen
 	 */
@@ -1202,9 +1201,12 @@
 		/*
 		 * IF listen fails delete the service and come out
 		 */
-		del_ipvs_entry(cvip, inet->sport, rip, protocol);
+		err = del_ipvs_entry(cvip, inet->sport, rip, protocol);
+		if (err)
+			printk(KERN_NOTICE "%s: delete IPVS real server %s:%d "
+				"failed %d\n", __FUNCTION__, (char *)&nodeinfo,
+				ntohs(inet->sport), err);
 	}
-#endif
 
 	return ret;
 }
@@ -1228,7 +1230,7 @@
 	u_int32_t cvip;  /* IPv4 cluster virutal IP */
 	u_int32_t rip;  /* real server  IP */
 	icsinfo_t nodeinfo;
-	int ret;
+	int ret, err;
 
 	/* ICS interface  address is the real server address*/
 	if (ics_geticsinfo(this_node, &nodeinfo))
@@ -1237,25 +1239,18 @@
 	rip = in_aton((char*)&nodeinfo);
 	cvip = addr->sin_addr.s_addr;
 
-#ifdef IPVS_SERVICE_RACE_FIX
-	if ((ret = inet_bind(sock, uaddr, addr_len)))
-		return ret;
-#endif
-
 	ret = register_ipvs_service(cvip, addr->sin_port, rip, protocol);
 	if (ret) {
 		/*
 		 * Failed to register the service
 		 */
 #ifdef IPVS_SERVICE_RACE_FIX
-		(void) inet_release(sock);
 		printk(KERN_ALERT "%s: register IPVS real server %s:%d failed %d\n",
-				__FUNCTION__, (char *)&nodeinfo, addr->sin_port, ret);
+				__FUNCTION__, (char *)&nodeinfo, ntohs(addr->sin_port), ret);
 #endif
 		return ret;
 	}
 
-#ifndef IPVS_SERVICE_RACE_FIX
 	/*
 	 * Do the actual  bind
 	 */
@@ -1264,9 +1259,12 @@
 		/*
 		 * IF bind fails delete the service and come out
 		 */
-		del_ipvs_entry(cvip, addr->sin_port, rip, protocol);
+		err = del_ipvs_entry(cvip, addr->sin_port, rip, protocol);
+		if (err)
+			printk(KERN_NOTICE "%s: delete IPVS real server %s:%d "
+				"failed %d\n", __FUNCTION__, (char *)&nodeinfo,
+				ntohs(addr->sin_port), err);
 	}
-#endif
 
 	return ret;
 }
@@ -1323,7 +1321,7 @@
 	if (ret) {
 		printk(KERN_NOTICE "%s: delete IPVS real server %s:%d "
 			"failed %d\n", __FUNCTION__, (char *)&nodeinfo,
-			port, ret);
+			ntohs(port), ret);
 	}
 #else
 	/* SSI_XXX: Not interested in the return value ?! */


------------------------------------------------------------------------------
Rapidly troubleshoot problems before they affect your business. Most IT 
organizations don't have a clear picture of how application performance 
affects their revenue. With AppDynamics, you get 100% visibility into your 
Java,.NET, & PHP application. Start your 15-day FREE TRIAL of AppDynamics Pro!
http://pubads.g.doubleclick.net/gampad/clk?id=84349831&iu=/4140/ostg.clktrk