Author: jschultz
Date: 2005-07-31 06:51:40 -0400 (Sun, 31 Jul 2005)
New Revision: 253
Added:
trunk/docs/flush/
trunk/docs/flush/docs/
trunk/docs/flush/docs/FL_connect.html
trunk/docs/flush/docs/FL_disconnect.html
trunk/docs/flush/docs/FL_error.html
trunk/docs/flush/docs/FL_flush.html
trunk/docs/flush/docs/FL_join.html
trunk/docs/flush/docs/FL_leave.html
trunk/docs/flush/docs/FL_more_msgs.html
trunk/docs/flush/docs/FL_multicast.html
trunk/docs/flush/docs/FL_poll.html
trunk/docs/flush/docs/FL_receive.html
trunk/docs/flush/docs/FL_version.html
trunk/docs/flush/docs/access.html
trunk/docs/flush/docs/error.html
trunk/docs/flush/docs/flush_spread_title.gif
trunk/docs/flush/docs/functions.html
trunk/docs/flush/docs/index.html
trunk/docs/flush/docs/messages.html
trunk/docs/flush/docs/misc.html
trunk/docs/flush/index.html
trunk/docs/flush/man/
trunk/docs/flush/man/FL_connect.3
trunk/docs/flush/man/FL_disconnect.3
trunk/docs/flush/man/FL_error.3
trunk/docs/flush/man/FL_flush.3
trunk/docs/flush/man/FL_join.3
trunk/docs/flush/man/FL_leave.3
trunk/docs/flush/man/FL_more_msgs.3
trunk/docs/flush/man/FL_multicast.3
trunk/docs/flush/man/FL_poll.3
trunk/docs/flush/man/FL_receive.3
trunk/docs/flush/man/FL_scat_multicast.3
trunk/docs/flush/man/FL_scat_receive.3
trunk/docs/flush/man/FL_scat_subgroupcast.3
trunk/docs/flush/man/FL_scat_unicast.3
trunk/docs/flush/man/FL_subgroupcast.3
trunk/docs/flush/man/FL_unicast.3
trunk/docs/flush/man/FL_version.3
trunk/flush/
trunk/flush/Makefile.in
trunk/flush/configure.in
trunk/flush/fl.c
trunk/flush/fl_p.h
trunk/flush/fl_time_memb.c
trunk/flush/scatp.c
trunk/flush/scatp.h
trunk/flush/sp_time_memb.c
trunk/flush/stats.c
trunk/flush/stats.h
trunk/flush/user.c
trunk/include/fl.h
Modified:
trunk/daemon/groups.c
Log:
Added in flush code (still needs to conform to new Spread semantics) as best I thought appropriate. Minor comments added to groups.c.
Modified: trunk/daemon/groups.c
===================================================================
--- trunk/daemon/groups.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/daemon/groups.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -2280,13 +2280,13 @@
litend_ptr = litbox_ptr + 1;
}
- /* NOTE: this code assumes that grp->mboxes contains no duplicates */
+ /* NOTE: the following code assumes that grp->mboxes contains no duplicates */
if (num_groups == 1) { /* no need to do extra copy work if only one group -> just use litbox 'array' directly */
break;
}
- if (num_mbox != 0) {
+ if (num_mbox != 0) { /* mboxes contains some entries already */
bigend_ptr = mboxes + num_mbox;
for (; litbox_ptr != litend_ptr; ++litbox_ptr)
@@ -2314,7 +2314,7 @@
bigbox_ptr = litbox_ptr;
bigend_ptr = litend_ptr;
- } else {
+ } else { /* otherwise use unique entries we've built in mboxes */
bigbox_ptr = mboxes;
bigend_ptr = mboxes + num_mbox;
}
Added: trunk/docs/flush/docs/FL_connect.html
===================================================================
--- trunk/docs/flush/docs/FL_connect.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_connect.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,166 @@
+<HTML><HEAD><TITLE>Manpage of FL_connect</TITLE>
+</HEAD>
+
+<BODY bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_connect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_connect - connect an application to a Spread daemon using Flush Spread semantics.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_connect(const char *</B><I>daemon_name</I><B>, const char *</B><I>user_name</I><B>, int </B><I>priority</I><B>, int </B><I>group_membership</I><B>, mailbox *</B><I>mbox</I><B>, char *</B><I>private_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_connect</B>
+
+is the initial call an application must make to establish a
+connection with a Spread daemon. All other Flush Spread calls
+refer to a valid
+<I>mbox </I>
+
+returned by this function.
+<P>
+The
+<I>daemon_name</I>
+
+is the name of the Spread daemon with which to connect. It should be a
+string in one of the following forms:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>4803</B>
+
+<DD>
+This will connect to the Spread daemon on the local
+machine running on port 4803. This form cannot be
+used to connect to a Windows95/NT machine.</p>
+
+<DT><B>4803@localhost</B>
+
+<DD>
+This will also connect to the Spread daemon
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.</p>
+
+<DT><B>[email protected]</B> or <B>[email protected]</B>
+
+<DD>
+This will connect to the machine identified by either
+domain name or IP address at the specified port.</p>
+</DL></DL>
+
+The
+<I>user_name </I>
+
+is the name this connection would like to be known as. It must be
+unique on the machine running the spread daemon. The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+<P>
+<I>priority</I>
+
+is a boolean (non-zero true, zero false) for whether this connection
+will be a "priority" connection or not. Currently this has no effect.
+<P>
+<I>group_membership</I>
+
+is a boolean for whether this connection will receive group membership
+messages or not. Usually for Flush Spread applications this parameter
+should be true. If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+<P>
+The
+<I>mbox</I>
+
+should be a pointer to a mailbox variable. After the
+<B>FL_connect</B>
+
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+<P>
+The
+<I>private_name</I>
+
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters. After the call returns it will contain the
+private group name of this connection. These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection. No other applications
+can join this special group.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns
+<B>ACCEPT_SESSION </B>
+
+on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+
+<DT><B>ILLEGAL_SPREAD</B>
+<DD>
+The
+<I>daemon_name</I>
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number. </p>
+
+<DT><B>COULD_NOT_CONNECT</B>
+
+<DD>
+Lower level socket calls failed to allow a connection to the
+specified spread daemon right now.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to establish the connection errors occured
+and the setup could not be completed.</p>
+
+<DT><B>REJECT_VERSION</B>
+
+<DD>
+The daemon and/or libraries have a version mismatch.</p>
+
+<DT><B>REJECT_NO_NAME</B>
+
+<DD>
+No user name was provided.</p>
+
+<DT><B>REJECT_ILLEGAL_NAME</B>
+
+<DD>
+Name provided violated some requirement (length or used an illegal character).</p>
+
+<DT><B>REJECT_NOT_UNIQUE</B>
+
+<DD>
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+<P>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_disconnect.html
===================================================================
--- trunk/docs/flush/docs/FL_disconnect.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_disconnect.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,59 @@
+<HTML><HEAD><TITLE>Manpage of FL_disconnect</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_disconnect</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_disconnect - destroy a Flush Spread connection between an application and a Spread daemon.
+
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_disconnect(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_disconnect</B>
+
+should be called when the application is finished
+with a connection to a Spread daemon. The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+<P>
+The
+<I>mbox </I>
+
+should be for the connection you wish to close.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or
+<B>ILLEGAL_SESSION</B>
+
+when the
+<I>mbox</I>
+
+given does not represent a valid connection.
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_error.html
===================================================================
--- trunk/docs/flush/docs/FL_error.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_error.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<HTML><HEAD><TITLE>Manpage of FL_error</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_error</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_error - Flush Spread error string reporting
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>void FL_error(int </B><I>error_code</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_error</B>
+
+prints an error string based on the Flush Spread or Spread error code
+<I>error_code</I>.
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_flush.html
===================================================================
--- trunk/docs/flush/docs/FL_flush.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_flush.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,136 @@
+<HTML><HEAD><TITLE>Manpage of FL_flush</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_flush</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_flush - flush a group in response to receiving a flush request message.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_flush(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_flush</B>
+
+flushes the group named
+<I>group_name</I>
+
+in which the connection represented by
+<I>mbox</I>
+
+has received a flush request message. A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have "gone away" (meaning
+that they are no longer in the group with this connection). However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group. Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+<P>
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group. For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+<P>
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow. There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back. In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+<P>
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group. However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset). Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+<P>
+Anyways, when the application is ready it can flush the group. Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group. If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to flush was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A connection may flush a group only once in response to each flush
+request message delivered in that group. If you violate this rule you
+will get an ILLEGAL_STATE error.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the flush could not be
+initiated.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_join.html
===================================================================
--- trunk/docs/flush/docs/FL_join.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_join.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,77 @@
+<HTML><HEAD><TITLE>Manpage of FL_join</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_join</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_join - join a connection to a Flush Spread group
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_join(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_join</B>
+
+joins the connection represented by
+<I>mbox</I>
+
+to a group with the name
+<I>group_name</I>.
+
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group. A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to join was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.</p>
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the join could not be
+initiated.</p>
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_leave.html
===================================================================
--- trunk/docs/flush/docs/FL_leave.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_leave.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,82 @@
+<HTML><HEAD><TITLE>Manpage of FL_leave</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_leave</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_leave - remove a connection from a Flush Spread group
+<A NAME="lbAC"> </A>
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_leave(mailbox </B><I>mbox</I><B>, const char *</B><I>group_name</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_leave</B>
+
+removes the connection represented by
+<I>mbox</I>
+
+from a Flush Spread group with the name
+<I>group_name</I>.
+
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership. Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+<H2>RETURN VALUES</H2>
+
+Returns 0 on success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.</p>
+
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal. Usually because it is not active.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication
+and the leave could not be initiated.
+</DL>
+
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_more_msgs.html
===================================================================
--- trunk/docs/flush/docs/FL_more_msgs.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_more_msgs.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,79 @@
+<HTML><HEAD><TITLE>Manpage of FL_more_msgs</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_more_msgs</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_more_msgs - returns the number of complete messages buffered and
+ready to be received on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_more_msgs(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_more_msgs</B>
+
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+<I>mbox</I>.
+
+<P>
+<B>NOTE</B>,
+
+that this function
+<B>CANNOT </B>
+
+be used as an I/O polling function to check if a receive call should
+be done. If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+<P>
+This function is merely a helper function to re-get the current status
+of the
+<I>more_messes</I>
+
+parameter returned from a call to
+<B>FL_receive</B>.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( < 0):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_multicast.html
===================================================================
--- trunk/docs/flush/docs/FL_multicast.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_multicast.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,242 @@
+<HTML><HEAD><TITLE>Manpage of FL_multicast</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_multicast</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast - multicast messages to subsets of Flush Spread groups.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, const char *</B><I>recvr_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_subgroupcast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int </B><I>num_recvrs</I><B>, const char </B><I>recvr_names</I><B>[][MAX_GROUP_NAME], int16 </B><I>mess_type</I><B>, const char *</B><I>scat</I><B>);</B>
+
+<P>
+<B>int FL_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, int </B><I>mess_len</I><B>, const char *</B><I>mess</I><B>);</B>
+
+<P>
+<B>int FL_scat_unicast(mailbox </B><I>mbox</I><B>, service </B><I>serv_type</I><B>, const char *</B><I>group_name</I><B>, int16 </B><I>mess_type</I><B>, const scatter *</B><I>scat</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_multicast</B>
+
+and its variants all multicast a message from the connection represented by
+<I>mbox</I>
+
+to a subset of the membership of the Flush Spread group named
+<I>group_name</I>.
+
+<P>
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+<P>
+In order to multicast a message to a group, the destination group
+<I>group_name</I>
+
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+<P>
+The
+<I>serv_type</I>
+
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+<P>
+<DL COMPACT><DT><DD>
+<B>UNRELIABLE_MESS</B>
+
+<BR>
+
+
+<B>RELIABLE_MESS </B>
+
+<BR>
+
+
+<B>FIFO_MESS</B>
+
+<BR>
+
+
+<B>CAUSAL_MESS</B>
+
+<BR>
+
+
+<B>AGREED_MESS</B>
+
+<BR>
+
+
+<B>SAFE_MESS</B>
+
+</DL>
+
+<P>
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired. Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+<P>
+The string
+<I>group_name</I>
+
+indicates the name of the Flush Spread group to which this message
+should be sent. This group can be either a regular or a private Flush
+Spread group, as dicussed above. The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message. For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+<I>recvr_name</I>.
+
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+<I>num_recvrs</I>
+
+and a doubly scripted array of the members' private group names in the
+variable
+<I>recvr_names</I>,
+
+where duplicate names are tolerated. For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group. If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+<P>
+The
+<I>mess_type</I>
+
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE. If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned. The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message. This value
+<B>WILL</B>
+
+be endian corrected upon returning.
+<P>
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent. The
+<I>mess_len</I>
+
+field gives the message length in bytes. While the
+<I>mess</I>
+
+field is a pointer to the buffer containing the message. For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+<I>scat_mess</I>,
+
+to a scatter structure, which is similiar to an iovec. This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather. However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS. If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes sent on success or one of the following
+errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+was illegal, usually because it is no longer active.</p>
+
+<DT><B>ILLEGAL_GROUP</B>
+
+<DD>
+The
+<I>group_name</I>
+
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length > MAX_GROUP_NAME. This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.</p>
+
+<DT><B>ILLEGAL_RECEIVERS</B>
+
+<DD>
+A unicast or subgroupcast specified private group names that weren't
+members of
+<I>group_name</I>'s
+
+most recent view/membership.</p>
+
+<DT><B>ILLEGAL_STATE</B>
+
+<DD>
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.</p>
+
+<DT><B>ILLEGAL_SERVICE</B>
+
+<DD>
+If the service type was using bits reserved by Flush Spread or Spread.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter, such as a negative array size was passed.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+Errors occurred during communication and the multicast could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_poll.html
===================================================================
--- trunk/docs/flush/docs/FL_poll.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_poll.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,68 @@
+<HTML><HEAD><TITLE>Manpage of FL_poll</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_poll</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_poll - returns the amount, in bytes, of activity on a connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_poll(mailbox </B><I>mbox</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_poll</B>
+
+is a way to poll the connection represented by
+<I>mbox</I>
+
+to see if there is any activity on that connection.
+<P>
+<B>NOTE</B>,
+
+however that activity does
+<B>NOT</B>
+
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read. If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+<H2>RETURN VALUES</H2>
+
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+is illegal, usually because it is not active.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_receive.html
===================================================================
--- trunk/docs/flush/docs/FL_receive.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_receive.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,522 @@
+<HTML><HEAD><TITLE>Manpage of FL_receive</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<BODY>
+<H1>FL_receive</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+Fl_receive, FL_scat_receive - receive a message on a Flush Spread connection.
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>int FL_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, int </B><I>max_mess_len</I><B>, char *</B><I>mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<P>
+<B>int FL_scat_receive(mailbox </B><I>mbox</I><B>, service *</B><I>serv_type</I><B>, char </B><I>sender</I><B>[MAX_GROUP_NAME], int </B><I>max_groups</I><B>, int *</B><I>num_groups</I><B>, char </B><I>groups</I><B>[][MAX_GROUP_NAME], int16 *</B><I>mess_type</I><B>, int *</B><I>endian_mismatch</I><B>, scatter *</B><I>scat_mess</I><B>, int *</B><I>more_messes</I><B>);</B>
+
+<H2>DESCRIPTION</H2>
+
+<B>FL_receive</B>
+
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox. A call to
+<B>FL_receive</B>
+
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.
+<P>
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc). The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+<P>
+Input Parameters:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><I>mbox</I>
+
+<DD>
+Represents the connection on which to receive a message.</p>
+<DT><I>serv_type</I>
+
+<DD>
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on. This bit field should be zeroed out before each call if
+no special receive service is to be requested.</p>
+<DT><I>sender</I>
+
+<DD>
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.</p>
+<DT><I>max_groups</I>
+
+<DD>
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.</p>
+<DT><I>num_groups</I>
+
+<DD>
+A pointer to an int.</p>
+<DT><I>groups</I>
+
+<DD>
+An array of group names containing storage for at least
+<I>max_groups</I>
+
+* MAX_GROUP_NAME characters.</p>
+
+<DT><I>mess_type</I>
+
+<DD>
+A pointer to an int16.</p>
+
+<DT><I>endian_mismatch</I>
+
+<DD>
+A pointer to an int.</p>
+
+<DT><I>max_mess_len</I>
+
+<DD>
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.</p>
+
+<DT><I>mess</I>
+
+<DD>
+A pointer to a buffer containing storage for at least
+<I>max_mess_len</I>
+
+bytes.</p>
+
+<DT><I>scat_mess</I>
+
+<DD>
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+</DL></DL>
+
+<P>
+Receive Semantics:
+<P>
+Normally, a call to receive will block if no messages are immediately
+available.
+<P>
+Normally, when calling a receive function if a user's buffer
+(<I>groups</I>,
+
+<I>mess</I>,
+
+or
+<I>scat_mess</I>)
+
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned. In
+this case, the
+<I>serv_type</I>,
+
+and
+<I>mess_type</I>
+
+of the message are filled in from the offending message and the
+parameters
+<I>num_groups</I>,
+
+and
+<I>endian_mismatch</I>
+
+reflect information about the necessary buffers' sizes. If
+<I>max_groups</I>
+
+was big enough, then
+*<I>num_groups</I>
+
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the
+<I>max_groups</I>
+
+was less than 7 and 7 group names can be received). If
+<I>max_mess_len</I>
+
+or
+<I>scat_mess</I>
+
+was big enough then
+*<I>endian_mismatch</I>
+
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received). The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+<P>
+The parameter
+<I>serv_type</I>
+
+allows the applications to request different receive services that
+affect the normal semantics of receive. Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+<P>
+The DONT_BLOCK service makes a receive call non-blocking. With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available. Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+<P>
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data. In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection. Also,
+*<I>num_groups</I>
+
+will still be set to the negative size of how many names were
+available (if
+<I>groups</I>
+
+wasn't big enough), but
+*<I>endian_mismatch</I>
+
+will not reflect the size of the data that could have been received (if
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not. Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was. This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+<P>
+Output Parameters:
+<P>
+Upon a successful return from receive,
+*<I>serv_type</I>
+
+will be filled out with the message type that was just received. The
+specific type of message received can be tested using the different
+message and membership type access macros. The rest of the
+parameters' meanings differ depending on the
+<I>serv_type</I>.
+
+<P>
+Regular Messages:
+<P>
+If the
+<I>serv_type</I>
+
+is a REG_MESSAGE (i.e. a data message) then:
+<P>
+The parameter
+<I>sender</I>
+
+will be filled with the private group name of the sending connection.
+<P>
+The parameter
+<I>num_groups</I>
+
+will be set to the number of group names filled into
+<I>groups</I>.
+
+<P>
+The
+<I>groups</I>
+
+array will contain the group to which this message was sent. If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+groups[*<I>num_groups</I>-1])
+
+will be the regular group to which this message was sent.
+<P>
+If DROP_RECV is being used, and
+<I>groups</I>
+
+is too small, then as many names as can fit will be filled in as above
+described. If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+groups[<I>max_groups</I>-1])
+
+will be the destination group for the message. If
+<I>max_groups</I>
+
+is zero then even the destination group is not reported.
+<P>
+The parameter
+*<I>mess_type</I>
+
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer. This value is endian corrected upon returning.
+<P>
+The parameter
+*<I>endian_mismatch</I>
+
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+<P>
+The actual message body being received will be filled into either the
+<I>mess</I>
+
+or
+<I>scat_mess</I>
+
+parameter.
+<P>
+Flush Request Message:
+<P>
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush).
+<P>
+The
+<I>sender</I>
+
+will be filled in with the name of the group this connection needs
+to flush. All the other fields are set to empty values.
+<P>
+Membership Messages:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+<P>
+The
+<I>sender</I>
+
+will be filled with the name of the group for which the membership
+change is occuring.
+<P>
+The
+<I>mess_type</I>
+
+field will be set to the index of this process in the
+<I>groups</I>
+
+array (see below).
+<P>
+The
+<I>endian_mismatch</I>
+
+field will be set to 0 since there are no endian issues with regular memberships.
+<P>
+The
+<I>groups</I>
+
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group. The
+<I>num_groups</I>
+
+field will be set to the number of members in the group in the
+<B>new</B>
+
+membership (i.e. after the change occured). Correspondingly, the
+<I>groups</I>
+
+array will be filled in with the private group names of all members of this
+group in the
+<B>new</B>
+
+membership. This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+<P>
+The second set of information is stored in the message body. The data
+buffer will include the following fields:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>group_id vid;</B>
+
+<DD>
+<BR>
+
+<DT><B>int num_members;</B>
+
+<DD>
+<BR>
+
+<DT><B>char dgroups[][MAX_GROUP_NAME];</B>
+
+<DD>
+</DL></DL>
+
+<P>
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names. The content
+of the dgroups array is dependent upon the type of the membership
+change:
+<DL COMPACT><DT><DD>
+<DL COMPACT>
+<DT><B>CAUSED_BY_JOIN:</B>
+
+<DD>
+dgroups contains the private group of the joining process.</p>
+<DT><B>CAUSED_BY_LEAVE:</B>
+
+<DD>
+dgroups contains the private group of the leaving process.</p>
+<DT><B>CAUSED_BY_DISCONNECT:</B>
+
+<DD>
+dgroups contains the private group of the disconnecting process.</p>
+<DT><B>CAUSED_BY_NETWORK:</B>
+
+<DD>
+dgroups contains the group names of the members of the new membership who came
+with
+<B>this</B>
+
+application to the new view/membership. Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change. Note, that a member can both leave and be added in
+a
+<B>single</B>
+
+NETWORK change.
+</DL></DL>
+
+<P>
+Transitional Message:
+<P>
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership. Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered. Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+<P>
+The
+<I>sender</I>
+
+will be filled in with the name of the group for which the membership
+change is occurring. All the other parameters will be set to empty values.
+<P>
+Self-Leave Message:
+<P>
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message. The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+<P>
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+<P>
+For self-leave messages the
+<I>sender</I>
+
+field will be filled in with the name of the group this connection is
+leaving. All the other fields are set to empty values.
+<P>
+<H2>RETURN VALUES</H2>
+
+Returns the size of the data portion of the message received on
+success or one of the following errors ( < 0 ):
+<DL COMPACT>
+<DT><B>ILLEGAL_SESSION</B>
+
+<DD>
+The connection represented by
+<I>mbox</I>
+
+was illegal, usually because it is not active.</p>
+
+<DT><B>ILLEGAL_PARAM</B>
+
+<DD>
+An illegal parameter was passed, like a negative array size.</p>
+
+<DT><B>ILLEGAL_MESSAGE</B>
+
+<DD>
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.</p>
+
+<DT><B>WOULD_BLOCK</B>
+
+<DD>
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.</p>
+
+<DT><B>GROUPS_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a groups array that was too small.
+*<I>num_groups</I>
+
+is filled in with the negative number of names that could have been
+returned. Note, that the msg buffer could also be too short, and this
+should be determined by examining
+<I>endian_mismatch</I>
+
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).</p>
+
+<DT><B>BUFFER_TOO_SHORT</B>
+
+<DD>
+The receive call was made with a msg buffer that was too small.
+*<I>endian_mismatch</I>
+
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used. If it is used then this error code
+actually represents success and
+<I>endian_mismatch</I>
+
+has the normal meaning (see above). Note, that the groups buffer
+could also be too short, this should be determined by examining
+<I>num_groups</I>
+
+as described above.</p>
+
+<DT><B>CONNECTION_CLOSED</B>
+
+<DD>
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+</DL>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/FL_version.html
===================================================================
--- trunk/docs/flush/docs/FL_version.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/FL_version.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,44 @@
+<HTML><HEAD><TITLE>Manpage of FL_version</TITLE>
+</HEAD>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<H1>FL_version</H1>
+Section: User Manuals (3)<BR>Updated: Dec 2000<BR>
+<HR>
+
+<H2>NAME</H2>
+
+FL_version - returns library version information.
+<A NAME="lbAC"> </A>
+<H2>SYNOPSIS</H2>
+
+<B>#include <fl.h></B>
+
+<P>
+<B>void FL_version(int *</B><I>major_ver</I><B>, int *</B><I>minor_ver</I><B>, int *</B><I>patch_ver</I><B>);</B>
+
+<A NAME="lbAD"> </A>
+<H2>DESCRIPTION</H2>
+
+<B>FL_version</B>
+
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+<A NAME="lbAE"> </A>
+<H2>AUTHOR</H2>
+
+John Schultz <<A HREF="mailto:[email protected]">[email protected]</A>>
+
+<!--#include virtual="/includes/footer" -->
+
+</BODY>
+</HTML>
Added: trunk/docs/flush/docs/access.html
===================================================================
--- trunk/docs/flush/docs/access.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/access.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,42 @@
+<html>
+<head><title>Flush Documentation: Message Type Access Macros</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Type Access Macros</em></h2>
+
+Flush Spread uses the same message type access macros as Spread. In addition to these
+<a href="http://www.spread.org/docs/access.html">common macros</a>, Flush Spread also
+adds the following message type access macros:
+
+<pre>
+#define Is_flush_req_mess(serv_type)
+#define Is_subgroup_mess(serv_type)
+</pre>
+
+<ol>
+<li><tt>Is_flush_req_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a flush request message or not.
+Flush request messages are an important part of the view synchrony GCS semantics, see <a
+href="FL_flush.html">FL_flush</a> for an in-depth discussion of them.</p>
+
+<li><tt>Is_subgroup_mess(serv_type):</tt></p>
+
+This macro checks the message type to determine if the message is a subgroup-multicast or
+not. A subgroup-multicast is a message that is only sent to a subset of a group, see
+<a href="FL_multicast.html">FL_subgroupcast</a> for a better description.</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/error.html
===================================================================
--- trunk/docs/flush/docs/error.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/error.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,80 @@
+<html>
+<head><title>Flush Documentation: Error Return Codes</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Error Return Codes</em></h2>
+
+Flush Spread uses the same error return codes as Spread. In addition to these
+<a href="http://www.spread.org/docs/error.html">common error codes</a>, Flush Spread
+also adds the following error codes:
+
+<pre>
+#define ILLEGAL_PARAM
+#define WOULD_BLOCK
+#define ILLEGAL_MESSAGE_TYPE
+#define ILLEGAL_STATE
+#define ILLEGAL_RECEIVERS
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_PARAM:</tt></p>
+
+An illegal parameter was passed to a call. Usually this is the result of specifying a
+negative array size.</p>
+
+<li><tt>WOULD_BLOCK:</tt></p>
+
+A receive call that requested the <tt>DONT_BLOCK</tt> <a href="messages.html">service</a> would
+have blocked.</p>
+
+<li><tt>ILLEGAL_MESSAGE_TYPE:</tt></p>
+
+A multicast call was made with an <a href="misc.html">illegal message type</a>.</p>
+
+<li><tt>ILLEGAL_STATE:</tt></p>
+
+A <a href="FL_multicast.html">FL_multicast</a> or <a href="FL_flush.html">FL_flush</a> call was made
+on a group while that group was in a prohibited state for that call.</p>
+
+<li><tt>ILLEGAL_RECEIVERS:</tt></p>
+
+A subgroup-multicast or unicast call specified receivers that were not currently
+members of the reference group.</p>
+</ol>
+
+Flush Spread also expands the meaning of the following Spread error codes:
+
+<pre>
+#define ILLEGAL_GROUP
+#define ILLEGAL_MESSAGE
+</pre>
+
+<ol>
+<li><tt>ILLEGAL_GROUP:</tt></p>
+
+In addition to representing illegal Spread group names, this error also represents that a
+group-specific call (i.e. - multicast, flush, join, leave) was made illegally while either
+not a member of the group (multicast, flush, leave) or while a member of the group
+(join).</p>
+
+<li><tt>ILLEGAL_MESSAGE:</tt></p>
+
+This error code in Flush Spread has the same meaning as it does in Spread. It means that
+either a send or receive message buffer or scatter was illegal in some manner. This can be
+returned if the size of a buffer is negative or if the number of scat elements is illegal
+(negative or greater than <tt><a href="misc.html">FL_MAX_SCATTER_ELEMENTS</a></tt>).</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/flush_spread_title.gif
===================================================================
(Binary files differ)
Property changes on: trunk/docs/flush/docs/flush_spread_title.gif
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: trunk/docs/flush/docs/functions.html
===================================================================
--- trunk/docs/flush/docs/functions.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/functions.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+<html>
+<head><title>Flush Documentation: Flush Spread C Function Interface</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Flush Spread C Function Interface</em></h2>
+
+<ul>
+<li><a href="FL_version.html">FL_version</a>
+<li><a href="FL_connect.html">FL_connect</a>
+<li><a href="FL_disconnect.html">FL_disconnect</a>
+<li><a href="FL_join.html">FL_join</a>
+<li><a href="FL_leave.html">FL_leave</a>
+<li><a href="FL_flush.html">FL_flush</a>
+<li><a href="FL_multicast.html">FL_unicast</a>
+<li><a href="FL_multicast.html">FL_scat_unicast</a>
+<li><a href="FL_multicast.html">FL_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_scat_subgroupcast</a>
+<li><a href="FL_multicast.html">FL_multicast</a>
+<li><a href="FL_multicast.html">FL_scat_multicast</a>
+<li><a href="FL_receive.html">FL_receive</a>
+<li><a href="FL_receive.html">FL_scat_receive</a>
+<li><a href="FL_more_msgs.html">FL_more_msgs</a>
+<li><a href="FL_poll.html">FL_poll</a>
+<li><a href="FL_error.html">FL_error</a>
+</ul>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
\ No newline at end of file
Added: trunk/docs/flush/docs/index.html
===================================================================
--- trunk/docs/flush/docs/index.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/index.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,115 @@
+<html>
+<head><title>Flush Documentation</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2>Documentation</h2>
+
+Flush Spread is an extension to the <a href="http://www.spread.org/">Spread Wide Area Group
+Communication System</a>. Flush Spread and Spread are extremely similiar group communication
+systems (GCSs), in the services that they provide, and in their general interface. Therefore,
+reading Spread's <a href="http://www.spread.org/docs/docspread.html">documentation</a> before
+reading Flush Spread's documentation is highly recommended to get a general grasp of group
+communication and to fill in any holes in this documentation :).</p>
+
+There are two main differences between Spread and Flush Spread:
+(1) Spread provides <a href="http://www.cs.jhu.edu/~yairamir/dcs-94.ps ">extended virtual
+synchrony</a> GCS semantics while Flush Spread provides view synchrony GCS semantics, and (2) Spread
+provides open-group semantics while Flush Spread only provides closed-group semantics.</p>
+
+<h3>Extended Virtual Synchrony (EVS) vs. View Synchrony Semantics (VS)</h3></p>
+
+The main difference between these two models is how they handle view/membership changes. In an EVS
+system, membership changes occur without a client's intervention, whereas in a VS system a client
+must give its permission before a new view/membership can be installed. Of course, in "real-life" a
+VS system cannot truly hold-off membership changes (as other clients can crash, network can
+partition, etc.), but what it can do is insulate a client from these changes and not allow new
+members (who either joined or merged) or their messages to be presented to the client without its
+permission.</p>
+
+In a VS system, when an underlying membership change occurs (another member joins/leaves/disconnects,
+network parititions or merges) the GCS generates a flush request message indicating that the current
+view is out-of-date and requesting permission to install a new view. The client is still allowed to
+send messages at this point, and they may be able to receive messages from surviving members. When
+the client is ready, it flushes the group, which gives the GCS permission to install a new view.
+After flushing, a client is not allowed to send any messages to the flushed group until they receive
+the new view of the group.</p>
+
+In a EVS system, when an underlying membership change occurs the GCS figures out the new membership
+and delivers a new view to the client in its normal stream of messages.</p>
+
+In both systems, using the safety guarantees of the different message types and the virtual synchrony
+property (i.e. transitional sets) the client can infer some information about what other members have
+seen without further communication.</p>
+
+Although the difference in these models seems slight, it can lead to quite astonishing differences
+in performance and ease of programming. In general, the EVS model is faster and scales better with
+respect to membership changes, especially for simple group changes such as joins and leaves. But
+this performance comes at a price, namely that the client has less control over what is transpiring
+and therefore in general it is harder to program algorithms under EVS than under VS. For example,
+under EVS the sender doesn't know which other clients might receive its messages until the message
+is delivered back to the sender. Also, membership changes can come fast and furious and your
+algorithm must be tolerant to every combination of such changes. Under VS, the client knows exactly
+which other clients <b>might</b> receive its message when it sends -- so the application doesn't
+have to worry about some arbitrary client receiving its messages and mis-interpreting them, for
+example. Also, membership changes only occur as fast as the applications want them to, although
+several underlying memberships may be collapsed into one. So they are generally easier to handle
+and track.</p>
+
+<h3>Open-Group vs. Closed-Group Semantics</h3></p>
+
+Open-group semantics allows clients to perform group specific calls, such as multicasts, without
+being a member of the group in question. Closed-group semantics requires a client to be a full
+fledged member of a group (see below) in order to perform group specific calls. In general, a
+closed-group GCS is much more picky than an open-group system about a client's state and the
+operations they perform with respect to any particular group.</p>
+
+Under Spread, a client can call group specific calls such as join, leave, and *cast (multicast and
+its variants) pretty much whenever they want regardless of their membership in that group. Also,
+under Spread the guarantees of multicast messages span differerent groups. For example, if a sender
+sends two FIFO messages to two different groups and a receiver receives both messages, they will
+always receive the first and then the second FIFO message.</p>
+
+Under Flush Spread, a client can make group specific calls such as join, leave, flush, and *cast
+only when in the proper membership state with respect to that group. For example, leave, flush, and
+*cast calls can only be made when the client is a full-fledged member of the group. A full-fledged
+member is a client that has joined a group, received a membership including them in the group and is
+not in the process of leaving the group. Join calls can only be made when the client is not a
+member of the group (either never joined it or whose most recent membership for that group was a
+self-leave) and not already in the process of joining that group.</p>
+
+Also under Flush Spread, *cast and flush calls (i.e. FL_flush) are restricted depending on the group
+membership state. Flush calls can only be made <b>ONCE</b> per view/membership in response to
+receiving a flush request message. Once a client flushes a group they are not allowed to *cast to
+that group until they receive the next view/membership for that group. Flush Spread also does not
+support cross group message semantics. For example, if a sender sends two FIFO messages to two
+different groups and a receiver receives both messages, they may receive the first and then the
+second <b>OR</b> vice-versa.</p>
+
+<hr>
+
+<h2>C/C++ Documentation</h2>
+
+The following documentation explains the user level C interface to Flush Spread as specified in the
+header file <a href="../src/fl.h">fl.h</a>. This documentation assumes a knowledge of the ideas,
+discussed above, of view synchronous group communication.
+
+<ol>
+<li><a href="messages.html">Message Types for Data and Membership Messages</a>
+<li><a href="access.html">Message Type Access Functions</a>
+<li><a href="error.html">Error Return Codes</a>
+<li><a href="misc.html">Miscellaneous Constants</a>
+<li><a href="functions.html">Flush Spread C Function Interface</a>
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/messages.html
===================================================================
--- trunk/docs/flush/docs/messages.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/messages.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,61 @@
+<html>
+<head><title>Flush Documentation: Message Types for Data and Membership Messages</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Message Types for Data and Membership Messages</em></h2>
+
+Flush Spread uses the same service and membership types as Spread. In addition to these
+<a href="http://www.spread.org/docs/messages.html">common types</a>, Flush Spread also
+adds the following service types:
+
+<pre>
+#define DONT_BLOCK
+#define FLUSH_REQ_MESS
+#define SUBGROUP_CAST
+</pre>
+
+<ol>
+
+<li><tt>DONT_BLOCK:</tt></p>
+
+This service type can be requested when making a receive call. If the receive call would block while
+receiving on the mailbox, the call will fail and return the error code <tt>WOULD_BLOCK</tt>.
+Currently, Spread client libraries (v3.14 and earlier) do not support non-blocking receives. This
+means that even if the caller sets the <tt>DONT_BLOCK</tt> service flag a call to receive may take
+longer than would normally be expected of a non-blocking I/O call. However, the call should never
+block permanently.</p>
+
+<li><tt>FLUSH_REQ_MESS:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the received
+message was a flush request message for the group contained in sender. A flush request message is
+generated by Flush Spread when the underlying membership of that group changes. To install the new
+membership and make progress, the receiver must eventually respond to this signal by flushing that
+group by calling <a href="FL_flush.html">FL_flush</a>. Receiving a flush request message is an
+important part of the view synchrony GCS semantics and has pretty drastic importance. See <a
+href="FL_receive.html">FL_receive</a> for a full description of the restrictions incurred by and
+ramifications of receiving a flush request. Note that a flush request is neither a membership message
+nor a regular message. See the <a href="access.html">message type access functions</a> for a macro to
+test if a message is a flush request message.</p>
+
+<li><tt>SUBGROUP_CAST:</tt></p>
+
+This message type can be set upon returning from a receive call. It indicates that the
+received message was only multicasted to a subset of a group. This message was only sent
+to the members in the groups array in the range [0, *num_groups-1). The group that this
+subgroup-multicast occurred in is contained in groups[*num_groups-1].</p>
+
+</ol>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/docs/misc.html
===================================================================
--- trunk/docs/flush/docs/misc.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/docs/misc.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,33 @@
+<html>
+<head><title>Flush Documentation: Miscellaneous Constants</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<a href="http://www.cnds.jhu.edu/research/group/flush_spread">
+<img src="flush_spread_title.gif" alt="FLUSH SPREAD" border=0>
+</a>
+
+<!--#include virtual="/includes/header-b" -->
+
+<h2><em>Miscellaneous Constants</em></h2>
+
+<pre>
+#define FL_MAX_SCATTER_ELEMENTS
+</pre>
+
+Flush Spread restricts the maximum number of scatter elements a caller is allowed to use
+in multicast calls to be less than or equal to <tt>FL_MAX_SCATTER_ELEMENTS</tt>. See Spread's
+<tt><a href="http://www.spread.org/docs/types.html">MAX_SCATTER_ELEMENTS</a></tt>.</p>
+
+<pre>
+#define FL_MIN_LEGAL_MESS_TYPE
+</pre>
+
+Flush Spread restricts the range of message types (int16) a caller is allowed to use in
+multicast calls to be greater than or equal to <tt>FL_MIN_LEGAL_MESS_TYPE</tt>.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/index.html
===================================================================
--- trunk/docs/flush/index.html 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/index.html 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,56 @@
+<html>
+<head><title>The Group Communication Project</title></head>
+
+<body bgcolor="#FFFFFF">
+
+<!--#include virtual="/includes/header-a" -->
+
+<img src="docs/flush_spread_title.gif" alt="FLUSH SPREAD">
+
+<!--#include virtual="/includes/header-b" -->
+
+</p>
+<font size= -1>
+<DD><A href="http://www.cs.jhu.edu/~yairamir">Yair Amir</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jschultz">John Schultz</A></DD>
+<DD><A href="http://www.cnds.jhu.edu/~jonathan">Jonathan Stanton</A></DD>
+</font>
+<HR>
+</p>
+
+<p>
+The Flush Spread project focuses on providing view synchrony group communication
+semantics built on top of Spread's extended virtual synchrony semantics. Currently, these
+semantics are provided by a client library that links with Spread's client library.
+</p>
+
+<p>
+Obviously you need Spread for Flush Spread to work ... so go check out
+<a href="http://www.spread.org">Spread</a>.
+</p>
+
+<p>
+Flush Spread includes a distribution of the Stdutil library, which is a toolkit of
+high-performance C data structures and utility functions. If you are interested, you
+can learn more about it <a href="http://www.cnds.jhu.edu/software/stdutil">here</a>.
+</p>
+
+<H3>Flush Spread Resources:</H3>
+
+<UL>
+<li>Read Flush Spread's <a href="README">README</a> file.
+
+<li>Read the Flush Spread <a href="FLUSH_LICENSE">license</a> or the Stdutil
+ <a href="stdutil/STDUTIL_LICENSE">license</a>.<br>
+
+<li>Read the Flush Spread <a href="docs/index.html">documentation</a>.<br>
+
+<li><a href="http://www.cnds.jhu.edu/download/flush_spread.tar.gz">Download</a> Flush Spread now!<br>
+</UL>
+
+E-mail <a href="mailto:[email protected]"><i>[email protected]</i></a>
+for more information.</p>
+
+<!--#include virtual="/includes/footer" -->
+
+</body></html>
Added: trunk/docs/flush/man/FL_connect.3
===================================================================
--- trunk/docs/flush/man/FL_connect.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_connect.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,105 @@
+.TH FL_connect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_connect \- connect an application to a Spread daemon using Flush Spread semantics.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_connect(const char *" daemon_name ", const char *" user_name ", int " priority ", int " group_membership ", mailbox *" mbox ", char *" private_name );
+.SH DESCRIPTION
+.B FL_connect
+is the initial call an application must make to establish a
+connection with a Spread daemon. All other Flush Spread calls
+refer to a valid
+.I mbox
+returned by this function.
+
+The
+.I daemon_name
+is the name of the Spread daemon with which to connect. It should be a
+string in one of the following forms:
+.RS
+.TP
+.B "4803"
+This will connect to the Spread daemon on the local
+machine running on port 4803. This form cannot be
+used to connect to a Windows95/NT machine.
+.TP
+.B "4803@localhost"
+This will also connect to the Spread daemon
+on the local machine running on port 4803.
+This form can be used on Windows95/NT machines.
+.TP
+.BR "[email protected] " or " [email protected]"
+This will connect to the machine identified by either
+domain name or IP address at the specified port.
+.RE
+
+The
+.I user_name
+is the name this connection would like to be known as. It must be
+unique on the machine running the spread daemon. The name can be an
+arbitrary length string with the same character restrictions as a
+group name (mainly it cannot contain the '#' character).
+
+.I priority
+is a boolean (non-zero true, zero false) for whether this connection
+will be a "priority" connection or not. Currently this has no effect.
+
+.I group_membership
+is a boolean for whether this connection will receive group membership
+messages or not. Usually for Flush Spread applications this parameter
+should be true. If your application doesn't need group membership
+messages, then Spread may provide the semantics that you need more
+efficiently than Flush Spread.
+
+The
+.I mbox
+should be a pointer to a mailbox variable. After the
+.B FL_connect
+call successfully returns, this variable will hold the valid mbox for
+this new connection.
+
+The
+.I private_name
+should be a pointer to a string big enough to hold at least
+MAX_GROUP_NAME characters. After the call returns it will contain the
+private group name of this connection. These group names are what are
+reported in membership messages and can be used to send unicast and
+subgroup-multicast messages to this connection. No other applications
+can join this special group.
+
+.SH "RETURN VALUES"
+Returns
+.B ACCEPT_SESSION
+on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SPREAD
+The
+.I daemon_name
+given to connect to was illegal for some reason. Usually because
+it was a unix socket on Windows95/NT, an improper format for a host
+or an illegal port number
+.TP
+.B COULD_NOT_CONNECT
+Lower level socket calls failed to allow a connection to the
+specified spread daemon right now.
+.TP
+.B CONNECTION_CLOSED
+During communication to establish the connection errors occured
+and the setup could not be completed.
+.TP
+.B REJECT_VERSION
+The daemon and/or libraries have a version mismatch.
+.TP
+.B REJECT_NO_NAME
+No user name was provided.
+.TP
+.B REJECT_ILLEGAL_NAME
+Name provided violated some requirement (length or used an illegal character)
+.TP
+.B REJECT_NOT_UNIQUE
+Name provided is not unique on this daemon. Recommended response is to try
+again with a different name.
+.SH AUTHOR
+John Schultz <[email protected]>
+
Added: trunk/docs/flush/man/FL_disconnect.3
===================================================================
--- trunk/docs/flush/man/FL_disconnect.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_disconnect.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,26 @@
+.TH FL_disconnect 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_disconnect \- destroy a Flush Spread connection between an application and a Spread daemon.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_disconnect(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_disconnect
+should be called when the application is finished
+with a connection to a Spread daemon. The application may have
+other connections still open to the daemon and may open a new
+connection after disconnecting. Any data that was available on
+the connection will be lost after the call returns.
+
+The
+.I mbox
+should be for the connection you wish to close.
+.SH "RETURN VALUES"
+Returns 0 on success or
+.B ILLEGAL_SESSION
+when the
+.I mbox
+given does not represent a valid connection.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_error.3
===================================================================
--- trunk/docs/flush/man/FL_error.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_error.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,13 @@
+.TH FL_error 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_error \- Flush Spread error string reporting
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_error(int " error_code ");"
+.SH DESCRIPTION
+.B FL_error
+prints an error string based on the Flush Spread or Spread error code
+.IR error_code .
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_flush.3
===================================================================
--- trunk/docs/flush/man/FL_flush.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_flush.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,94 @@
+.TH FL_flush 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_flush \- flush a group in response to receiving a flush request message.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_flush(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_flush
+flushes the group named
+.I group_name
+in which the connection represented by
+.I mbox
+has received a flush request message. A flush request message is
+delivered in a group when the underlying membership of that group
+changes; it is a signal to the application that the membership has
+changed and some of the original members may have "gone away" (meaning
+that they are no longer in the group with this connection). However,
+Flush Spread will not install a new membership in the group until each
+of the members of the new membership flush the group. Flushing a
+group gives Flush Spread the permission of this connection to go ahead
+and install a new view/membership for the group.
+
+During the time after receiving a flush request message but before
+flushing the group the application must be _VERY_ careful about
+calling receive functions; if the application isn't careful enough it
+can permantently block the entire group. For example, say the
+underlying membership of a group of 10 members changed such that this
+connection is now alone in the group (there is no way an application
+can detect the make-up of a group after receiving a flush request).
+If the application simply calls receive before flushing the group and
+hasn't ensured that it will receive a message somehow (see below) it
+will permanently block itself in the receive and any other members
+that might later be added to that group.
+
+There are a couple ways to avoid this problem, which isn't a bug by
+the way :P, (1) don't call receive functions after receiving a flush
+request message and before flushing the group, (2) call receive but
+set the DONT_BLOCK service flag; if there aren't any messages to
+receive it will break out with a WOULD_BLOCK error, (3) ensure that
+there will be a message to receive somehow. There are several ways to
+do this, but one 100% sure way to do this is if the application sends
+or has sent a message on this connection (with the SELF_DISCARD
+service _NOT_ set) that it has not yet received back. In this case,
+the application will eventually receive its own message (even if it
+was to another group) on the connection and can therefore be assured
+of not blocking permanently.
+
+The application _IS_ allowed to send messages to a group after
+receiving a flush request for that group and before flushing the
+group. However, only a subset of the original members of the current
+view will receive these messages (note that I said a subset, not a
+strict subset). Technically, this is always the case in Spread: an
+application can determine which other members received certain
+messages for sure by (1) application level message receipt
+acknowledgement, (2) employing the safety properties of SAFE messages
+(although this doesn't guarantee that those members actually processed
+and handled the message), (3) employing the virtual synchrony property
+and the transitional set of the new membership when it is installed
+(although, again, this doesn't guarantee that those members actually
+processed and handled the messages). See FL_receive or read up on
+group communication for more in-depth discussions of this matter.
+
+Anyways, when the application is ready it can flush the group. Once
+it flushes the group it is not allowed to send any messages to that
+group, until it receives the new membership for that group. If the
+application breaks this rule, it will receive an ILLEGAL_STATE error.
+
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to flush was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error can also be returned
+when a group is flushed for a connection that it has not yet joined or
+is already leaving.
+.TP
+.B ILLEGAL_STATE
+A connection may flush a group only once in response to each flush
+request message delivered in that group. If you violate this rule you
+will get an ILLEGAL_STATE error.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the flush could not be
+initiated.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_join.3
===================================================================
--- trunk/docs/flush/man/FL_join.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_join.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,38 @@
+.TH FL_join 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_join \- join a connection to a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_join(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_join
+joins the connection represented by
+.I mbox
+to a group with the name
+.IR group_name .
+If the group does not exist among the Spread daemons it is created,
+otherwise the connection is joined to the existing group. A
+connection may not join a group it is already a member of, which it
+has already joined, or from which it is currently leaving.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to join was illegal for some reason, usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if a
+group with which this connection is already involved (i.e. - already
+joining, already joined, currently leaving) is joined again.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the join could not be
+initiated.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_leave.3
===================================================================
--- trunk/docs/flush/man/FL_leave.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_leave.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,40 @@
+.TH FL_leave 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_leave \- remove a connection from a Flush Spread group
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_leave(mailbox " mbox ", const char *" group_name ");"
+.SH DESCRIPTION
+.B FL_leave
+removes the connection represented by
+.I mbox
+from a Flush Spread group with the name
+.IR group_name .
+A connection may only leave a group after it has been installed as a
+member of the group by being included in a Flush Spread
+view/membership. Also a connection may not leave a group if it has
+already requested to leave that group and the self-leave message has
+not yet been received.
+.SH "RETURN VALUES"
+Returns 0 on success or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to leave was illegal for some reason. Usually because it was of
+length 0 or length > MAX_GROUP_NAME. This error is also returned if
+the connection was not a full-fledged member (i.e. - not yet included
+in a Flush membership message for that group, or already leaving) of
+the group.
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal. Usually because it is not active.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication
+and the leave could not be initiated.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_more_msgs.3
===================================================================
--- trunk/docs/flush/man/FL_more_msgs.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_more_msgs.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,41 @@
+.TH FL_more_msgs 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_more_msgs \- returns the number of complete messages buffered and
+ready to be received on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_more_msgs(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_more_msgs
+allows an application to check to see if any complete messages are
+already buffered and ready to be received on the connection represented by
+.IR mbox .
+
+.BR NOTE ,
+that this function
+.B CANNOT
+be used as an I/O polling function to check if a receive call should
+be done. If this function returns zero there still might be a message
+ready to receive on the connection: FL_poll, and file descriptor
+selects/polls can detect if there is activity (not necessarily a
+message) on a connection, while DONT_BLOCK receive semantics can
+detect whether or not there is a message on the connection and
+FL_more_msgs can only answer if there are any buffered messages
+already on the connection.
+
+This function is merely a helper function to re-get the current status
+of the
+.I more_messes
+parameter returned from a call to
+.BR FL_receive .
+.SH "RETURN VALUES"
+Returns the number of complete buffered messages ready to be received,
+or one of the following errors ( < 0):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_multicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_multicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,162 @@
+.TH FL_multicast 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_unicast, FL_scat_unicast, FL_subgroupcast, FL_scat_subgroupcast, FL_multicast, FL_scat_multicast \- Multicast messages to subsets of Flush Spread groups.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", const char *" recvr_name ", int16 " mess_type ", const scatter *" scat ");"
+
+.BI "int FL_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_subgroupcast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int " num_recvrs ", const char " recvr_names "[][MAX_GROUP_NAME], int16 " mess_type ", const char *" scat ");"
+
+.BI "int FL_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", int " mess_len ", const char *" mess ");"
+
+.BI "int FL_scat_unicast(mailbox " mbox ", service " serv_type ", const char *" group_name ", int16 " mess_type ", const scatter *" scat ");"
+.SH DESCRIPTION
+.B FL_multicast
+and its variants all multicast a message from the connection represented by
+.I mbox
+to a subset of the membership of the Flush Spread group named
+.IR group_name .
+
+Note, that unlike Spread, Flush Spread does not provide guarantees
+about messages sent to different groups; meaning that FIFO, CAUSAL,
+SAFE, etc. safety properties are not maintained across different Flush
+Spread groups, but only within a group to which they are sent. For
+example, if an application sends a FIFO message A to the private group
+G and then sends another FIFO message B to the regular group H, of
+which G is a member, then G could receive B before A or vice versa.
+
+In order to multicast a message to a group, the destination group
+.I group_name
+must either be a private group name, or this connection must be a
+full-fledged member of that regular group (has joined and been
+included in a membership of the group, and is not leaving the group,
+see FL_join, FL_leave) and not be in an illegal state to multicast (by
+having recently flushed the group and not yet received its next
+view/membership, see FL_flush).
+
+The
+.I serv_type
+is a type field that should be set to the service type this message
+requires. The valid flags for multicasting messages are:
+
+.RS
+.B UNRELIABLE_MESS
+.br
+.TB
+.B RELIABLE_MESS
+.br
+.TB
+.B FIFO_MESS
+.br
+.TB
+.B CAUSAL_MESS
+.br
+.TB
+.B AGREED_MESS
+.br
+.TB
+.B SAFE_MESS
+.RE
+
+This type field can be bit-wise ORed with other flags like
+SELF_DISCARD if desired. Currently SELF_DISCARD is the only
+additional flag for multicasts: SELF_DISCARD will cause the
+multicasted message to NOT be delivered back to this connection.
+
+The string
+.I group_name
+indicates the name of the Flush Spread group to which this message
+should be sent. This group can be either a regular or a private Flush
+Spread group, as dicussed above. The multicast variants sends the
+message to all members of a group, while the unicast and subgroupcast
+variants allow the sender to specify a subset of the group to which to
+send the message. For the unicast variants the sender specifies the
+private group name of the intended recipient in the variable
+.IR recvr_name .
+For the subgroupcast variants the sender specifies the number of
+recipients in the variable
+.I num_recvrs
+and a doubly scripted array of the members' private group names in the
+variable
+.IR recvr_names ,
+where duplicate names are tolerated. For the unicast and subgroupcast
+variants all specified receivers must have been members of the most
+recently installed view/membership of the group. If the application
+breaks this rule it will receive an ILLEGAL_RECEIVERS error.
+
+The
+.I mess_type
+is a 16 bit integer which can be used by the application arbitrarily.
+However, Flush Spread restricts which values can be used to be greater
+than or equal to FL_MIN_LEGAL_MESS_TYPE. If the application sends
+with a message type of less than FL_MIN_LEGAL_MESS_TYPE, then the
+error ILLEGAL_MESSAGE_TYPE will be returned. The intent of this
+message type is that the application can use it to name different
+kinds of data messages so they can be differentiated without looking
+into the body of the message. This value
+.B WILL
+be endian corrected upon returning.
+
+The non-scatter variants use a single buffer to pass the body of the
+message to be sent. The
+.I mess_len
+field gives the message length in bytes. While the
+.I mess
+field is a pointer to the buffer containing the message. For the
+scatter variants, both of these parameters are replaced with one
+pointer,
+.IR scat_mess ,
+to a scatter structure, which is similiar to an iovec. This allows
+messages made up of several parts to be sent without an extra copy on
+systems which support scatter-gather. However, the number of scatter
+elements is restricted to be non-negative and less than or equal to
+FL_MAX_SCATTER_ELEMENTS. If a buffer length is negative or an illegal
+number of scatter elements is used then ILLEGAL_MESSAGE will be
+returned.
+.SH "RETURN VALUES"
+Returns the number of bytes sent on success or one of the following
+errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+was illegal, usually because it is no longer active.
+.TP
+.B ILLEGAL_GROUP
+The
+.I group_name
+given to multicast to was illegal for some reason. Usually because it
+was of length 0 or length > MAX_GROUP_NAME. This error can also be
+returned if the connection was not a full-fledged member (i.e. - not
+yet included in a Flush membership message for the group, or already
+leaving) of a regular group.
+.TP
+.B ILLEGAL_RECEIVERS
+A unicast or subgroupcast specified private group names that weren't
+members of
+.IR group_name 's
+most recent view/membership.
+.TP
+.B ILLEGAL_STATE
+A multicast to a group was attempted after flushing the group and
+before receiving the next view/membership for that group.
+.TP
+.B ILLEGAL_SERVICE
+If the service type was using bits reserved by Flush Spread or Spread.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter, such as a negative array size was passed.
+.TP
+.B ILLEGAL_MESSAGE
+The message had an illegal structure, like a negative buffer size or
+an illegal number of scatter elements.
+.TP
+.B CONNECTION_CLOSED
+Errors occurred during communication and the multicast could not be completed.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_poll.3
===================================================================
--- trunk/docs/flush/man/FL_poll.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_poll.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,32 @@
+.TH FL_poll 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+FL_poll \- returns the amount, in bytes, of activity on a connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_poll(mailbox " mbox ");"
+.SH DESCRIPTION
+.B FL_poll
+is a way to poll the connection represented by
+.I mbox
+to see if there is any activity on that connection.
+
+.BR NOTE ,
+however that activity does
+.B NOT
+necessarily mean that a message is available to be read, as in Spread.
+Only a call to FL_more_msgs that returns a positive number of buffered
+messages or a DONT_BLOCK receive call can check for sure to see if a
+message is now available to be read. If you don't use either of these
+semantics, then you may enter into a blocking receive call, which can
+sometimes be dangerous (see FL_flush).
+.SH "RETURN VALUES"
+Returns the number of bytes of activity currently on the connection,
+or one of the following errors ( < 0 ):
+.TP
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+is illegal, usually because it is not active.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_receive.3
===================================================================
--- trunk/docs/flush/man/FL_receive.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_receive.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,388 @@
+.TH FL_receive 3 "Dec 2000" "Flush Spread" "User Manuals"
+.SH NAME
+Fl_receive, FL_scat_receive \- receive a message on a Flush Spread connection.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "int FL_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", int " max_mess_len ", char *" mess ", int *" more_messes ");"
+
+.BI "int FL_scat_receive(mailbox " mbox ", service *" serv_type ", char " sender "[MAX_GROUP_NAME], int " max_groups ", int *" num_groups ", char " groups "[][MAX_GROUP_NAME], int16 *" mess_type ", int *" endian_mismatch ", scatter *" scat_mess ", int *" more_messes ");"
+.SH DESCRIPTION
+.B FL_receive
+is the general purpose message receipt function for Flush Spread.
+Messages for all groups joined on this connection will arrive on the
+same mailbox. A call to
+.B FL_receive
+will perform a receive on a connection to get a single message from
+any one of the groups to which it is joined.
+
+After a call to receive completes, a number of the passed fields are
+set to values indicating meta-information about the message (such as
+destination group, message type, endianness, etc). The meanings of
+these different meta fields depends on the type of message received
+and the receipt services requested.
+
+Input Parameters:
+.RS
+.TP 1.1i
+.I mbox
+Represents the connection on which to receive a message.
+.TP 1.1i
+.I serv_type
+A pointer to a service bit field, requesting some receive services.
+This field can have the DONT_BLOCK and/or DROP_RECV service bit flags
+flipped on. This bit field should be zeroed out before each call if
+no special receive service is to be requested.
+.TP 1.1i
+.I sender
+A pointer to a character array with storage for at least
+MAX_GROUP_NAME characters.
+.TP 1.1i
+.I max_groups
+A non-negative int representing how many group names the application
+is willing to receive in this receive call.
+.TP 1.1i
+.I num_groups
+A pointer to an int.
+.TP 1.1i
+.I groups
+An array of group names containing storage for at least
+.I max_groups
+* MAX_GROUP_NAME characters.
+.TP 1.1i
+.I mess_type
+A pointer to an int16.
+.TP 1.1i
+.I endian_mismatch
+A pointer to an int.
+.TP 1.1i
+.I max_mess_len
+A non-negative integer representing how many bytes of data the
+application is willing to receive in this call.
+.TP 1.1i
+.I mess
+A pointer to a buffer containing storage for at least
+.I max_mess_len
+bytes.
+.TP 1.1i
+.I scat_mess
+A pointer to a scatter (a cousin of an iovec) with a non-negative
+number of scatter elements that is less than or equal to
+MAX_SCATTER_ELEMENTS (that's not a typo :), and all non-negative
+buffer lengths in the indicated scatter elements to get scatter-gather
+semantics.
+.RE
+
+Receive Semantics:
+
+Normally, a call to receive will block if no messages are immediately
+available.
+
+Normally, when calling a receive function if a user's buffer
+.RI ( groups ,
+.IR mess ,
+or
+.IR scat_mess )
+is too small to contain the data to be returned, then a
+GROUPS_TOO_SHORT or BUFFER_TOO_SHORT error code will be returned. In
+this case, the
+.IR serv_type ,
+and
+.IR mess_type
+of the message are filled in from the offending message and the
+parameters
+.IR num_groups ,
+and
+.I endian_mismatch
+reflect information about the necessary buffers' sizes. If
+.I max_groups
+was big enough, then
+.RI * num_groups
+will be zero, otherwise it will be the negative of how many group
+names are available to be received (i.e. -7 means the
+.I max_groups
+was less than 7 and 7 group names can be received). If
+.I max_mess_len
+or
+.I scat_mess
+was big enough then
+.RI * endian_mismatch
+will be zero, otherwise it will be the negative of how much data is
+available to be received (i.e. -32768 means the msg buffer was too
+small and 32768 bytes of data can be received). The offending message
+is still available to be received through later calls to receive with
+appropriately sized user buffers.
+
+The parameter
+.I serv_type
+allows the applications to request different receive services that
+affect the normal semantics of receive. Currently, the only services
+are DONT_BLOCK and DROP_RECV.
+
+The DONT_BLOCK service makes a receive call non-blocking. With this
+service, the receive call will return quickly either with a message or
+with the error code WOULD_BLOCK if no message is available. Using
+this service is the only way to detect if a message is ready on the
+connection in a non-blocking manner in Flush Spread.
+
+The DROP_RECV service forces Flush Spread to read the current message
+off of the connection regardless of whether or not the user's buffers
+are big enough to fit all of the data. In this case, a TOO_SHORT
+error code is still returned even though the call actually
+succeeded. Anyway, as much data as can be fit into the user's buffers
+will be stuffed into them, and that message will no longer be on the
+connection. Also,
+.RI * num_groups
+will still be set to the negative size of how many names were
+available (if
+.I groups
+wasn't big enough), but
+.RI * endian_mismatch
+will not reflect the size of the data that could have been received (if
+.I mess
+or
+.I scat_mess
+wasn't big enough), instead it indicates whether or not the sending
+machine had an opposite endianness or not. Using DROP_RECV in this
+manner, when a buffer problem occurs there is no way to determine how
+big the data portion of the message actually was. This service is
+meant to be used when a call to receive without DROP_RECV fails with a
+buffer error, but it is determined that the message isn't all that
+important, or something along those lines; otherwise reallocate your
+receive buffers appropriately and re-call receive again without
+the DROP_RECV service.
+
+Output Parameters:
+
+Upon a successful return from receive,
+.RI * serv_type
+will be filled out with the message type that was just received. The
+specific type of message received can be tested using the different
+message and membership type access macros. The rest of the
+parameters' meanings differ depending on the
+.IR serv_type .
+
+Regular Messages:
+
+If the
+.I serv_type
+is a REG_MESSAGE (i.e. a data message) then:
+
+The parameter
+.I sender
+will be filled with the private group name of the sending connection.
+
+The parameter
+.I num_groups
+will be set to the number of group names filled into
+.IR groups .
+
+The
+.I groups
+array will contain the group to which this message was sent. If the
+message is a SUBGROUP_CAST (check the service type, see FL_multicast)
+then all of the recipients' private group names will be listed and the
+last name in the array (i.e.
+.RI groups[* num_groups -1])
+will be the regular group to which this message was sent.
+
+If DROP_RECV is being used, and
+.I groups
+is too small, then as many names as can fit will be filled in as above
+described. If the message was a SUBGROUP_CAST, then the last group in
+the array (i.e.
+.RI groups[ max_groups -1])
+will be the destination group for the message. If
+.I max_groups
+is zero then even the destination group is not reported.
+
+The parameter
+.RI * mess_type
+will be set to the message type field the application sent with
+the original message, which is a restricted (see FL_multicast) 16 bit
+integer. This value is endian corrected upon returning.
+
+The parameter
+.RI * endian_mismatch
+will be set to true (non-zero) if the endianness of the sending
+machine is the opposite of this receiving machine's.
+
+The actual message body being received will be filled into either the
+.I mess
+or
+.IR scat_mess
+parameter.
+
+Flush Request Message:
+
+If this is a FLUSH_REQ_MESS then this represents that the underlying
+membership has changed and that this application needs to flush this
+group before the new view/membership can be installed (see
+FL_flush).
+
+The
+.I sender
+will be filled in with the name of the group this connection needs
+to flush. All the other fields are set to empty values.
+
+Membership Messages:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it
+specifically is a REG_MEMB_MESS, then:
+
+The
+.I sender
+will be filled with the name of the group for which the membership
+change is occuring.
+
+The
+.I mess_type
+field will be set to the index of this process in the
+.I groups
+array (see below).
+
+The
+.I endian_mismatch
+field will be set to 0 since there are no endian issues with regular memberships.
+
+The
+.I groups
+array and message
+body are used to provide two kinds of membership information about the change that just
+occured in this group. The
+.I num_groups
+field will be set to the number of members in the group in the
+.B new
+membership (i.e. after the change occured). Correspondingly, the
+.I groups
+array will be filled in with the private group names of all members of this
+group in the
+.B new
+membership. This list of names is always in the same order at all receipients
+and thus can be used to deterministically pick a group representative if
+one is needed by the application.
+
+The second set of information is stored in the message body. The data
+buffer will include the following fields:
+.RS
+.TP
+.B group_id vid;
+.br
+.TP
+.B int num_members;
+.br
+.TP
+.B char dgroups[][MAX_GROUP_NAME];
+.RE
+
+The vid and num_members parameters will already be endian corrected.
+The dgroups array will contain num_members group names. The content
+of the dgroups array is dependent upon the type of the membership
+change:
+.RS
+.TP 0.8i
+.B CAUSED_BY_JOIN:
+dgroups contains the private group of the joining process.
+.TP
+.B CAUSED_BY_LEAVE:
+dgroups contains the private group of the leaving process.
+.TP
+.B CAUSED_BY_DISCONNECT:
+dgroups contains the private group of the disconnecting process.
+.TP
+.B CAUSED_BY_NETWORK:
+dgroups contains the group names of the members of the new membership who came
+with
+.B this
+application to the new view/membership. Using this data the previous
+membership listing and the new membership listing an application can
+determine all the members that left and were added in the NETWORK
+membership change. Note, that a member can both leave and be added in
+a
+.B single
+NETWORK change.
+.RE
+
+Transitional Message:
+
+If this is a MEMB_MESSAGE and specifically it is a TRANSITION_MESS,
+then this means that one or more of the current members of the group
+has left and not all of the safety guarantees can be met w.r.t. all of
+the original members for the messages delivered after this signal and
+before the next view/membership. Transitional messages never have a
+CAUSED_BY type because they only serve to signal up to where SAFE
+delivery and AGREED delivery (with no holes) is guaranteed in the
+view/membership in which they are delivered. Only one TRANSITION_MESS
+is delivered per group view/membership per connection.
+
+The
+.I sender
+will be filled in with the name of the group for which the membership
+change is occurring. All the other parameters will be set to empty values.
+
+Self-Leave Message:
+
+If this is a MEMB_MESSAGE (i.e. membership message) and it is neither
+a REG_MEMB_MESS or a TRANSITION_MESS, then it represents exactly the
+situtation where the member receiving this message has left a group
+and this is notification that the leave has occured, thus it is
+sometimes called a self-leave message. The simplest test for this is
+if a message is CAUSED_BY_LEAVE and REG_MEMB_MESS is false (zero),
+then it is a self-leave message.
+
+The other members of the group that this member just left will receive
+a normal TRANSITION_MESS, REG_MEMB_MESS pair as described above
+showing this connection leaving.
+
+For self-leave messages the
+.I sender
+field will be filled in with the name of the group this connection is
+leaving. All the other fields are set to empty values.
+
+.SH "RETURN VALUES"
+Returns the size of the data portion of the message received on
+success or one of the following errors ( < 0 ):
+.TP 0.8i
+.B ILLEGAL_SESSION
+The connection represented by
+.I mbox
+was illegal, usually because it is not active.
+.TP
+.B ILLEGAL_PARAM
+An illegal parameter was passed, like a negative array size.
+.TP
+.B ILLEGAL_MESSAGE
+The msg buffer had an illegal structure, like an illegal number of
+scatter elements or a negatively sized buffer.
+.TP
+.B WOULD_BLOCK
+The receive call was made with the DONT_BLOCK service and it would
+have blocked because no messages were ready to be received.
+.TP
+.B GROUPS_TOO_SHORT
+The receive call was made with a groups array that was too small.
+.RI * num_groups
+is filled in with the negative number of names that could have been
+returned. Note, that the msg buffer could also be too short, and this
+should be determined by examining
+.I endian_mismatch
+as described above. If DROP_RECV was used this error code actually
+represents success (see above).
+.TP
+.B BUFFER_TOO_SHORT
+The receive call was made with a msg buffer that was too small.
+.RI * endian_mismatch
+is filled in with the negative number of bytes that could have been
+received if DROP_RECV isn't used. If it is used then this error code
+actually represents success and
+.I endian_mismatch
+has the normal meaning (see above). Note, that the groups buffer
+could also be too short, this should be determined by examining
+.I num_groups
+as described above.
+.TP
+.B CONNECTION_CLOSED
+During communication to receive, communication errors occurred
+and the receive could not be completed.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/docs/flush/man/FL_scat_multicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_multicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_multicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_scat_receive.3
===================================================================
--- trunk/docs/flush/man/FL_scat_receive.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_receive.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_receive
Added: trunk/docs/flush/man/FL_scat_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_subgroupcast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_subgroupcast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_scat_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_scat_unicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_scat_unicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_subgroupcast.3
===================================================================
--- trunk/docs/flush/man/FL_subgroupcast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_subgroupcast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_unicast.3
===================================================================
--- trunk/docs/flush/man/FL_unicast.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_unicast.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1 @@
+.so FL_multicast.3
Added: trunk/docs/flush/man/FL_version.3
===================================================================
--- trunk/docs/flush/man/FL_version.3 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/docs/flush/man/FL_version.3 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,14 @@
+.TH FL_version 3 "Dec 2000" "Flush Spread" "User Manuals"
+
+.SH NAME
+FL_version \- returns library version information.
+.SH SYNOPSIS
+.B #include <fl.h>
+
+.BI "void FL_version(int *" major_ver ", int *" minor_ver ", int *" patch_ver ");"
+.SH DESCRIPTION
+.B FL_version
+returns the major, minor, and patch version numbers of the Flush
+Spread library in use.
+.SH AUTHOR
+John Schultz <[email protected]>
Added: trunk/flush/Makefile.in
===================================================================
--- trunk/flush/Makefile.in 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/Makefile.in 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,57 @@
+CC=@CC@
+AR=@AR@
+
+SP_INC_DIR = -I../include
+SP_LIB_DIR = -L../daemon
+
+STD_INC_DIR = -I../stdutil/src
+STD_LIB_DIR = -L../stdutil/lib
+
+# use thread safe libs and enable std locking mechanisms (optional)
+THREAD_SAFE = -D_REENTRANT
+
+# don't use debugging (asserts and dprintfs) (optional)
+NO_DEBUG = -DNDEBUG
+
+# libraries apps need to link with: pthreads (optional), flush, stdutil, spread
+LIBS = @LIBS@ $(TARGET_DIR)/libflush.a -ltspread -lstdutil
+
+# where to put the libraries (libflush.a, libstdutil.a, etc.) and exes
+TARGET_DIR = .
+APP_DIR = .
+
+# all the defined variables
+DEFINES = $(THREAD_SAFE) $(NO_DEBUG) $(DO_TIMINGS)
+
+# all the include directories
+INCLUDES = $(SP_INC_DIR) -I. -I../stdutil/src
+
+# all the library linkage diretories
+LDFLAGS = $(SP_LIB_DIR)
+
+# gcc specific compiler flags
+CFLAGS = -Wall -O $(INCLUDES) $(DEFINES) # -g
+
+TARGETS = $(TARGET_DIR)/libflush.a $(APP_DIR)/flush_user $(APP_DIR)/sp_time_memb $(APP_DIR)/fl_time_memb
+
+OBJS = fl.o scatp.o
+
+all: $(TARGETS)
+
+$(TARGET_DIR)/libflush.a: $(OBJS)
+ $(AR) -rs $(TARGET_DIR)/libflush.a $(OBJS)
+
+$(APP_DIR)/flush_user: $(TARGET_DIR)/libflush.a user.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/flush_user user.o $(LIBS)
+
+$(APP_DIR)/sp_time_memb: $(TARGET_DIR)/libflush.a sp_time_memb.o stats.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/sp_time_memb sp_time_memb.o stats.o $(LIBS)
+
+$(APP_DIR)/fl_time_memb: $(TARGET_DIR)/libflush.a fl_time_memb.o stats.o
+ $(CC) $(CFLAGS) $(LDFLAGS) -o $(APP_DIR)/fl_time_memb fl_time_memb.o stats.o $(LIBS)
+
+clean:
+ rm -f *~ *.o $(TARGETS) core *.bak
+
+distclean: clean
+ rm -f Makefile config.*
Added: trunk/flush/configure.in
===================================================================
--- trunk/flush/configure.in 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/configure.in 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,8 @@
+AC_INIT(fl.c)
+
+AC_PROG_CC
+AC_CHECK_PROGS(AR, ar)
+AC_CHECK_LIB(pthread, pthread_mutex_init)
+AC_CHECK_LIB(m, sqrt)
+
+AC_OUTPUT(Makefile)
Added: trunk/flush/fl.c
===================================================================
--- trunk/flush/fl.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,2141 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <stdlib.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+#include <fl_p.h>
+#include <stdutil/stdutil.h>
+
+/* glob_conns is a look up table that maps mailboxes to fl_conn*s */
+static stdmutex glob_conns_lock;
+static stdhash glob_conns = STDHASH_STATIC_CONSTRUCT(sizeof(mailbox), sizeof(fl_conn*), NULL, NULL, 0);
+
+/********************************* public flush layer interface ********************************/
+
+int FL_lib_init(void)
+{
+ static stdbool first_time = STDTRUE; /* *TRY* to mitigate bad call race conditions to FL_lib_init */
+ int ret = ILLEGAL_STATE;
+
+ if (first_time) {
+ first_time = STDFALSE;
+ ret = stdmutex_construct(&glob_conns_lock, STDMUTEX_FAST);
+ }
+
+ return ret;
+}
+
+void FL_version(int *major, int *minor, int *patch) {
+ *major = FL_MAJOR_VERSION;
+ *minor = FL_MINOR_VERSION;
+ *patch = FL_PATCH_VERSION;
+}
+
+/* Establish a new fl connection. If SP_connect succceeds, create a new fl_conn instance. */
+int FL_connect(const char *daemon, const char *user, int priority,
+ mailbox *mbox, char private[MAX_GROUP_NAME]) {
+ int ret;
+ fl_conn *conn;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_connect: daemon '%s', user '%s', priority %d\n",
+ daemon, user, priority));
+ if (FL_SP_version() < (float) 3.12) { /* flush depends on the DROP_RECV semantics */
+ DEBUG(std_stkfprintf(stderr, 0, "REJECT_VERSION: SP too old v%f < v3.12\n", FL_SP_version()));
+ ret = REJECT_VERSION;
+ } else if ((ret = SP_connect(daemon, user, priority, 1, mbox, private)) == ACCEPT_SESSION) {
+ DEBUG(std_stkfprintf(stderr, 0, "mbox %d, private '%s'\n", *mbox, private));
+
+ if ((conn = (fl_conn*) calloc(1, sizeof(fl_conn))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_conn));
+
+ stdmutex_construct(&conn->reserve_lock, STDMUTEX_FAST);
+ conn->reservations = 0;
+ conn->disconnecting = 0;
+ stdcond_construct(&conn->destroy_cond);
+
+ stdmutex_construct(&conn->recv_lock, STDMUTEX_FAST);
+ stdmutex_construct(&conn->conn_lock, STDMUTEX_FAST);
+
+ conn->mbox = *mbox;
+ conn->priority = priority;
+ conn->group_memb = 1;
+ strncpy(conn->daemon, daemon, MAX_GROUP_NAME);
+ strncpy(conn->user, user, MAX_GROUP_NAME);
+ strncpy(conn->private, private, MAX_GROUP_NAME);
+
+ stdhash_construct(&conn->groups, sizeof(char*), sizeof(fl_group*),
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ stddll_construct(&conn->mess_queue, sizeof(gc_buff_mess*)); /* <gc_buff_mess*> */
+ conn->bytes_queued = 0;
+
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+ stdhash_insert(&glob_conns, 0, mbox, &conn); /* add mbox -> conn mapping */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_connect: ret %d\n", ret));
+ return ret;
+}
+
+/* Destroy a mbox. This fcn recursively reclaims all resources
+ associated with a connection named mbox, in a synchronized, proper
+ fashion and returns whatever SP_disconnect does. If mbox doesn't
+ represent a valid fl connection, it just returns ILLEGAL_SESSION.
+*/
+int FL_disconnect(mailbox mbox) {
+ stdit hit;
+ stdit lit;
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_disconnect: mbox %d\n", mbox));
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+ if (!stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox))) { /* mbox found */
+ conn = *(fl_conn**) stdhash_it_val(&hit);
+ stdhash_erase(&glob_conns, &hit); /* remove mbox -> fl_conn* mapping */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ /* now threads cannot come behind us and make_reservations() on this fl_conn */
+
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ conn->disconnecting = 1; /* set disconnection indicator */
+ ret = SP_disconnect(mbox); /* unblock any blocked receivers */
+
+ if (conn->reservations != 0) { /* WAIT if connection still in use */
+ DEBUG(std_stkfprintf(stderr, 0, "Waiting for mbox(%d, %p) to no longer be in use: "
+ "%d reservations!\n", mbox, conn, conn->reservations));
+ stdcond_wait(&conn->destroy_cond, &conn->reserve_lock);
+ }
+ assert(conn->reservations == 0); /* should be no more reservations */
+
+ /* now, no other threads are using the connection -> safe to reclaim */
+ DEBUG(std_stkfprintf(stderr, 0, "Mbox(%d, %p) no longer in use: reclaiming!\n", mbox, conn));
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+
+ stdmutex_destruct(&conn->reserve_lock); /* recursively destroy fl_conn and sub-structures */
+ stdcond_destruct(&conn->destroy_cond);
+ stdmutex_destruct(&conn->recv_lock);
+ stdmutex_destruct(&conn->conn_lock);
+
+ for (stdhash_begin(&conn->groups, &hit); !stdhash_is_end(&conn->groups, &hit); stdhash_it_next(&hit))
+ free_fl_group(*(fl_group**) stdhash_it_val(&hit));
+ stdhash_destruct(&conn->groups);
+
+ for (stddll_begin(&conn->mess_queue, &lit); !stddll_is_end(&conn->mess_queue, &lit); stddll_it_next(&lit))
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ stddll_destruct(&conn->mess_queue);
+
+ free(conn);
+ } else {
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_disconnect: ret %d\n", ret));
+ return ret; /* return SP_disconnect return value */
+}
+
+int FL_join(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_join: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ /* ACHTUNG! You may not join a group when you are already involved with that group */
+ if ((group = get_group(conn, grp)) == 0) {
+ ret = SP_join(mbox, grp);
+ unlock_conn(conn);
+ if (ret != 0) {
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ else if (ret != ILLEGAL_GROUP)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_join: unexpected error %d\n",
+ __FILE__, __LINE__, mbox, grp, ret);
+ }
+ } else {
+ ret = ILLEGAL_GROUP;
+ unlock_conn(conn);
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp,
+ group, group == 0 ? -1 : group->mstate));
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_join: ret %d\n", ret));
+ return ret;
+}
+
+int FL_leave(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_leave: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ /* ACHTUNG! If you are a joined member of a fl group only then you can leave it only once! */
+ if ((group = get_group(conn, grp)) != 0 && group->mstate == JOINED) {
+ group->mstate = LEAVING;
+ ret = SP_leave(mbox, grp);
+ unlock_conn(conn);
+ if (ret != 0) {
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ else if (ret == ILLEGAL_GROUP)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: ILLEGAL_GROUP\n",
+ __FILE__, __LINE__, mbox, grp);
+ else
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_leave: unexpected error %d\n",
+ __FILE__, __LINE__, mbox, grp, ret);
+ }
+ } else {
+ ret = ILLEGAL_GROUP;
+ unlock_conn(conn);
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', %p) in mstate %d\n", grp,
+ group, group == 0 ? -1 : group->mstate));
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_leave: ret %d\n", ret));
+ return ret;
+}
+
+int FL_flush(mailbox mbox, const char *grp) {
+ fl_conn *conn;
+ fl_group *group;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_flush: mbox %d, group '%s'\n", mbox, grp));
+ if ((conn = lock_conn(mbox)) != 0) {
+ if ((group = get_group(conn, grp)) != 0)
+ ret = FL_int_flush(conn, group);
+ else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_GROUP('%s', 0)\n", grp));
+ ret = ILLEGAL_GROUP;
+ }
+ unlock_conn(conn);
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_flush: ret %d\n"));
+ return ret;
+}
+
+int FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat) {
+ return FL_int_scat_multicast(mbox, serv_type | SUBGROUP_CAST, grp,
+ num_recvrs, recvrs, mess_type, scat);
+}
+
+int FL_subgroupcast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_subgroupcast(mbox, serv_type, grp, num_recvrs, recvrs, mess_type, &msg);
+}
+
+int FL_scat_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr,
+ int16 mess_type, const scatter *scat) {
+ return FL_scat_subgroupcast(mbox, serv_type, grp, 1, (char(*)[MAX_GROUP_NAME]) recvr,
+ mess_type, scat);
+}
+
+int FL_unicast(mailbox mbox, service serv_type, const char *grp, const char *recvr,
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_unicast(mbox, serv_type, grp, recvr, mess_type, &msg);
+}
+
+int FL_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+ int16 mess_type, const scatter *scat) {
+ return FL_int_scat_multicast(mbox, serv_type & ~SUBGROUP_CAST, grp, 0, 0, mess_type, scat);
+}
+
+int FL_multicast(mailbox mbox, service serv_type, const char *grp,
+ int16 mess_type, int mess_len, const char *mess) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_multicast(mbox, serv_type, grp, mess_type, &msg);
+}
+
+int FL_scat_receive(mailbox mbox, service *serv_type, char *sender, int max_groups,
+ int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type,
+ int *endian_mismatch, scatter *scat_mess, int *more_messes) {
+ int blocking = (*serv_type & DONT_BLOCK) == 0; /* user not using DONT_BLOCK */
+ gc_recv_mess msg; /* used to contain user's parameters */
+ fl_conn *conn;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_scat_receive: mbox %d, service 0x%X, max_groups %d\n",
+ mbox, *serv_type, max_groups));
+
+ if (max_groups < 0) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal max_groups!\n"));
+ msg.ret = ILLEGAL_PARAM;
+ } else {
+ /* I use msg rather than the individual parameters to pass user's variables around */
+ msg.mbox = mbox;
+ msg.orig_serv_type = *serv_type;
+ msg.serv_type = serv_type;
+ msg.sender = sender;
+ msg.max_groups = max_groups;
+ msg.num_groups = num_groups;
+ msg.groups = groups;
+ msg.mess_type = mess_type;
+ msg.endian_mismatch = endian_mismatch;
+ msg.scat_mess = scat_mess;
+ msg.delivered = 0; /* does msg contain values to be returned to the user? */
+
+ if ((conn = make_reservation(mbox)) != 0) { /* get conn and make sure it stays valid */
+ /* NEED TO ENFORCE THAT THERE IS ONLY ONE READING THREAD AT A TIME PER CONNECTION, */
+ /* BECAUSE A THREAD COULD BECOME BLOCKED IN SP_recv AND THEN MSGS ARE GENERATED BY */
+ /* ANOTHER THREAD AND PUT INTO THE QUEUE TO BE RETURNED -> THE BLOCKED THREAD */
+ /* WOULDN'T REALIZE IT! SERIALIZATION ON RECV_LOCK ACCOMPLISHES THIS!!!! */
+ if (acquire_recv_lock(conn)) {
+ if (acquire_conn_lock(conn)) { /* get conn lock for so I can read connection info */
+ int have_conn_lock = 1; /* do I have the conn lock? */
+
+ FL_scat_re_receive:
+ msg.vulnerable = 0; /* is the contained message vulnerable? */
+ msg.num_new_grps = 0; /* ensure new_grps and new_msg are empty */
+ msg.new_msg.num_elements = 0; /* used for DROP_RECV msgs (see FL_int_receive) */
+
+ if (stddll_empty(&conn->mess_queue)) { /* empty msg queue */
+ if (blocking || SP_poll(mbox) != 0) { /* blocking allowed, or msg available */
+ have_conn_lock = recv_and_handle(conn, &msg); /* call SP_recv and enter FSA */
+
+ if (msg.num_new_grps != 0) /* reclaim any allocated buffers */
+ free(msg.new_grps);
+
+ if (msg.new_msg.num_elements != 0)
+ free(msg.new_msg.elements[0].buf);
+
+ if (!msg.delivered) { /* if msg doesn't contain returnable values */
+ DEBUG(std_stkfprintf(stderr, 0, "No return msg, going to top of recv loop!\n"));
+ goto FL_scat_re_receive; /* need to try and get a msg again */
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Msg contains values for user!\n"));
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Call to receive would have blocked!\n"));
+ msg.ret = WOULD_BLOCK; /* would have blocked */
+ }
+ } else { /* message in queue! */
+ gc_buff_mess *mess; /* get buff mess */
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 0, "Msg in buffer, reading in to user's parameters\n"));
+ mess = *(gc_buff_mess**) stddll_it_val(stddll_begin(&conn->mess_queue, &lit));
+ if (buffm_to_userm(&msg, mess)) { /* msg copy to user's parameters successful */
+ stddll_erase(&conn->mess_queue, &lit); /* pop mess off of the top of the queue */
+ conn->bytes_queued -= mess->total_size; /* update bytes queued */
+ free_gc_buff_mess(mess); /* destroy the gc_buff_mess */
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Msg incompatible with user's parameters!\n"));
+ }
+ if (have_conn_lock) { /* if I didn't lose the conn lock */
+ if (msg.ret >= 0)
+ *more_messes = (int) stddll_size(&conn->mess_queue); /* how many msgs left */
+ release_conn_lock(conn); /* let go of the conn lock */
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire conn lock -> disconnecting!\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ release_recv_lock(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't acquire recv lock -> disconnecting!\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ cancel_reservation(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't get a reservation\n"));
+ msg.ret = ILLEGAL_SESSION;
+ }
+ DEBUG( /* only execute if statment when DEBUGGING */
+ if (msg.ret >= 0) {
+ int i;
+
+ std_stkfprintf(stderr, 0, "Groups(%d):\n", *num_groups);
+ for (i = 0; i < *num_groups; ++i)
+ std_stkfprintf(stderr, 0, "\t'%s'\n", groups[i]);
+ });
+ if (msg.ret == CONNECTION_CLOSED || msg.ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_scat_receive: ret %d, mbox %d, service 0x%X, sender '%s'"
+ ", num_groups %d, mess_type %d, endian %d, more_messes %d\n",
+ msg.ret, mbox, *serv_type, msg.ret >= 0 ? sender : "", *num_groups,
+ *mess_type, *endian_mismatch, *more_messes));
+ return msg.ret;
+}
+
+int FL_receive(mailbox mbox, service *serv_type, char *sender, int max_groups,
+ int *num_groups, char groups[][MAX_GROUP_NAME], int16 *mess_type,
+ int *endian_mismatch, int max_mess_len, char *mess, int *more_messes) {
+ scatter msg;
+
+ msg.num_elements = 1;
+ msg.elements[0].len = max_mess_len;
+ msg.elements[0].buf = (char*) mess;
+
+ return FL_scat_receive(mbox, serv_type, sender, max_groups, num_groups, groups,
+ mess_type, endian_mismatch, &msg, more_messes);
+}
+
+int FL_more_msgs(mailbox mbox) {
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_more_msgs: mbox %d\n", mbox));
+ if ((conn = lock_conn(mbox)) != 0) {
+ ret = (int) stddll_size(&conn->mess_queue);
+ unlock_conn(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_more_msgs: ret %d\n", ret));
+ return ret;
+}
+
+int FL_poll(mailbox mbox) {
+ fl_conn *conn;
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_poll: mbox %d\n", mbox));
+ if ((conn = lock_conn(mbox)) != 0) {
+ ret = conn->bytes_queued + SP_poll(mbox);
+ unlock_conn(conn);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_poll: ret %d\n", ret));
+ return ret;
+}
+
+void FL_error(int err) {
+ switch (err) {
+ case ILLEGAL_PARAM:
+ printf("FL_error: (%d) Illegal parameter (eg a negative size) passed to a function.\n", err);
+ break;
+ case WOULD_BLOCK:
+ printf("FL_error: (%d) Function call would have blocked.\n", err);
+ break;
+ case ILLEGAL_MESSAGE_TYPE:
+ printf("FL_error: (%d) Illegal message type (int16) used, "
+ "value < FL_MIN_LEGAL_MESS_TYPE(%d).\n", err, FL_MIN_LEGAL_MESS_TYPE);
+ break;
+ case ILLEGAL_STATE:
+ printf("FL_error: (%d) Function call peformed in a prohibited state.\n", err);
+ break;
+ case ILLEGAL_RECEIVERS:
+ printf("FL_error: (%d) Illegal receivers specified.\n", err);
+ break;
+ default:
+ SP_error(err);
+ break;
+ }
+}
+
+/************************************ private interface ****************************************/
+
+/* structure routines */
+/* When I initially create a group structure I set it up to be ready to insert in to a groups */
+/* hash of a connection. I set mstate to JOINING indicating this group hasn't installed this */
+/* connection as a member yet. I set vstate to AGREE, which will trigger auto-flush oks from */
+/* this joining connection on SP memb events, until it finally gets installed. I create a */
+/* fl_view with only the connection in fl_view->membs. I do this for three reasons: (1) */
+/* sp_view and fl_view are always pointing at valid views now (no special checks for null), */
+/* (2) in case this connection is joined through a NETWORK event, it will report the proper */
+/* singleton vs set, and (3) this also gives me correct behavior on delivery of user's data */
+/* msgs (is sender in my current fl view?). */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name) {
+ fl_group *group;
+ group_id gid = {};
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_fl_group: conn_name '%s', group '%s'\n",
+ conn_name, group_name));
+
+ if ((group = (fl_group*) calloc(1, sizeof(fl_group))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(fl_group));
+
+ strncpy(group->group, group_name, MAX_GROUP_NAME);
+ group->mstate = JOINING; /* not a JOINED member yet */
+ group->vstate = AGREE; /* when I handle_next_memb_change first time -> sends FLUSH_OK */
+ group->curr_change = 0;
+ group->sp_view = group->fl_view = create_view(gid); /* put an empty view to start */
+ fill_view(group->fl_view, 0, 1, (char(*)[MAX_GROUP_NAME]) conn_name, 0); /* insert self */
+ group->fl_view->in_trans_memb = 1; /* don't deliver any trans sigs until a member */
+ group->flush_recvs = 0;
+ stddll_construct(&group->mess_queue, sizeof(gc_buff_mess*));
+ stddll_construct(&group->memb_queue, sizeof(sp_memb_change*));
+ stdhash_construct(&group->pmemb_hash, sizeof(group_id), sizeof(sp_memb_change*),
+ group_id_cmp, group_id_hashcode, 0); /* use default key fcns for group_id */
+ DEBUG(std_stkfprintf(stderr, -1, "create_fl_group: group('%s', %p)\n", group->group, group));
+ return group;
+}
+
+static void free_fl_group(fl_group *group) {
+ stdit lit;
+ stdit hit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "free_fl_group: group('%s', %p)\n", group->group, group));
+ /* sp view is either equal to fl_view or points in to one of the sp_memb_change's */
+ free_view(group->fl_view);
+
+ for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit))
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ stddll_destruct(&group->mess_queue);
+
+ /* curr_change and all sp_memb_changes contained in memb_queue are also in pmemb */
+ stddll_destruct(&group->memb_queue);
+
+ for (stdhash_begin(&group->pmemb_hash, &hit); !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit))
+ free_sp_memb_change(*(sp_memb_change**) stdhash_it_val(&hit));
+ stdhash_destruct(&group->pmemb_hash);
+
+ free(group);
+ DEBUG(std_stkfprintf(stderr, -1, "free_fl_group\n"));
+}
+
+static void free_gc_buff_mess(gc_buff_mess *msg) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_gc_buff_mess: m %p, serv 0x%X\n", msg, msg->serv_type));
+ if (msg->num_groups)
+ free(msg->groups);
+
+ if (msg->mess_len)
+ free(msg->mess);
+
+ free(msg);
+ DEBUG(std_stkfprintf(stderr, -1, "free_gc_buff_mess\n"));
+}
+
+static sp_memb_change *create_sp_memb_change(group_id gid) {
+ sp_memb_change *spc;
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_sp_memb_change: gid %d %d %d\n",
+ gid.id[0], gid.id[1], gid.id[2]));
+
+ if ((spc = (sp_memb_change*) calloc(1, sizeof(sp_memb_change))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", __FILE__, __LINE__, sizeof(sp_memb_change));
+
+ spc->memb_info = create_view(gid);
+ spc->memb_mess_recvd = 0;
+ spc->memb_change_age = 0;
+ spc->delta_member = 0;
+ stdhash_construct(&spc->flok_senders, MAX_GROUP_NAME, 0,
+ group_name_cmp, group_name_hashcode, 0);
+ spc->gid = gid;
+
+ DEBUG(std_stkfprintf(stderr, -1, "create_sp_memb_change: ret %p, gid %d %d %d\n", spc,
+ spc->memb_info->gid.id[0], spc->memb_info->gid.id[1],
+ spc->memb_info->gid.id[2]));
+ return spc;
+}
+
+static void free_sp_memb_change(sp_memb_change *spc) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_sp_memb_change: spc %p\n", spc));
+
+ if (spc->memb_info != 0)
+ free_view(spc->memb_info);
+
+ if (spc->delta_member != 0)
+ free(spc->delta_member);
+
+ stdhash_destruct(&spc->flok_senders);
+ free(spc);
+ DEBUG(std_stkfprintf(stderr, -1, "free_sp_memb_change\n"));
+}
+
+static view *create_view(group_id gid) {
+ view *v;
+
+ DEBUG(std_stkfprintf(stderr, 1, "create_view: gid %d %d %d\n",
+ gid.id[0], gid.id[1], gid.id[2]));
+
+ if ((v = (view*) calloc(1, sizeof(view))) == 0)
+ stderr_abort("(%s, %d): calloc(1, %u)\n", sizeof(view));
+
+ v->gid = gid;
+ v->memb_type = 0;
+ v->in_trans_memb = 0;
+ v->orig_num_membs = 0;
+ v->membs_names = 0;
+ stdhash_construct(&v->orig_membs, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ stdhash_construct(&v->curr_membs, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+ v->my_index = -1;
+
+ DEBUG(std_stkfprintf(stderr, -1, "create_view: ret %p\n", v));
+ return v;
+}
+
+static void free_view(view *v) {
+ DEBUG(std_stkfprintf(stderr, 1, "free_view: view %p, gid %d %d %d\n", v, v->gid.id[0],
+ v->gid.id[1], v->gid.id[2]));
+ if (v->orig_num_membs != 0)
+ free(v->membs_names);
+ stdhash_destruct(&v->orig_membs);
+ stdhash_destruct(&v->curr_membs);
+ free(v);
+ DEBUG(std_stkfprintf(stderr, -1, "free_view\n"));
+}
+
+static void fill_view(view *v, service memb_type, int num_membs,
+ char (*membs)[MAX_GROUP_NAME], int16 index) {
+ size_t byte_size = num_membs * MAX_GROUP_NAME;
+
+ DEBUG(std_stkfprintf(stderr, 1, "fill_view: view %p, serv 0x%X, num_membs %d, index %d\n",
+ v, memb_type, num_membs, index));
+
+ assert(num_membs > 0 && index >= 0 && index < num_membs && v->orig_num_membs == 0 &&
+ stdhash_empty(&v->orig_membs) && stdhash_empty(&v->orig_membs));
+
+ v->memb_type = memb_type;
+ v->orig_num_membs = num_membs;
+
+ if ((v->membs_names = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc (%d)\n", __FILE__, __LINE__, byte_size);
+ memcpy(v->membs_names, membs, byte_size);
+
+ for (membs = v->membs_names; num_membs--; ++membs) /* I set membs = v->membs_names!!! */
+ stdhash_insert(&v->orig_membs, 0, &membs, 0), stdhash_insert(&v->curr_membs, 0, &membs, 0);
+
+ v->my_index = index;
+ DEBUG(std_stkfprintf(stderr, -1, "fill_view: view %p, serv 0x%X, num_membs %lu, index %d\n",
+ v, v->memb_type, stdhash_size(&v->orig_membs), v->my_index));
+}
+
+/* Looks up a fl_conn "named" mbox, gets an outstanding reservation on
+ it and acquires its conn_lock. Returns the valid locked fl_conn* on
+ success or 0 on failure -> ILLEGAL_SESSION.
+*/
+static fl_conn *lock_conn(mailbox mbox) {
+ fl_conn *conn;
+
+ if ((conn = make_reservation(mbox)) == 0)
+ return 0;
+
+ if (!acquire_conn_lock(conn)) /* error -> disconnecting */
+ return cancel_reservation(conn), (fl_conn*) 0;
+
+ return conn;
+}
+
+/* Unlock a connection successfully locked with lock_conn. */
+static void unlock_conn(fl_conn *conn) {
+ release_conn_lock(conn);
+ cancel_reservation(conn);
+}
+
+/* Get a pointer to a fl_conn "named" mbox and "acquire an outstanding
+ reservation" by increasing a reservation counter. A successful
+ return *ONLY* entitles the caller to make subsequent calls to
+ acquire/release_conn_lock and acquire/release_recv_lock on the
+ returned fl_conn. Each successful call to make_reservation MUST be
+ matched by a call to cancel_reservation on the returned
+ fl_conn. This outstanding reservation only ensures that the
+ returned fl_conn is valid between a successful call to
+ make_reservation and the subsequent call to cancel_reservation on
+ the returned fl_conn. If mbox is not a valid fl connection this fcn
+ fails and returns 0. NOTE: it is imperative that this fcn acquires
+ the reserve lock before releasing the glob_conns_lock.
+*/
+static fl_conn *make_reservation(mailbox mbox) {
+ stdit hit;
+ fl_conn *conn;
+
+ stdmutex_grab(&glob_conns_lock); /* LOCK CONNS TAB */
+
+ if (stdhash_is_end(&glob_conns, stdhash_find(&glob_conns, &hit, &mbox))) /* no such mbox */
+ return stdmutex_drop(&glob_conns_lock), (fl_conn*) 0; /* UNLOCK CONNS TAB */
+
+ conn = *(fl_conn**) stdhash_it_val(&hit);
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ stdmutex_drop(&glob_conns_lock); /* UNLOCK CONNS TAB */
+ if (!conn->disconnecting) /* check if disconnecting */
+ ++conn->reservations; /* increment reservation cnt */
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+
+ return !conn->disconnecting ? conn : 0;
+}
+
+/* Relinquish a reservation this thread owns on a connection. */
+static void cancel_reservation(fl_conn *conn) {
+ stdmutex_grab(&conn->reserve_lock); /* LOCK RESERVE_LOCK */
+ assert(conn->reservations != 0);
+ if (--conn->reservations == 0 && conn->disconnecting) /* decrement reservation cnt */
+ stdcond_wake_all(&conn->destroy_cond); /* signal disconnector thread */
+ stdmutex_drop(&conn->reserve_lock); /* UNLOCK RESERVE_LOCK */
+}
+
+/* Acquire a connection's conn lock. The caller must have an
+ outstanding reservation on conn. Once this lock is acquired, a
+ thread can freely examine and/or modify the data in conn as
+ indicated in <fl_p.h>. Returns 0 on failure -> ILLEGAL_SESSION.
+ Each successful call made by a thread on a particular conn must be
+ matched with a call to release_conn_lock to allow that conn to be
+ used by other threads. A thread may not make a second call to
+ acquire_conn_lock after a successful return on a particular conn
+ without an intervening call to release_conn_lock on that conn to
+ prevent self-deadlocks.
+*/
+static int acquire_conn_lock(fl_conn *conn) {
+ stdmutex_grab(&conn->conn_lock); /* LOCK CONN_LOCK */
+ if (conn->disconnecting) /* conn is disconnecting */
+ return stdmutex_drop(&conn->conn_lock), 0;
+
+ return 1;
+}
+
+/* Release a connection's conn lock. The caller must have an
+ oustanding reservation on release. The calling thread must also
+ have made a successful call to acquire_conn_lock on release without
+ any intervening calls to release_conn_lock on release.
+*/
+static void release_conn_lock(fl_conn *release) {
+ stdmutex_drop(&release->conn_lock); /* UNLOCK CONN_LOCK */
+}
+
+/* Acquire a connection's recv lock. The caller must have an
+ outstanding reservation on conn. Once this lock is acquired, a
+ thread can proceed into the body of FL_scat_recv. Returns 0
+ on failure -> ILLEGAL_SESSION. Each successful call made by a
+ thread on a particular conn must be matched with a call to
+ release_recv_lock to allow that conn to be used by other threads. A
+ thread may not make a second call to acquire_recv_lock after a
+ successful return on a particular conn without an intervening call
+ to release_recv_lock on that conn to prevent self-deadlocks. NOTE
+ THAT THIS LOCK DOES NOT GIVE A THREAD ANY RIGHTS TO EXAMINE OR
+ MODIFY _ANY_ OF THE CONNECTION'S DATA!! IT ONLY ALLOWS A THREAD TO
+ ENTER THE BODY OF FL_scat_recv!!
+*/
+static int acquire_recv_lock(fl_conn *conn) {
+ stdmutex_grab(&conn->recv_lock); /* LOCK RECV_LOCK */
+ if (conn->disconnecting) /* conn is disconnecting */
+ return stdmutex_drop(&conn->recv_lock), 0;
+
+ return 1;
+}
+
+/* Release a connection's recv lock. The caller must have an
+ oustanding reservation on release. The calling thread must have
+ made a successful call to acquire_recv_lock on release without any
+ intervening calls to release_recv_lock on release.
+*/
+static void release_recv_lock(fl_conn *release) {
+ stdmutex_drop(&release->recv_lock); /* UNLOCK RECV_LOCK */
+}
+
+static fl_group *add_group(fl_conn *conn, const char *group_name) {
+ fl_group *group;
+ char *group_name_ptr;
+
+ DEBUG(std_stkfprintf(stderr, 1, "add_group: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group_name));
+ group = create_fl_group(conn->private, group_name);
+ group_name_ptr = group->group;
+ stdhash_insert(&conn->groups, 0, &group_name_ptr, &group);
+ DEBUG(std_stkfprintf(stderr, -1, "add_group: return group('%s', %p)\n", group->group, group));
+ return group;
+}
+
+static void remove_group(fl_conn *conn, fl_group *group) {
+ stdit hit;
+ char *group_name_ptr;
+
+ DEBUG(std_stkfprintf(stderr, 1, "remove_group: mbox(%d, %p), group ('%s', %p)\n",
+ conn->mbox, conn, group->group, group));
+ group_name_ptr = group->group;
+ stdhash_find(&conn->groups, &hit, &group_name_ptr);
+ assert(!stdhash_is_end(&conn->groups, &hit) && group == *(fl_group**) stdhash_it_val(&hit));
+ stdhash_erase(&conn->groups, &hit);
+ free_fl_group(group);
+ DEBUG(std_stkfprintf(stderr, -1, "remove_group\n"));
+}
+
+static fl_group *get_group(fl_conn *conn, const char *grp) {
+ stdit hit;
+
+ return (!stdhash_is_end(&conn->groups, stdhash_find(&conn->groups, &hit, &grp)) ?
+ *(fl_group**) stdhash_it_val(&hit) : 0);
+}
+
+/* Pass null for bm if the data to be delivered is already in um. */
+/* Otherwise fill out a gc_buff_mess with the appropriate data and indicate whether */
+/* that buff mess is dynamically allocated or not. If it is, then if the message needs to */
+/* be buffered, then bm itself is buffered. If the dynamically allocated message doesn't */
+/* need to be buffered, then the msg is reclaimed (freed) before returning to the caller. */
+/* Returns a pointer to the gc_buff_mess that was buffered, null if none was buffered. */
+/* um can be null if you force buffering somehow. */
+static gc_buff_mess *deliver(fl_conn *conn, gc_recv_mess *um, gc_buff_mess *bm,
+ int bm_alloced) {
+ gc_buff_mess *ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver: mbox(%d, %p), um %p, bm %p, bm_alloced %d\n",
+ conn->mbox, conn, um, bm, (int) bm_alloced));
+ if (!um->delivered && stddll_empty(&conn->mess_queue) && (bm == 0 || buffm_to_userm(um, bm))) {
+ DEBUG(std_stkfprintf(stderr, 0, "able to deliver to user's parameters!\n"));
+ um->delivered = 1;
+ ret = 0;
+ if (bm != 0 && bm_alloced) {
+ DEBUG(std_stkfprintf(stderr, 0, "bm was alloc'ed -> freeing %p\n", bm));
+ free_gc_buff_mess(bm);
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "UNABLE to deliver to user's parameters: BUFFERING!\n"));
+ if (bm == 0 || !bm_alloced) { /* if I need to alloc a gc_buff_mess */
+ if ((ret = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+
+ if (bm == 0) { /* if data was contained in user parameters */
+ DEBUG(std_stkfprintf(stderr, 0, "data was in user's parameters copying to buff mess\n"));
+ userm_to_buffm(ret, um);
+ } else { /* else data was contained in bm */
+ DEBUG(std_stkfprintf(stderr, 0, "data was in bm, copying in to buff mess\n"));
+ *ret = *bm;
+ }
+ } else { /* buff mess was already alloc'ed and filled out */
+ DEBUG(std_stkfprintf(stderr, 0, "data in bm, but alloc'ed: stealing bm for buffer\n"));
+ ret = bm;
+ }
+ /* calculate total size of buffered message */
+ ret->total_size = sizeof(gc_buff_mess) + ret->num_groups * MAX_GROUP_NAME + ret->mess_len;
+ stddll_push_back(&conn->mess_queue, &ret);
+ conn->bytes_queued += ret->total_size;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "deliver: ret buff mess %p\n", ret));
+ return ret;
+}
+
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group) {
+ gc_buff_mess trans_sig = {}, *ret; /* sets everything to zero */
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver_trans_sig: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group));
+ trans_sig.mbox = conn->mbox;
+ trans_sig.serv_type = TRANSITION_MESS;
+ strncpy(trans_sig.sender, group, MAX_GROUP_NAME);
+ ret = deliver(conn, um, &trans_sig, 0);
+
+ DEBUG(std_stkfprintf(stderr, -1, "deliver_trans_sig: ret buff mess %p\n", ret));
+ return ret;
+}
+
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group) {
+ gc_buff_mess flush_req = {}, *ret; /* sets everything to zero */
+
+ DEBUG(std_stkfprintf(stderr, 1, "deliver_flush_req: mbox(%d, %p), group '%s'\n",
+ conn->mbox, conn, group));
+ flush_req.mbox = conn->mbox;
+ flush_req.serv_type = FLUSH_REQ_MESS;
+ strncpy(flush_req.sender, group, MAX_GROUP_NAME);
+ ret = deliver(conn, um, &flush_req, 0);
+
+ DEBUG(std_stkfprintf(stderr, -1, "deliver_flush_req: ret buff mess %p\n", ret));
+ return ret;
+}
+
+/* copy info from buffered msg to a user's parameters and make it presentable to the user */
+/* return 1 on successful copy to user, 0 on failure (incompatible user parameters) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm) {
+ int ret = 1;
+ scatp pos;
+ long err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n",
+ um, bm, bm->serv_type, bm->sender));
+
+ if (scatp_begin(&pos, um->scat_mess) == 0 &&
+ (err = scatp_cpy2(&pos, bm->mess, bm->mess_len)) >= 0) { /* user's scatter is legal */
+ um->mbox = bm->mbox;
+ *um->serv_type = bm->serv_type;
+ *um->mess_type = bm->mess_type;
+
+ if (err == bm->mess_len && bm->num_groups <= um->max_groups) { /* no buffer problem */
+ *um->num_groups = bm->num_groups;
+ strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+ memcpy(um->groups, bm->groups,
+ (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+ *um->endian_mismatch = bm->endian_mismatch;
+ um->ret = bm->mess_len;
+ } else { /* buffer problem */
+ DEBUG(std_stkfprintf(stderr, 0, "User buffer problem!\n"));
+ if ((um->orig_serv_type & DROP_RECV) == 0) { /* user didn't request DROP_RECV */
+ DEBUG(std_stkfprintf(stderr, 0, "User didn't request DROP_RECV\n"));
+ if (err != bm->mess_len) {
+ um->ret = BUFFER_TOO_SHORT;
+ *um->endian_mismatch = -bm->mess_len;
+ } else
+ *um->endian_mismatch = 0;
+
+ if (bm->num_groups > um->max_groups) {
+ um->ret = GROUPS_TOO_SHORT;
+ *um->num_groups = -bm->num_groups;
+ } else
+ *um->num_groups = 0;
+
+ ret = 0; /* error: buffer problem and DROP_RECV not requested */
+ } else { /* user requested DROP_RECV */
+ DEBUG(std_stkfprintf(stderr, 0, "User DID request DROP_RECV!\n"));
+ if (err != bm->mess_len)
+ um->ret = BUFFER_TOO_SHORT;
+
+ if (bm->num_groups > um->max_groups) {
+ um->ret = GROUPS_TOO_SHORT;
+ *um->num_groups = -bm->num_groups;
+ } else
+ *um->num_groups = bm->num_groups;
+
+ strncpy(um->sender, bm->sender, MAX_GROUP_NAME);
+ memcpy(um->groups, bm->groups,
+ (*um->num_groups >= 0 ? *um->num_groups : um->max_groups) * MAX_GROUP_NAME);
+ *um->endian_mismatch = bm->endian_mismatch;
+ }
+ }
+ } else { /* error: user's scat msg is illegal */
+ um->ret = ILLEGAL_MESSAGE;
+ ret = 0;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "buffm_to_userm: um %p, bm %p, serv 0x%X, sender '%s'\n",
+ um, bm, bm->serv_type, bm->sender));
+ return ret;
+}
+
+/* copy info from a user's parameters to a buffered message */
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um) {
+ bm->mbox = um->mbox;
+ bm->serv_type = *um->serv_type;
+ strncpy(bm->sender, um->sender, MAX_GROUP_NAME);
+ bm->mess_type = *um->mess_type;
+ bm->endian_mismatch = *um->endian_mismatch;
+
+ DEBUG(std_stkfprintf(stderr, 1, "userm_to_buffm: ret %d, vuln %d, delivered %d, serv 0x%X, "
+ "sender '%s'\n", um->ret, (int) um->vulnerable, (int) um->delivered,
+ *um->serv_type, um->sender));
+ /* if um isn't an error without a msg body and groups */
+ if (um->ret >= 0 || um->ret == GROUPS_TOO_SHORT || um->ret == BUFFER_TOO_SHORT) {
+ scatter *scat;
+ size_t groups_size;
+ scatp um_pos;
+ char *groups;
+ long err;
+
+ /* get groups info out of the msg: either in um->groups or um->new_grps */
+ get_groups_info(um, &bm->num_groups, (char(**)[MAX_GROUP_NAME]) &groups);
+ if ((groups_size = bm->num_groups * MAX_GROUP_NAME) != 0) {
+ if ((bm->groups = (char(*)[MAX_GROUP_NAME]) malloc(groups_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, groups_size);
+ memcpy(bm->groups, groups, groups_size);
+ } else
+ bm->groups = 0;
+
+ /* get scat info out of the msg: either in um->scat_mess or um->new_msg */
+ get_scat_info(um, &bm->mess_len, &scat);
+ if (bm->mess_len != 0) {
+ if ((bm->mess = (char*) malloc(bm->mess_len)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, bm->mess_len);
+
+ err = scatp_begin(&um_pos, scat);
+ assert(err == 0);
+ err = scatp_cpy1(bm->mess, &um_pos, bm->mess_len);
+ assert(err == bm->mess_len);
+ } else
+ bm->mess = 0;
+ } else {
+ stderr_abort("not sure about this path: if ever triggered think about it\n");
+ DEBUG(std_stkfprintf(stderr, 0, "copying an error message to a buffm: ret %d\n", um->ret));
+ bm->num_groups = 0;
+ bm->groups = 0;
+ bm->mess_len = um->ret;
+ bm->mess = 0;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "userm_to_buffm\n"));
+}
+
+static int FL_int_flush(fl_conn *conn, fl_group *group) {
+ int ret;
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_int_flush: mbox(%d, %p), group ('%s', %p)\n",
+ conn->mbox, conn, group->group, group));
+ /* ACHTUNG! You must be a non-leaving, AUTHORIZE member of a fl group to flush it */
+ if (group->mstate != LEAVING) {
+ if (group->vstate == AUTHORIZE) {
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AGREE: Actually sending "
+ "the FLUSH_OK_MESS!\n"));
+ group->vstate = AGREE;
+ ret = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_OK_MESS,
+ sizeof(group_id), (char*) &group->curr_change->memb_info->gid);
+
+ if (ret != sizeof(group_id)) {
+ if (ret != CONNECTION_CLOSED && ret != ILLEGAL_SESSION)
+ stderr_abort("(%s, %d): mbox %d: group %s: SP_multicast: unexpected error(%d)\n",
+ __FILE__, __LINE__, conn->mbox, group->group, ret);
+ } else
+ ret = 0; /* set to zero -> success */
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Group not in AUTHORIZE state!\n"));
+ ret = ILLEGAL_STATE;
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Group is in LEAVING state!\n"));
+ ret = ILLEGAL_GROUP;
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "FL_int_flush: ret %d\n", ret));
+ return ret;
+}
+
+static int FL_int_scat_multicast(mailbox mbox, service serv_type, const char *grp,
+ int num_recvrs, char recvrs[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *user_scat) {
+ fl_conn *conn;
+ fl_group *group = 0;
+ int grp_not_priv; /* is grp not a private group ? */
+ int ret, i;
+
+ int fix_scat = 0; /* did I modify the user's scat? */
+ scatter *scat = (scatter*) user_scat; /* get rid of const warnings */
+ scat_element senders_elem; /* for copy of user's element I might overwrite */
+ char copy_buf[sizeof(group_id) + sizeof(int16) + MAX_GROUP_NAME]; /* append buffer */
+ char *curr_ptr = copy_buf, *key; /* current pos in append buffer */
+
+ DEBUG( /* debug print out user's parameters */
+ std_stkfprintf(stderr, 1, "FL_int_scat_multicast: mbox %d, serv 0x%X, grp '%s', "
+ "mess_type %d, mess_len %ld\n", mbox, serv_type, grp,
+ mess_type, scat_capacity(scat));
+ std_stkfprintf(stderr, 0, "Recvrs(%d):\n", num_recvrs);
+ for (i = 0; i < num_recvrs; ++i) {
+ std_stkfprintf(stderr, 0, "\t'%s'\n", recvrs[i]);
+ });
+
+ if ((conn = lock_conn(mbox)) != 0) { /* LOCK CONNECTION */
+ if (num_recvrs < 0) { /* num_recvrs? */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal num_recvrs %d!\n", num_recvrs));
+ ret = ILLEGAL_PARAM;
+ } else if (IS_ILLEGAL_SEND_STYPE(serv_type)) { /* used reserved bits in serv_type */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved service bits\n"));
+ ret = ILLEGAL_SERVICE;
+ } else if (IS_ILLEGAL_SEND_MTYPE(mess_type)) { /* used a reserved mess type */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal use of reserved message type!\n"));
+ ret = ILLEGAL_MESSAGE_TYPE;
+ } else if (scat->num_elements < 0 || scat->num_elements > FL_MAX_SCATTER_ELEMENTS) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal scatter num_elements %d\n", scat->num_elements));
+ ret = ILLEGAL_MESSAGE;
+ } else if ((grp_not_priv = !is_private_group(grp)) && /* I allow sends to private groups */
+ ((group = get_group(conn, grp)) == 0 || group->mstate != JOINED)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal send to group of which not a member!\n"));
+ ret = ILLEGAL_GROUP; /* not a member of that group */
+ } else if (grp_not_priv && group->vstate == AGREE) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal send: state AGREE: group '%s'!\n", group->group));
+ ret = ILLEGAL_STATE;
+ } else { /* attempt to send msg: still need to check on recvrs for subgroup_casts */
+ if (grp_not_priv && is_vulnerable_mess(group, serv_type)) { /* mess type, current fl vid */
+ DEBUG(std_stkfprintf(stderr, 0, "Sending a vulnerable message!\n"));
+ fix_scat = 1;
+ memcpy(curr_ptr, &mess_type, sizeof(int16));
+ curr_ptr += sizeof(int16);
+ memcpy(curr_ptr, &group->fl_view->gid, sizeof(group_id));
+ curr_ptr += sizeof(group_id);
+ mess_type = VULNERABLE_MESS;
+ }
+ if (!Is_subgroup_mess(serv_type)) { /* normal multicast */
+ if (fix_scat) { /* actually change the scatter to reflect appendings */
+ senders_elem = scat->elements[scat->num_elements]; /* save user's element */
+ scat->elements[scat->num_elements].buf = copy_buf;
+ scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+ ++scat->num_elements;
+ }
+ ret = SP_scat_multicast(mbox, serv_type, grp, mess_type, scat);
+ } else { /* subgroup multicast */
+ DEBUG(std_stkfprintf(stderr, 0, "Sending a subgroupcast!\n"));
+ if (grp_not_priv) { /* check all recvrs were members of my current fl view */
+ stdhash *orig_membs = &group->fl_view->orig_membs;
+ stdit find;
+
+ for (i = 0; i < num_recvrs; ++i) {
+ key = (char*) (recvrs + i);
+ if (stdhash_is_end(orig_membs, stdhash_find(orig_membs, &find, &key))) { /* a recvr wasn't */
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+ break;
+ }
+ }
+ } else { /* allow subgroupcast to private group */
+ for (i = 0; i < num_recvrs; ++i) { /* require ALL "receivers" to be same as grp */
+ key = (char*) (recvrs + i);
+ if (strncmp(grp, key, MAX_GROUP_NAME) != 0) {
+ DEBUG(std_stkfprintf(stderr, 0, "Illegal recvr '%s' not in ref '%s'\n", key, grp));
+ break;
+ }
+ }
+ }
+ fix_scat = 1; /* subgroup msgs have reference group appended to msg */
+ strncpy(curr_ptr, grp, MAX_GROUP_NAME);
+ curr_ptr += MAX_GROUP_NAME;
+
+ /* actually change the scatter to reflect appendings */
+ senders_elem = scat->elements[scat->num_elements]; /* save user's element */
+ scat->elements[scat->num_elements].buf = copy_buf;
+ scat->elements[scat->num_elements].len = (int) (curr_ptr - copy_buf);
+ ++scat->num_elements;
+
+ if (i == num_recvrs) /* all recvrs were legal above */
+ ret = SP_multigroup_scat_multicast(mbox, serv_type, num_recvrs,
+ (const char(*)[MAX_GROUP_NAME]) recvrs,
+ mess_type, scat);
+ else
+ ret = ILLEGAL_RECEIVERS;
+ }
+ if (fix_scat) { /* I modified user's scatter: I need to restore it */
+ --scat->num_elements;
+ if (ret >= 0) { /* successful return: need to adjust err to ignore appendings */
+ if (ret >= scat->elements[scat->num_elements].len)
+ ret -= scat->elements[scat->num_elements].len;
+ else
+ stderr_abort("(%s, %d): mbox %d: serv 0x%X: group '%s': SP_multicast returned %d\n",
+ __FILE__, __LINE__, mbox, serv_type, grp, ret);
+ }
+ scat->elements[scat->num_elements] = senders_elem; /* restore modified elem */
+ }
+ }
+ unlock_conn(conn); /* UNLOCK CONNECTION */
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "FL ILLEGAL_SESSION(%d)\n", mbox));
+ ret = ILLEGAL_SESSION;
+ }
+ if (ret == CONNECTION_CLOSED || ret == ILLEGAL_SESSION)
+ FL_disconnect(mbox);
+
+ DEBUG(std_stkfprintf(stderr, -1, "FL_int_scat_multicast: ret %d, mbox %d\n", ret, mbox));
+ return ret;
+}
+
+/* try to read in a msg from the SP layer into m and make it presentable to the user
+ returns non-zero if user's parameters shouldn't be processed, but, instead returned
+*/
+static int FL_int_receive(gc_recv_mess *m) {
+ int alloced_groups = 0; /* boolean - did I alloc groups buffers? */
+ int alloced_buffer = 0; /* boolean - did I alloc a message buffer? */
+ int max_groups; /* max_groups to be passed to receive */
+ int orig_max_groups = m->max_groups; /* user's max_groups */
+ char (*groups)[MAX_GROUP_NAME] = m->groups; /* groups to be passed to receive */
+ scatter *scat = m->scat_mess; /* scatter to be passed to receive */
+ int success = 0; /* boolean - did it succeed? */
+
+ /* here I reserve space in groups for the destination group if it is a SUBGROUP_CAST msg */
+ if (m->max_groups != 0)
+ max_groups = m->max_groups - 1;
+ else
+ max_groups = 0;
+
+ /* don't use DROP_RECV semantics: flush needs to see entire data of msg: then we can drop */
+ *m->serv_type = (m->orig_serv_type & ~DROP_RECV);
+
+ DEBUG(std_stkfprintf(stderr, 1, "FL_int_receive: mbox %d, serv 0x%X, max groups %d, "
+ "grps %p, scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups,
+ groups, scat, scat_capacity(scat)));
+
+ /* assert that there aren't any alloc'ed receive buffers, the user's
+ max_groups was legal and m doesn't already have a msg */
+ assert(m->num_new_grps == 0 && m->new_msg.num_elements == 0 &&
+ orig_max_groups >= 0 && !m->delivered);
+
+ /* perform the first receive */
+ m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+ groups, m->mess_type, m->endian_mismatch, scat);
+
+ /* check for buffer errors: should be if buffers too short, I didn't use DROP_RECV */
+ if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT) {
+ DEBUG(std_stkfprintf(stderr, 0, "Buffer error after initial recv: ret %d, num_groups %d, "
+ "endian %d\n", m->ret, *m->num_groups, *m->endian_mismatch));
+
+ /* if the groups buffer was too short and the message was a SUBGROUP_CAST then the groups
+ buffer didn't have room for the destination group too: so decrement num_groups to reflect */
+ if (*m->num_groups < 0 && Is_subgroup_mess(*m->serv_type)) {
+ DEBUG(std_stkfprintf(stderr, 0, "GROUPS error, subgroup_mess: decrementing num_groups\n"));
+ --*m->num_groups;
+ }
+
+ /* if the user requested DROP_RECV, or if the user's groups was exactly big enough (because
+ I shrunk max_groups above) and there was no msg buffer error (reported by endian_mismatch)
+ or I can't tell if there was a buffer error because 3.12 didn't report that properly then
+ allocate the necessary buffers and re-receive - else return buffer error to user
+ */
+ if ((m->orig_serv_type & DROP_RECV) != 0 ||
+ (-*m->num_groups <= orig_max_groups &&
+ (*m->endian_mismatch == 0 || FL_SP_version() == (float) 3.12))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Either DROP_RECV or might be able to re-receive!\n"));
+
+ /* user's groups array actually was too small to contain the groups, so allocate some */
+ if (-*m->num_groups > orig_max_groups) { /* implies *m->num_groups is negative */
+ size_t byte_size;
+
+ max_groups = -*m->num_groups; /* set max_groups to be big enough */
+ byte_size = max_groups * MAX_GROUP_NAME; /* calculate necessary memory size */
+
+ DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was actually TOO SMALL %d < %d\n",
+ orig_max_groups, -*m->num_groups));
+
+ alloced_groups = 1; /* record that I alloc'ed group buffers */
+ m->num_new_grps = max_groups; /* record size of alloc'ed group buffers */
+
+ /* actually allocate a new groups array to be used in re-receive */
+ if ((groups = m->new_grps = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, byte_size);
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "User's GROUPS buff was big enough %d >= %d\n",
+ orig_max_groups, -*m->num_groups));
+ max_groups = orig_max_groups; /* groups will fit into user's original buffers */
+ }
+
+ /* now check and see if the user's msg buffer was too small: necessary size reported in
+ endian_mismatch in versions later than 3.12, 3.12 didn't do it but did do DROP_RECV */
+ if (*m->endian_mismatch < 0 || FL_SP_version() == (float) 3.12) {
+ DEBUG(std_stkfprintf(stderr, 0, "Endian mismatch reports msg buffer is too small\n"));
+ alloced_buffer = 1; /* record that I alloc'ed a mess buffer */
+ scat = &m->new_msg; /* point scat at my alloc'ed mess buffer */
+ scat->num_elements = 1;
+
+ /* figure out how big the mess buffer needs to be */
+ if (FL_SP_version() != (float) 3.12)
+ scat->elements[0].len = -*m->endian_mismatch; /* capacity of msg to be recvd */
+ else
+ scat->elements[0].len = 102400; /* endian_mismatch is buggy, use max msg size */
+
+ /* actually allocate the new mess buffer */
+ if ((scat->elements[0].buf = (char*) malloc(scat->elements[0].len)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, scat->elements[0].len);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "No msg buffer error reported\n"));
+
+ /* try to receive again (shouldn't fail now) and don't use DROP_RECV again */
+ *m->serv_type = (m->orig_serv_type & ~DROP_RECV);
+ DEBUG(std_stkfprintf(stderr, 0, "Re-receive: mbox %d, serv 0x%X, max groups %d, grps %p, "
+ "scat %p, scat_cap %ld\n", m->mbox, *m->serv_type, max_groups,
+ groups, scat, scat_capacity(scat)));
+
+ /* perform the re-receive */
+ m->ret = SP_scat_receive(m->mbox, m->serv_type, m->sender, max_groups, m->num_groups,
+ groups, m->mess_type, m->endian_mismatch, scat);
+
+ /* if we had a buffer problem then code above or SP is buggy -> abort */
+ if (m->ret == GROUPS_TOO_SHORT || m->ret == BUFFER_TOO_SHORT)
+ stderr_abort("(%s, %d): mbox %d: buggy SP_recv DROP_RECV ret %d: "
+ "max_groups %d: num_groups %d: scat_cap %ld: endian_mismatch %d\n",
+ __FILE__, __LINE__, m->mbox, m->ret, max_groups, *m->num_groups,
+ scat_capacity(scat), *m->endian_mismatch);
+ }
+ }
+ /* receive or re-receive succeeded */
+ if (m->ret >= 0) {
+ int not_end = 1; /* boolean - did I not seek to end of msg yet? */
+ scatp pos, user_pos;
+ long err;
+
+ /* successful recv: debug print out what I got! */
+ DEBUG(std_stkfprintf(stderr, 0, "Successful recv: ret %d, serv 0x%X, sender '%s', "
+ "num_groups %d, mess_type %d, endian %d, Groups:\n", m->ret,
+ *m->serv_type, m->sender, *m->num_groups,
+ *m->mess_type, *m->endian_mismatch);
+ for (err = 0; err < *m->num_groups; ++err) {
+ std_stkfprintf(stderr, 0, "\t`%s'\n", groups[err]);
+ });
+
+ /* subgroup and flush messages have appended data that needs to be extracted and removed */
+ if (Is_regular_mess(*m->serv_type)) {
+ if (Is_subgroup_mess(*m->serv_type)) { /* subgroup msgs have destination group appended */
+ char *ref_grp;
+
+ assert(m->ret >= MAX_GROUP_NAME); /* ensure fully recvd */
+ m->ret -= MAX_GROUP_NAME; /* correct to ignore the appendage */
+
+ err = scatp_set(&pos, scat, m->ret, SEEK_SET); /* seek to copy position */
+ assert(err == 0);
+ not_end = 0; /* pos was seeked to new end of msg */
+
+ /* copy out reference group: put after last used element in groups: should fit */
+ ref_grp = (char*) (groups + *m->num_groups);
+ err = scatp_cpy1(ref_grp, &pos, MAX_GROUP_NAME);
+ assert(err == MAX_GROUP_NAME);
+ ++*m->num_groups; /* increment num_groups to include destination group */
+ DEBUG(std_stkfprintf(stderr, 0, "Subgroupcast mess: Ref group is '%s'!\n", ref_grp));
+ }
+
+ /* all reserved FLUSH msg types have a group_id attached on end of msg */
+ if (IS_ILLEGAL_SEND_MTYPE(*m->mess_type)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd an internal flush mess of type %s\n",
+ *m->mess_type == FLUSH_OK_MESS ? "FLUSH_OK_MESS" :
+ (*m->mess_type == FLUSH_RECV_MESS ? "FLUSH_RECV_MESS" :
+ (*m->mess_type == VULNERABLE_MESS ? "VULNERABLE_MESS" :
+ "UNKNOWN TYPE!"))));
+ assert(m->ret >= sizeof(group_id)); /* ensure fully recvd */
+ m->ret -= sizeof(group_id); /* correct to ignore the appendage */
+
+ if (not_end) {
+ err = scatp_set(&pos, scat, m->ret, SEEK_SET); /* seek to copy position */
+ assert(err == 0);
+ not_end = 0; /* seeked to end */
+ } else {
+ err = scatp_jbackward(&pos, sizeof(group_id)); /* move back from end */
+ assert(err == sizeof(group_id));
+ }
+ err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id)); /* copy out vid */
+ assert(err == sizeof(group_id));
+
+ /* endian correct the vid if necessary */
+ if (*m->endian_mismatch != 0) {
+ stdflip32(m->dst_gid.id);
+ stdflip32(m->dst_gid.id + 1);
+ stdflip32(m->dst_gid.id + 2);
+ }
+ DEBUG(std_stkfprintf(stderr, 0, "Dst gid is %d %d %d!\n", m->dst_gid.id[0],
+ m->dst_gid.id[1], m->dst_gid.id[2]));
+
+ /* VULNERABLE msgs also have the original user's msg type appended */
+ if (*m->mess_type == VULNERABLE_MESS) {
+ assert(m->ret >= sizeof(int16)); /* ensure fully recvd */
+ m->ret -= sizeof(int16); /* correct to ignore the appendage */
+
+ m->vulnerable = 1; /* mark m as vulnerable */
+
+ /* here should already be seeked to end */
+ err = scatp_jbackward(&pos, sizeof(int16)); /* move back from end again */
+ assert(err == sizeof(int16));
+ err = scatp_cpy1((char*) m->mess_type, &pos, sizeof(int16)); /* cpy out mess type */
+ assert(err == sizeof(int16));
+
+ /* endian correct the message type if necessary */
+ if (*m->endian_mismatch != 0)
+ stdflip16(m->mess_type);
+
+ DEBUG(std_stkfprintf(stderr, 0, "VULNERABLE: Orig msg type is %d!\n", *m->mess_type));
+ }
+ }
+ } else if (Is_reg_memb_mess(*m->serv_type)) {
+ /* if its a regular membership message read the group id out of the msg */
+ assert(m->ret >= sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME); /* ensure received */
+ err = scatp_begin(&pos, scat);
+ assert(err == 0);
+ err = scatp_cpy1((char*) &m->dst_gid, &pos, sizeof(group_id)); /* copy out new gid */
+ assert(err == sizeof(group_id)); /* already endian corrected */
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a SP reg memb mess: New SP gid is %d %d %d\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ }
+
+ /* if I alloc'ed a msg buffer -> user probably requested DROP_RECV, need to copy from
+ alloc'ed buffers into user's buffers and set m->ret to an appropriate value */
+ if (alloced_buffer) {
+ m->new_msg.elements[0].len = m->ret; /* record size of user msg */
+ err = scatp_begin(&user_pos, m->scat_mess);
+ assert(err == 0);
+ err = scatp_begin(&pos, &m->new_msg);
+ assert(err == 0);
+
+ /* perform copy from alloc'ed scat to user's scat */
+ err = scatp_cpy0(&user_pos, &pos, m->ret);
+ assert(err >= 0 && err <= m->ret); /* shouldn't detect illegal scat here */
+
+ /* possible that removing appended data made the user's msg buffer big enough */
+ /* this also catches if I didn't really need to allocate a msg buffer (SP 3.12 bug) */
+ if (err != m->ret) { /* didn't all fit into user's msg buff */
+ DEBUG(std_stkfprintf(stderr, 0, "User msg buffer still too short!\n"));
+
+ /* 3.12: if !DROP_RECV, then report negative msg size correctly in endian_mismatch */
+ if ((m->orig_serv_type & DROP_RECV) == 0 && FL_SP_version() == (float) 3.12) {
+ DEBUG(std_stkfprintf(stderr, 0, "Alloced msg buff: SP 3.12 DROP_RECV bug: error!\n"));
+ success = -1; /* non-zero success indicates an error that should be returned now! */
+ *m->endian_mismatch = -m->ret;
+ }
+ m->ret = BUFFER_TOO_SHORT;
+ } else /* their buffer was big enough! */
+ DEBUG(std_stkfprintf(stderr, 0, "Unnecessary msg buff alloc! No BUFFER_TOO_SHORT!\n"));
+ }
+
+ /* if I alloc'ed groups buffers then user's buffers were too small and requested DROP_RECV */
+ if (alloced_groups) {
+ DEBUG(std_stkfprintf(stderr, 0, "Alloced groups buff, copying over: GROUPS_TOO_SHORT\n"));
+ memcpy(m->groups, m->new_grps, m->max_groups * MAX_GROUP_NAME);
+
+ /* if it is a subgroup msg and there is space put the destination group at end of groups */
+ if (Is_subgroup_mess(*m->serv_type) && m->max_groups > 0)
+ memcpy(m->groups[m->max_groups - 1], m->new_grps[*m->num_groups - 1], MAX_GROUP_NAME);
+
+ *m->num_groups = -*m->num_groups; /* report too short */
+ m->ret = GROUPS_TOO_SHORT;
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Error from recv or re-recv that needs to be returned\n"));
+ success = -1; /* non-zero success indicates an error that should be given to user now! */
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "SP_int_receive: success %d, ret %d, mbox %d, serv 0x%X, "
+ "mess_type %d\n", success, m->ret, m->mbox, *m->serv_type,
+ *m->mess_type));
+ return success;
+}
+
+/* state machine code */
+/* recv_and_handle: main director function: returns whether or not still have conn lock */
+static int recv_and_handle(fl_conn *conn, gc_recv_mess *m) {
+ fl_group *group;
+ int ret = 1; /* do I still have the conn lock? */
+
+ DEBUG(std_stkfprintf(stderr, 1, "recv_and_handle: mbox(%d, %p)\n", conn->mbox, conn));
+ release_conn_lock(conn); /* let go of conn lock so I don't block sends on this connection */
+
+ if (FL_int_receive(m) != 0) { /* if an immediate return recv error: return to user */
+ DEBUG(std_stkfprintf(stderr, 0, "immediate return from FL_int_receive %d\n", m->ret));
+ m->delivered = 1;
+ ret = 0;
+ } else if (!acquire_conn_lock(conn)) { /* couldn't acquire conn lock -> disconnecting */
+ DEBUG(std_stkfprintf(stderr, 0, "Couldn't reacquire conn lock -> disconnecting\n"));
+ m->ret = ILLEGAL_SESSION;
+ m->delivered = 1;
+ ret = 0;
+ } else if (Is_regular_mess(*m->serv_type)) { /* SP regular message */
+ char (*dst_grp)[MAX_GROUP_NAME]; /* group this message is intended for */
+ int num_groups;
+
+ /* for regular msgs dst group is the first group, for subgroup msgs it is the last group */
+ get_groups_info(m, &num_groups, &dst_grp);
+ if (Is_subgroup_mess(*m->serv_type))
+ dst_grp += num_groups - 1;
+
+ DEBUG(std_stkfprintf(stderr, 0, "A regular mess destined for group '%s'\n", dst_grp));
+ if ((group = get_group(conn, (char*) dst_grp)) != 0) { /* look up fl_group structure */
+ if (*m->mess_type == FLUSH_OK_MESS)
+ handle_recv_flush_ok(conn, group, m);
+ else if (*m->mess_type == FLUSH_RECV_MESS)
+ handle_recv_flush_recv(conn, group, m);
+ else
+ handle_recv_reg_mess(conn, group, m);
+ } else if (strncmp((char*) dst_grp, conn->private, MAX_GROUP_NAME) == 0) { /* my pgroup */
+ DEBUG(std_stkfprintf(stderr, 0, "Mess is addressed to my private group! Deliver!\n"));
+ deliver(conn, m, 0, 0); /* deliver msg immediately */
+ } else
+ stderr_msg("(%s, %d): WARNING: a regular msg for a group of which I'm not a member!\n"
+ "mbox %d: serv 0x%X: sender '%s': dst_grp '%s': mess_type %d\n", __FILE__,
+ __LINE__, m->mbox, *m->serv_type, m->sender, dst_grp, *m->mess_type);
+ } else if (Is_membership_mess(*m->serv_type)) { /* SP membership message */
+ group = get_group(conn, m->sender); /* look up fl_group structure: might not exist */
+
+ DEBUG(std_stkfprintf(stderr, 0, "A memb mess destined for group ('%s', %p)\n",
+ m->sender, group));
+ if (Is_reg_memb_mess(*m->serv_type))
+ handle_recv_reg_memb_mess(conn, group, m);
+ else if (Is_transition_mess(*m->serv_type))
+ handle_recv_trans_memb_mess(conn, group, m);
+ else if (Is_caused_leave_mess(*m->serv_type))
+ handle_recv_self_leave_memb_mess(conn, group, m);
+ else
+ stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected membership 0x%X: "
+ "group '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+ } else
+ stderr_abort("(%s, %d): mbox %d: recv_and_handle: unexpected service 0x%X: "
+ "sender '%s'\n", __FILE__, __LINE__, conn->mbox, *m->serv_type, m->sender);
+
+ DEBUG(std_stkfprintf(stderr, -1, "recv_and_handle: ret %d (have_conn_lock)\n", (int) ret));
+ return ret;
+}
+
+static void handle_recv_flush_ok(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ sp_memb_change *spc;
+ stdit hit;
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ /* see if I know about this flok's gid yet or not: if not create a sp_memb_change */
+ if (!stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for a known gid %d %d %d: spc %p\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+ assert(SP_equal_group_ids(m->dst_gid, spc->memb_info->gid));
+ } else {
+ spc = create_sp_memb_change(m->dst_gid);
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a flok for an UNKNWON gid %d %d %d: CREATED spc %p\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2], spc));
+ stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc); /* put into pmemb_hash */
+ }
+ stdhash_insert(&spc->flok_senders, 0, m->sender, 0); /* record flok from sender */
+ DEBUG(std_stkfprintf(stderr, 0, "curr_change(%p) ?= spc(%p) now has %lu floks, needs %d\n",
+ group->curr_change, spc, stdhash_size(&spc->flok_senders),
+ spc->memb_mess_recvd ? spc->memb_info->orig_num_membs : -1));
+ switch (group->vstate) {
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ if (spc == group->curr_change &&
+ stdhash_size(&spc->flok_senders) == spc->memb_info->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd last needed flok for curr change! INSTALL!\n"));
+ /* Received all the floks necessary to install the next fl view! */
+ install_new_view(conn, group, m); /* deliver an appropriate fl membership msg */
+
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state VERIFY!\n"));
+ group->vstate = VERIFY; /* update group state */
+ if (group->mstate == JOINING)
+ group->mstate = JOINED; /* has been installed in a group */
+
+ update_fl_view(group); /* update the view information for this group */
+
+ /* if no cascading membership: send a FLUSH_RECV, else deliver a FLUSH_REQ mess */
+ if (stddll_empty(&group->memb_queue)) {
+ int err;
+
+ DEBUG(std_stkfprintf(stderr, 0, "No cascading memberships -> sending FLUSH_RECV\n"));
+ err = SP_multicast(conn->mbox, FIFO_MESS, group->group, FLUSH_RECV_MESS,
+ sizeof(group_id), (char*) &group->fl_view->gid);
+
+ if (err == CONNECTION_CLOSED || err == ILLEGAL_SESSION) { /* immediate return error */
+ DEBUG(std_stkfprintf(stderr, 0, "SP_multicast failure %ld\n", err));
+ m->delivered = 1; /* return to user immediately */
+ m->ret = (int) err;
+ break;
+ } else if (err != sizeof(group_id))
+ stderr_abort("(%s, %d): SP_multicast unexpected return %d\n", __FILE__, __LINE__, err);
+ } else { /* buffer a FLUSH_REQ mess, update curr_change, etc. */
+ DEBUG(std_stkfprintf(stderr, 0, "Cascading Memberships(%lu) to handle!\n",
+ stddll_size(&group->memb_queue)));
+ handle_next_memb_change(conn, group, m);
+ }
+ /* if new members have already died (after they sent their floks): deliver TRANS */
+ if (stdhash_size(&group->fl_view->curr_membs) < group->fl_view->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "New view members have already died! Deliver TRANS!\n"));
+ assert(!group->fl_view->in_trans_memb);
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group); /* will buffer */
+ }
+ /* deliver any VULNERABLE messages that were pre-delivered */
+ stddll_begin(&group->mess_queue, &lit);
+ for (; !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Delivering a vulnerable SP pre-delivered msg!\n"));
+ deliver(conn, m, *(gc_buff_mess**) stddll_it_val(&lit), 1); /* will buffer */
+ }
+ stddll_clear(&group->mess_queue); /* empty postponed vulnerable queue */
+ } else
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ case AUTHORIZE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ case STEADY: case VERIFY:
+ assert(group->curr_change == 0);
+ assert(!spc->memb_mess_recvd ||
+ stdhash_size(&spc->flok_senders) < spc->memb_info->orig_num_membs);
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_ok: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_flush_recv(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ switch (group->vstate) {
+ case VERIFY:
+ assert(group->curr_change == 0);
+ if (SP_equal_group_ids(m->dst_gid, group->fl_view->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for curr FL vid need %d flush_recvs!\n",
+ group->fl_view->orig_num_membs));
+ if (++group->flush_recvs == group->fl_view->orig_num_membs) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got last FLUSH_RECV: going to state STEADY!\n"));
+ group->vstate = STEADY;
+ }
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in VERIFY!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ if (SP_equal_group_ids(m->dst_gid, group->curr_change->memb_info->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a FLUSH_RECV for CURR CHANGE need %d flush_recvs!\n",
+ group->fl_view->orig_num_membs));
+ ++group->flush_recvs;
+ assert(group->flush_recvs < group->curr_change->memb_info->orig_num_membs);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AGREE!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case STEADY:
+ assert(group->curr_change == 0);
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in STEADY!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ case AUTHORIZE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ DEBUG(std_stkfprintf(stderr, 0, "Ignoring FLUSH_RECV for vid %d %d %d while in AUTHORIZE!\n",
+ m->dst_gid.id[0], m->dst_gid.id[1], m->dst_gid.id[2]));
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_flush_recv: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_recv_reg_mess(fl_conn *conn, fl_group *group, gc_recv_mess *um) {
+ stdit hit;
+ gc_buff_mess *bm;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ switch (group->vstate) {
+ case AGREE:
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ /* if vulnerable message meant for the next membership from a member of the next one */
+ if (um->vulnerable) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a vulnerable msg!\n"));
+ if (SP_equal_group_ids(um->dst_gid, group->curr_change->memb_info->gid)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Addressed to my next FL vid!\n"));
+ if (!stdhash_is_end(&group->curr_change->memb_info->curr_membs, stdhash_find(&group->curr_change->memb_info->curr_membs,
+ &hit, &um->sender))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Recvd a vuln msg that needs to be POSTPONED!\n"));
+ if ((bm = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", __FILE__, __LINE__, sizeof(gc_buff_mess));
+
+ userm_to_buffm(bm, um);
+ stddll_push_back(&group->mess_queue, &bm);
+ break;
+ } else
+ stderr_abort("'%s' Not in my next FL curr membs!\n", um->sender);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Vuln msg not to next FL vid: should ignore below\n"));
+ } /* ELSE FALL THROUGH TO NORMAL CASES, BELOW (other case statements)!!!! */
+ case STEADY: case AUTHORIZE: case VERIFY:
+ /* if the message is from a current flush member then deliver it */
+ if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs, &hit, &um->sender))) {
+ DEBUG(std_stkfprintf(stderr, 0, "Deliver reg mess from group memb '%s'\n", um->sender));
+ deliver(conn, um, 0, 0);
+ } else
+ DEBUG(std_stkfprintf(stderr, 0, "Ignore reg mess from non group memb '%s'\n", um->sender));
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_mess: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_reg_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ sp_memb_change *spc, *new_curr_change;
+ stdhash leavers; /* leavers<char*, nil>: members that left since last SP memb */
+ stdit hit, flit;
+ stdit lit;
+ char (*groups)[MAX_GROUP_NAME];
+ int num_groups;
+ int fl_memb_shrunk = 0;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_reg_memb_mess: mbox(%d, %p), group('%s', %p), %s"
+ ", gid %d %d %d\n", conn->mbox, conn, m->sender, group,
+ group != 0 ? state_str(group->vstate) : "NO STATE", m->dst_gid.id[0],
+ m->dst_gid.id[1], m->dst_gid.id[2]));
+
+ if (group == 0) { /* no fl_group struct for this group yet -> create one and put in conn */
+ group = add_group(conn, m->sender);
+ DEBUG(std_stkfprintf(stderr, 0, "No group structure yet, CREATED group %p!\n", group));
+ }
+ /* if a sp_memb_change for this gid doesn't exist yet: create one and put in pmemb */
+ if (stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &m->dst_gid))) {
+ spc = create_sp_memb_change(m->dst_gid);
+ DEBUG(std_stkfprintf(stderr, 0, "Unknown gid: CREATED spc %p\n", spc));
+ stdhash_insert(&group->pmemb_hash, 0, &m->dst_gid, &spc);
+ } else {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ DEBUG(std_stkfprintf(stderr, 0, "KNOWN gid: got from pmemb_hash %p\n", spc));
+ }
+ get_groups_info(m, &num_groups, &groups); /* fill in rest of data for spc */
+ assert(!spc->memb_mess_recvd);
+ spc->memb_mess_recvd = 1; /* I got the memb mess for this gid */
+ spc->delta_member = determine_leavers(&leavers, group->sp_view, m); /* who left from last SP */
+ fill_view(spc->memb_info, *m->serv_type, num_groups, groups, *m->mess_type); /* spc view */
+
+ /* update the current fl_view: shrink the fl membership by removing any leavers */
+ for (stdhash_begin(&leavers, &hit); !stdhash_is_end(&leavers, &hit); stdhash_it_next(&hit)) {
+ if (!stdhash_is_end(&group->fl_view->curr_membs, stdhash_find(&group->fl_view->curr_membs,
+ &flit, stdhash_it_key(&hit)))) {
+ DEBUG(std_stkfprintf(stderr, 0, "FL member '%s' left in memb event, remove from fl view\n",
+ *(char**) stdhash_it_key(&hit)));
+ fl_memb_shrunk = 1; /* remember the membership shrunk */
+ stdhash_erase(&group->fl_view->curr_membs, &flit); /* remove from fl_view->curr_membs */
+ }
+ }
+ /* remove any memberships that this SP memb change invalidated */
+ age_and_invalidate_pmembs(group); /* age unknown spc's and remove them if too old */
+ new_curr_change = collapse_memberships(group, &leavers, spc); /* call before pushing spc!! */
+ stdhash_destruct(&leavers); /* after collapse members, leavers values could be invalid */
+
+ /* push spc on to end of memb queue and point sp view at spc's view */
+ /* spc must be pushed on to end of memb_queue after collapse memberships due to leave prob */
+ stddll_push_back(&group->memb_queue, &spc); /* push this SP memb change on to the memb queue */
+ group->sp_view = spc->memb_info; /* sp_view refers to most recent SP view */
+
+ /* if I wasn't working on a SP memb change or I just invalidated it -> update curr_change */
+ /* this will deliver a FLUSH_REQ message if I wasn't already working on a change */
+ if (group->curr_change != new_curr_change) {
+ DEBUG(std_stkfprintf(stderr, 0, "New SP memb change to work on now: %p -> %p!!\n",
+ group->curr_change, new_curr_change));
+
+ /* drop any pre-delivered messages as they must have come from partioned members */
+ for (stddll_begin(&group->mess_queue, &lit); !stddll_is_end(&group->mess_queue, &lit); stddll_it_next(&lit)) {
+ DEBUG(std_stkfprintf(stderr, 0, "Destroying pre-delivered msg from leaver member\n"));
+ free_gc_buff_mess(*(gc_buff_mess**) stddll_it_val(&lit));
+ }
+ stddll_clear(&group->mess_queue);
+ handle_next_memb_change(conn, group, m); /* possibly deliver a FLUSH_REQ */
+ }
+ /* if the group shrunk and I haven't delivered a trans yet -> deliver a trans signal */
+ if (!group->fl_view->in_trans_memb && fl_memb_shrunk) {
+ DEBUG(std_stkfprintf(stderr, 0, "FL memb shrunk, haven't delivered TRANS: deliver TRANS\n"));
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_reg_memb mess mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void handle_next_memb_change(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ stdit lit;
+ int err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ assert(!stddll_empty(&group->memb_queue));
+ /* special case for JOINING: don't have a curr change yet! */
+ assert((group->vstate == STEADY || group->vstate == VERIFY) ?
+ group->curr_change == 0 : group->mstate == JOINING || group->curr_change != 0);
+
+ group->curr_change = *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit));
+ assert(group->curr_change != 0 && group->curr_change->memb_mess_recvd);
+ group->flush_recvs = 0;
+
+ switch (group->vstate) {
+ case STEADY: case VERIFY:
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+ group->vstate = AUTHORIZE;
+ deliver_flush_req(conn, m, group->group);
+ break;
+ case AUTHORIZE: /* already waiting for a response from user */
+ break;
+ case AGREE:
+ DEBUG(std_stkfprintf(stderr, 0, "Going to state AUTHORIZE!\n"));
+ group->vstate = AUTHORIZE;
+ DEBUG(std_stkfprintf(stderr, 0, "AGREE: already auth'ed memb, send flok to new gid!\n"));
+ if ((err = FL_int_flush(conn, group)) != 0) { /* unrecoverable error */
+ DEBUG(std_stkfprintf(stderr, 0, "FL_int flush error: return to user\n"));
+ m->ret = err;
+ m->delivered = 1;
+ }
+ break;
+ default: stderr_abort("(%s, %d): impossible vstate %d\n", __FILE__, __LINE__, group->vstate);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_next_memb_change: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_trans_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m) {
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+ group->sp_view->in_trans_memb = 1;
+ if (!group->fl_view->in_trans_memb) {
+ DEBUG(std_stkfprintf(stderr, 0, "Got a TRANS for first time in FL view: deliver TRANS\n"));
+ group->fl_view->in_trans_memb = 1;
+ deliver_trans_sig(conn, m, group->group);
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_trans_memb_mess: mbox(%d, %p), group('%s', %p), "
+ "%s\n", conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void
+handle_recv_self_leave_memb_mess(fl_conn *conn, fl_group *group, gc_recv_mess *m){
+ DEBUG(std_stkfprintf(stderr, 1, "handle_recv_self_leave_memb_mess: mbox(%d, %p), "
+ "group('%s', %p), %s\n", conn->mbox, conn, group->group, group,
+ state_str(group->vstate)));
+ remove_group(conn, group);
+ deliver(conn, m, 0, 0);
+ DEBUG(std_stkfprintf(stderr, -1, "handle_recv_self_leave_memb_mess: mbox(%d, %p)\n",
+ conn->mbox, conn));
+}
+
+/* install the current view I was trying to finish */
+static void install_new_view(fl_conn *conn, const fl_group *group, gc_recv_mess *m) {
+ view *new_fl_view = group->curr_change->memb_info, *last_fl_view = group->fl_view;
+ int not_network = !Is_caused_network_mess(new_fl_view->memb_type);
+ char (*curr_vs)[MAX_GROUP_NAME];
+ gc_buff_mess *memb_mess;
+ size_t size, byte_size;
+
+ DEBUG(std_stkfprintf(stderr, 1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+
+ /* I create a gc_buff_mess and then deliver it to the user for to simplify msg construction */
+ if ((memb_mess = (gc_buff_mess*) malloc(sizeof(gc_buff_mess))) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", sizeof(gc_buff_mess));
+
+ memb_mess->mbox = conn->mbox; /* set mbox, service type, group, and num_groups */
+ memb_mess->serv_type = new_fl_view->memb_type;
+ assert(Is_reg_memb_mess(memb_mess->serv_type));
+ strncpy(memb_mess->sender, group->group, MAX_GROUP_NAME);
+ memb_mess->num_groups = new_fl_view->orig_num_membs;
+
+ byte_size = memb_mess->num_groups * MAX_GROUP_NAME; /* fill in the groups array */
+ if ((memb_mess->groups = (char(*)[MAX_GROUP_NAME]) malloc(byte_size)) == 0)
+ stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+ memcpy(memb_mess->groups, new_fl_view->membs_names, byte_size);
+
+ memb_mess->mess_type = new_fl_view->my_index; /* set mess_type and endian_mismatch */
+ memb_mess->endian_mismatch = 0;
+
+ if (not_network) { /* set the mess_len (return val) */
+ size = 1;
+ memb_mess->mess_len = sizeof(group_id) + sizeof(int) + MAX_GROUP_NAME;
+ } else {
+ size = stdhash_size(&last_fl_view->curr_membs);
+ byte_size = size * MAX_GROUP_NAME;
+ memb_mess->mess_len = sizeof(group_id) + sizeof(int) + byte_size;
+ }
+ if ((memb_mess->mess = (char*) malloc(memb_mess->mess_len)) == 0) /* allocate body of msg */
+ stderr_abort("(%s, %d): malloc(%u)\n", byte_size);
+
+ memcpy(memb_mess->mess, &new_fl_view->gid, sizeof(group_id)); /* fill in body: gid, num_vs */
+ memcpy(memb_mess->mess + sizeof(group_id), &size, sizeof(int));
+ curr_vs = (char(*)[MAX_GROUP_NAME]) (memb_mess->mess + sizeof(group_id) + sizeof(int));
+
+ /* fill in body of msg: vs_membs */
+ if (not_network)
+ memcpy(curr_vs, group->curr_change->delta_member, MAX_GROUP_NAME);
+ else {
+ char (*old_grps)[MAX_GROUP_NAME] = last_fl_view->membs_names;
+ char (*old_grps_end)[MAX_GROUP_NAME] = old_grps + last_fl_view->orig_num_membs;
+ stdit hit;
+
+ /* Step through the last fl_view's membs_names checking to see if each name is still */
+ /* in the membs (vs) set. This ensures the vs set is deterministically ordered. */
+ for (; old_grps != old_grps_end; ++old_grps)
+ if (!stdhash_is_end(&last_fl_view->curr_membs, stdhash_find(&last_fl_view->curr_membs, &hit, &old_grps)))
+ memcpy(curr_vs++, old_grps, MAX_GROUP_NAME);
+ }
+ deliver(conn, m, memb_mess, 1);
+ DEBUG(std_stkfprintf(stderr, -1, "install_new_view: mbox(%d, %p), group('%s', %p), %s\n",
+ conn->mbox, conn, group->group, group, state_str(group->vstate)));
+}
+
+static void update_fl_view(fl_group *group) {
+ stdit hit;
+ stdit lit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "update_fl_view: group('%s', %p), %s, old gid %d %d %d\n",
+ group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+ group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+ free_view(group->fl_view); /* delete old fl_view */
+ group->fl_view = group->curr_change->memb_info; /* steal curr_change's view */
+ group->curr_change->memb_info = 0; /* don't let fl_view be free'd below */
+
+ stddll_begin(&group->memb_queue, &lit);
+ assert(!stddll_is_end(&group->memb_queue, &lit) &&
+ group->curr_change == *(sp_memb_change**) stddll_it_val(&lit));
+ stddll_erase(&group->memb_queue, &lit); /* remove curr_change from head of memb_queue */
+ assert(stddll_is_end(&group->memb_queue, &lit) ||
+ group->curr_change != *(sp_memb_change**) stddll_it_val(&lit));
+
+ stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid);
+ assert(!stdhash_is_end(&group->pmemb_hash, &hit) &&
+ group->curr_change == *(sp_memb_change**) stdhash_it_val(&hit));
+ stdhash_erase(&group->pmemb_hash, &hit); /* remove curr_change from pmemb_hash */
+ assert(stdhash_is_end(&group->pmemb_hash, stdhash_find(&group->pmemb_hash, &hit, &group->fl_view->gid)));
+
+ free_sp_memb_change(group->curr_change); /* reclaim non-stolen parts */
+ group->curr_change = 0;
+ DEBUG(std_stkfprintf(stderr, -1, "update_fl_view: group('%s', %p), %s, new gid %d %d %d\n",
+ group->group, group, state_str(group->vstate), group->fl_view->gid.id[0],
+ group->fl_view->gid.id[1], group->fl_view->gid.id[2]));
+}
+
+static char*
+determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m) {
+ char delta[MAX_GROUP_NAME], *delta_ptr = delta, *ret = 0;
+ int num_vs, i, *num_vs_ptr = &num_vs;
+ scatter *scat;
+ scatp pos;
+ long err;
+
+ DEBUG(std_stkfprintf(stderr, 1, "determine_leavers: old gid %d %d %d -> new gid %d %d %d\n",
+ last_sp_view->gid.id[0], last_sp_view->gid.id[1],
+ last_sp_view->gid.id[2], m->dst_gid.id[0], m->dst_gid.id[1],
+ m->dst_gid.id[2]));
+
+ get_scat_info(m, &i, &scat); /* i is a dummy variable here */
+ if (!Is_caused_network_mess(*m->serv_type)) { /* join, leave, or disconnection */
+ DEBUG(std_stkfprintf(stderr, 0, "Not caused by network! "));
+ stdhash_construct(leavers, sizeof(char*), 0,
+ group_name_ptr_cmp, group_name_ptr_hashcode, 0);
+
+ err = scatp_set(&pos, scat, sizeof(group_id) + sizeof(int), SEEK_SET);
+ assert(err == 0);
+ err = scatp_cpy1(delta, &pos, MAX_GROUP_NAME); /* read out joiner/leaver/disconnector */
+ assert(err == MAX_GROUP_NAME);
+ delta[MAX_GROUP_NAME - 1] = 0; /* ensure null termination */
+
+ if ((ret = (char*) malloc(MAX_GROUP_NAME)) == 0)
+ stderr_abort("(%s, %d): malloc(%d)\n", __FILE__, __LINE__, MAX_GROUP_NAME);
+ strncpy(ret, delta, MAX_GROUP_NAME); /* strndup delta member for return */
+
+ DEBUG(std_stkfprintf(stderr, 0, "delta memb: '%s'\n", delta));
+ if (!Is_caused_join_mess(*m->serv_type)) { /* insert leaver into leavers */
+ DEBUG(std_stkfprintf(stderr, 0, "Wasn't a JOIN: adding delta to leavers\n"));
+ stdhash_insert(leavers, 0, &ret, 0);
+ }
+ } else {
+ DEBUG(std_stkfprintf(stderr, 0, "Caused by network! "));
+ stdhash_copy_construct(leavers, &last_sp_view->orig_membs); /* copy last SP membership */
+
+ err = scatp_set(&pos, scat, sizeof(group_id), SEEK_SET);
+ assert(err == 0);
+ err = scatp_adv_cpy1((char**) &num_vs_ptr, &pos, sizeof(int), 0, 1); /* read out num_vs */
+ assert(err == sizeof(int));
+
+ DEBUG(std_stkfprintf(stderr, 0, "Num_vs %d\n", num_vs));
+ for (i = 0; i < num_vs; ++i) { /* remove members that came with me: leaves who left */
+ err = scatp_adv_cpy1(&delta_ptr, &pos, MAX_GROUP_NAME, 0, 1);
+ assert(err == MAX_GROUP_NAME);
+ delta[MAX_GROUP_NAME - 1] = 0; /* ensure null termination */
+ err = stdhash_size(leavers);
+ stdhash_erase_key(leavers, &delta_ptr);
+ assert(err == stdhash_size(leavers) + 1);
+ }
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "determine_leavers: delta('%s', %p), leavers size is %lu\n",
+ ret != 0 ? ret : "", ret, stdhash_size(leavers)));
+ return ret; /* strndup'ed delta member or null */
+}
+
+static void age_and_invalidate_pmembs(fl_group *group) {
+ sp_memb_change *spc;
+ stdit hit;
+
+ DEBUG(std_stkfprintf(stderr, 1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+ group->group, group, state_str(group->vstate)));
+
+ stdhash_begin(&group->pmemb_hash, &hit);
+ for (; !stdhash_is_end(&group->pmemb_hash, &hit); stdhash_it_next(&hit)) {
+ spc = *(sp_memb_change**) stdhash_it_val(&hit);
+ if (!spc->memb_mess_recvd && ++spc->memb_change_age > FLOK_AGE_LIMIT) {
+ DEBUG(std_stkfprintf(stderr, 0, "destroy too old spc %p, %d %d %d\n", spc,
+ spc->memb_info->gid.id[0], spc->memb_info->gid.id[1],
+ spc->memb_info->gid.id[2]));
+ stdhash_erase(&group->pmemb_hash, &hit);
+ free_sp_memb_change(spc);
+ break; /* only one memb can become too old per SP memb event */
+ }
+ }
+ DEBUG(std_stkfprintf(stderr, -1, "age_and_invalidate_pmembs: group('%s', %p), %s\n",
+ group->group, group, state_str(group->vstate)));
+}
+
+static sp_memb_change*
+collapse_memberships(fl_group *group, stdhash *leavers, sp_memb_change *c) {
+ stdit leavers_it, spc_it, flok_it, pmemb_it;
+ int invalidating = 0;
+ sp_memb_change *spc;
+ stdit lit;
+ char *leaver;
+
+ DEBUG(std_stkfprintf(stderr, 1, "collapse_memberships: group('%s', %p), %s, num_leavers %lu\n",
+ group->group, group, state_str(group->vstate), stdhash_size(leavers)));
+ if (!stddll_empty(&group->memb_queue) && !stdhash_empty(leavers)) {
+ for (stddll_begin(&group->memb_queue, &lit); !stddll_is_end(&group->memb_queue, &lit); ) { /* loop SP membs */
+ spc = *(sp_memb_change**) stddll_it_val(&lit);
+
+ stdhash_begin(leavers, &leavers_it); /* loop over leavers */
+ for (; !stdhash_is_end(leavers, &leavers_it); stdhash_it_next(&leavers_it)) {
+ leaver = *(char**) stdhash_it_key(&leavers_it);
+
+ /* see if a leaver is a member of this SP memb change */
+ if (!stdhash_is_end(&spc->memb_info->curr_membs, stdhash_find(&spc->memb_info->curr_membs, &spc_it, &leaver))) {
+ DEBUG(std_stkfprintf(stderr, 0, "leaver '%s' is in a pending membership!\n", leaver));
+ /* if leaver hasn't sent his flok yet, then invalidate this sp_memb_change: NO & !!! */
+ if (stdhash_is_end(&spc->flok_senders, stdhash_find(&spc->flok_senders, &flok_it, leaver))) {
+ DEBUG(std_stkfprintf(stderr, 0, "leaver didn't send flok: collapsing membship!\n"));
+ invalidating = 1;
+ stdhash_find(&group->pmemb_hash, &pmemb_it, &spc->memb_info->gid);
+ assert(!stdhash_is_end(&group->pmemb_hash, &pmemb_it));
+ stdhash_erase(&group->pmemb_hash, &pmemb_it); /* removes spc from pmemb hash */
+ stddll_erase(&group->memb_queue, &lit); /* removes spc from memb_queue and advances lit */
+ free_sp_memb_change(spc);
+ break; /* break out of inner loop: move on to next spc */
+ } else /* remove leaver from &spc->memb_info->curr_membs because he left */
+ stdhash_erase(&spc->memb_info->curr_membs, &spc_it);
+ }
+ }
+ if (stdhash_is_end(leavers, &leavers_it)) { /* didn't break out of loop due to an invalidation */
+ if (invalidating) { /* did invalidate some spc's previous to this one */
+ DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates here: CAUSED_BY_NETWORK\n"));
+ invalidating = 0;
+ spc->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK; /* collapse them here */
+ }
+ stddll_it_next(&lit); /* advance lit to next sp_memb_change */
+ }
+ }
+ if (invalidating) { /* invalidated last few on the memb queue: collapse them into c */
+ DEBUG(std_stkfprintf(stderr, 0, "collapsing invalidates into current spc\n"));
+ c->memb_info->memb_type = REG_MEMB_MESS | CAUSED_BY_NETWORK;
+ }
+ }
+ spc = (stddll_empty(&group->memb_queue) ? c :
+ *(sp_memb_change**) stddll_it_val(stddll_begin(&group->memb_queue, &lit)));
+
+ DEBUG(std_stkfprintf(stderr, -1, "collapse_memberships: curr change %p, group('%s', %p), %s\n",
+ spc, group->group, group, state_str(group->vstate)));
+ return spc;
+}
+
+static void get_groups_info(const gc_recv_mess *m, int *num_groups,
+ char (**groups)[MAX_GROUP_NAME]) {
+ if (m->num_new_grps == 0) {
+ *num_groups = *m->num_groups;
+ *groups = m->groups;
+ } else {
+ *num_groups = m->num_new_grps;
+ *groups = m->new_grps;
+ }
+}
+
+static void get_scat_info(const gc_recv_mess *m, int *mess_len,
+ scatter **scat_mess) {
+ if (m->new_msg.num_elements == 0) {
+ *mess_len = m->ret;
+ *scat_mess = (scatter*) m->scat_mess;
+ } else {
+ *mess_len = m->new_msg.elements[0].len;
+ *scat_mess = (scatter*) &m->new_msg;
+ }
+}
+
+/* is a group name a private group name or not? */
+static int is_private_group(const char group[MAX_GROUP_NAME]) {
+ const char *end = group + MAX_GROUP_NAME;
+
+ while (group != end && *group != 0 && *group != '#')
+ ++group;
+
+ return group != end && *group != 0;
+}
+
+/* check if a msg wrt a particular group is vulnerable or not */
+static int is_vulnerable_mess(const fl_group *group, service serv_type) {
+ /* need to include AUTHORIZE state because of coming here directly from the VERIFY state */
+ return ((group->vstate == AUTHORIZE || group->vstate == VERIFY) &&
+ (serv_type & (UNRELIABLE_MESS | RELIABLE_MESS | FIFO_MESS)) != 0);
+}
+
+static const char *state_str(fl_view_state state) {
+ switch (state) {
+ case STEADY: return "STEADY";
+ case AUTHORIZE: return "AUTHORIZE";
+ case AGREE: return "AGREE";
+ case VERIFY: return "VERIFY";
+ default: return "UNKOWN STATE!!!\n";
+ }
+}
+
+static float FL_SP_version(void) {
+#ifdef SPREAD_VERSION
+ int major, minor, patch, divBy;
+
+ SP_version(&major, &minor, &patch);
+
+ if (minor < 100)
+ divBy = 100;
+ else
+ divBy = 1000;
+
+ return major + (float) minor / divBy;
+#else
+ return SP_version();
+#endif
+}
+
+/* hashing fcns for group names */
+static stdbool group_name_cmp(const void *str1, const void *str2) {
+ return strncmp((const char*) str1, (const char*) str2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_hashcode(const void *str) {
+ const stduint8 *key = (const stduint8*) str, *end = key + MAX_GROUP_NAME;
+ size_t ret = 0;
+
+ while (key != end && *key != 0)
+ ret = ret * 33 + *key++;
+
+ return ret;
+}
+
+static stdbool group_name_ptr_cmp(const void *strptr1, const void *strptr2) {
+ return strncmp(*(const char**) strptr1, *(const char**) strptr2, MAX_GROUP_NAME);
+}
+
+static size_t group_name_ptr_hashcode(const void *strptr) {
+ return group_name_hashcode(*(const void**) strptr);
+}
+
+static stdbool group_id_cmp(const void *gid1, const void *gid2) {
+ return memcmp(gid1, gid2, sizeof(group_id));
+}
+
+static size_t group_id_hashcode(const void *gid) {
+ group_id *gid2 = (group_id*) gid;
+
+ return (gid2->id[0] ^ gid2->id[1] ^ gid2->id[2]);
+}
+
Added: trunk/flush/fl_p.h
===================================================================
--- trunk/flush/fl_p.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_p.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,336 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+#define flush_p_h_2000_04_24_14_16_31_jschultz_at_cnds_jhu_edu
+
+/* In this library I assume that the stdutil library I am linking with
+ was compiled with the -D STD_USE_EXCEPTIONS flag. This allows me to
+ assume that all reasonable calls to stdutil fcns won't fail, and if
+ they do (due to a system problem, such as malloc failing) they abort
+ with an appropriate error message, line number, etc.
+*/
+#include <fl.h>
+#include <scatp.h>
+#include <stdutil/stddefines.h>
+#include <stdutil/stderror.h>
+#include <stdutil/stdthread.h>
+#include <stdutil/stddll.h>
+#include <stdutil/stdhash.h>
+
+#define FL_MAJOR_VERSION 1
+#define FL_MINOR_VERSION 0
+#define FL_PATCH_VERSION 3
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The type fl_view_state specifies the possible states a connection
+ can be in, in reference to a particular group's membership algorithm.
+
+ STEADY - normal operation, member of the group
+ AUTHORIZE - a SP lvl memb event has occured, user needs to flush this group
+ AGREE - user has flushed this group, collecting floks for next fl view
+ VERIFY - new flush view installed, protect against non-causal msgs
+*/
+typedef enum { STEADY, AUTHORIZE, AGREE, VERIFY } fl_view_state;
+
+/* The type member_join_state specifies the possible states a connection
+ can be in, in reference to being a member of a particular group. Only
+ JOINED connections are allowed to leave and send to a group.
+
+ JOINING - this connection is in the process of joining this group
+ JOINED - this connection has been installed in a fl view of this group
+ LEAVING - this connection is leaving this group
+*/
+typedef enum { JOINING, JOINED, LEAVING } member_join_state;
+
+/* A gc_buff_mess contains all of the information necessary to replicate a message. */
+/* This is the struct I use when I have to buffer messages. */
+typedef struct {
+ mailbox mbox;
+ service serv_type;
+ char sender[MAX_GROUP_NAME];
+ int num_groups;
+ char (*groups)[MAX_GROUP_NAME];
+ int16 mess_type;
+ int endian_mismatch;
+ int mess_len;
+ char *mess;
+ int total_size; /* used for reporting number of bytes buffered: set by deliver() */
+} gc_buff_mess;
+
+/* A gc_recv_mess contains references to user's parameters from a call to receive. */
+/* I use this struct to pass data around in the internal state machine code. */
+/* The struct member "delivered" indicates whether or not the user's parameters have been */
+/* filled with legal return values and is currently returnable to the user or not. */
+typedef struct {
+ mailbox mbox;
+ service orig_serv_type; /* user's original service type request to receive */
+ service *serv_type;
+ char *sender;
+ int max_groups;
+ int *num_groups;
+ char (*groups)[MAX_GROUP_NAME];
+ int16 *mess_type;
+ int *endian_mismatch;
+ scatter *scat_mess;
+ int ret; /* return value from receive call */
+
+ int delivered; /* does this recv_mess already contain a delivered msg? */
+ int vulnerable; /* is this msg vulnerable? */
+ group_id dst_gid; /* the view id this message is addressed to */
+
+ int num_new_grps; /* potentially alloc'ed buffers for DROP_RECV msgs */
+ char (*new_grps)[MAX_GROUP_NAME];
+ scatter new_msg;
+} gc_recv_mess;
+
+/* This struct contains information that specifies a group membership view. */
+typedef struct {
+ group_id gid;
+ service memb_type;
+ int in_trans_memb; /* has a transitional msg been delivered in this view? */
+ int orig_num_membs;
+ char (*membs_names)[MAX_GROUP_NAME];
+ stdhash orig_membs, curr_membs; /* <char*, nil>, ptrs into membs_names */
+ int16 my_index; /* index of this connection in membs_names */
+} view;
+
+/* This struct contains information on a spread level membership
+ change for a particular group that has not yet been fully handled
+ by the flush layer. This structure is used by the fl memb protocol
+ while it attempts to install a view. These structures are built in
+ two different scenarios: (1) a FLUSH_OK or (2) a spread membership
+ message is received for a previously unseen/unknown view id.
+
+ There are 2 states that this structure can be in. These two states
+ correspond to whether or not a spread membership message for the
+ particular view id has been received yet or not. Which state the
+ structure is in is indicated by the struct member memb_mess_recvd.
+
+ The struct member memb_change_age indicates how many spread
+ membership messages have been received in this group since this
+ structure was built. If this value exceeds the value FLOK_AGE_LIMIT
+ and !memb_mess_recvd, then I take this as an indication that the
+ spread membership message that this structure is waiting for was
+ already delivered in a view of which I wasn't a member. In that
+ case, I will never receive that membership message and I can
+ "safely" discard this struct, as that membership change was already
+ handled by other members (this assumption is not 100% correct as
+ FIFO messages can jump before an unbounded number of memberships;
+ however, I feel that the probability of a FIFO message jumping
+ before FLOK_AGE_LIMIT memberships is vanishingly small).
+
+ If !memb_mess_recvd then the only fields that contain meaningful data
+ are memb_change_age, flok_senders, and memb_info->gid.
+
+ If memb_mess_recvd then delta_member contains either (1) a
+ malloc'ed copy of the delta member (if the membership change was
+ caused by join/leave/disconnect) or (2) null. All of the info in
+ memb_info is appropriately filled in when in this state.
+
+ The struct member flok_senders contains by value (ie - alloc'ed by
+ flok_senders) members' names who have flok'ed to this view_id.
+*/
+typedef struct {
+ view *memb_info;
+ int memb_mess_recvd;
+ int memb_change_age;
+ char *delta_member;
+ stdhash flok_senders; /* <char[MAX_GROUP_NAME], nil> */
+ group_id gid; /* temporary: erase me */
+} sp_memb_change;
+
+/* This struct contains all of the group level context for a connection in
+ reference to a particular group.
+
+ group_name - the name of the group with which this information is associated
+ mstate - current join state this member is in
+ vstate - current fl memb protocol state this member is in
+ curr_change - points to current sp_memb_change fl is trying to resolve (head of memb_queue)
+ sp_view - my most recently installed spread membership view (initially just me)
+ fl_view - my most recently installed flush membership view (initially just me)
+ flush_recvs - a count of how many flush_recvs have been recvd for fl_view
+ mess_queue - a queue of gc_buff_mess*s to be delivered later, in order
+ memb_queue - a queue of sp_memb_change*s to be handled in order
+ pmemb_hash - sp_memb_change*s that are being handled currently
+
+ sp_view either points at fl_view or at the most recent pending SP
+ memb event's view in memb_queue (see handle_recv_reg_memb_mess)
+
+ fl_view steals curr_change's view when curr_change is installed (see update_fl_view)
+*/
+typedef struct {
+ char group[MAX_GROUP_NAME];
+
+ member_join_state mstate;
+ fl_view_state vstate;
+
+ sp_memb_change *curr_change;
+
+ view *sp_view;
+ view *fl_view;
+ int flush_recvs;
+
+ stddll mess_queue; /* <gc_buff_mess*> */
+ stddll memb_queue; /* <sp_memb_change*> */
+ stdhash pmemb_hash; /* <group_id, sp_memb_change*> */
+} fl_group;
+
+/* This struct contains all of the connection level context for a connection */
+typedef struct {
+ stdmutex reserve_lock; /* protects reservations, disconnecting; destroy_cond's mutex */
+ size_t reservations; /* count of reservations on this connection */
+ int disconnecting; /* is this connection being disconnected? */
+ stdcond destroy_cond; /* disconnector thread waits on this until reservations == 0 */
+
+ stdmutex recv_lock; /* used to serialize calls to FL_recv on a connection */
+ stdmutex conn_lock; /* lock for access to the connection's data, see below */
+
+ /* acquiring conn_lock allows a thread to examine everything below here */
+ mailbox mbox;
+ int priority;
+ int group_memb;
+ char daemon[MAX_GROUP_NAME];
+ char user[MAX_GROUP_NAME];
+ char private[MAX_GROUP_NAME];
+
+ /* acquiring conn_lock allows a thread to examine and modify everything below here */
+ stdhash groups; /* information on groups involved in: <char*, fl_group*> */
+ stddll mess_queue; /* messages that the user can read out: <gc_buff_mess*> */
+ int bytes_queued; /* number of bytes available in mess_queue, used by FL_poll */
+} fl_conn;
+
+/************************* Private Variables, Functions and Macros *****************************/
+
+/* leave in or take out debugging printf's and checks? */
+#ifdef NDEBUG
+# define DEBUG(statement)
+#else
+# define DEBUG(statement) statement
+#endif
+
+/* age limit for a sp_memb_change with no sp memb mess */
+#define FLOK_AGE_LIMIT 200
+
+#define IS_ILLEGAL_SEND_STYPE(serv_type) \
+(((serv_type) & (FLUSH_REQ_MESS | MEMBERSHIP_MESS | ENDIAN_RESERVED | RESERVED)) != 0)
+
+/* mess types reserved by flush layer indicating special flush messages */
+/* MAKE SURE THAT FL_MIN_LEGAL_MESS_TYPE (in fl.h) IS ONE MORE THAN THE HIGHEST RESERVED MESS TYPE!!! */
+#define FLUSH_OK_MESS ((int16) -32768)
+#define FLUSH_RECV_MESS ((int16) -32767)
+#define VULNERABLE_MESS ((int16) -32766)
+#define IS_ILLEGAL_SEND_MTYPE(mess_type) ((mess_type) < FL_MIN_LEGAL_MESS_TYPE)
+
+/* structure routines */
+static fl_group *create_fl_group(const char *conn_name, const char *group_name);
+static void free_fl_group(fl_group *group);
+
+static void free_gc_buff_mess(gc_buff_mess *msg);
+
+static sp_memb_change *create_sp_memb_change(group_id gid);
+static void free_sp_memb_change(sp_memb_change *change);
+
+static view *create_view(group_id gid);
+static void free_view(view *v);
+static void fill_view(view *v, service memb_type, int num_membs,
+ char (*membs)[MAX_GROUP_NAME], int16 index);
+
+/* interface for reserving/locking connections */
+static fl_conn *lock_conn(mailbox mbox); /* reserve and lock connection */
+static void unlock_conn(fl_conn *conn);
+static fl_conn *make_reservation(mailbox mbox); /* declare interest in connection */
+static void cancel_reservation(fl_conn *conn);
+static int acquire_conn_lock(fl_conn *conn); /* get rights to read/modify */
+static void release_conn_lock(fl_conn *release);
+static int acquire_recv_lock(fl_conn *conn); /* get rights to enter FL_recv */
+static void release_recv_lock(fl_conn *release);
+
+/* interface for groups for a connection */
+static fl_group *add_group(fl_conn *conn, const char *group);
+static void remove_group(fl_conn *conn, fl_group *group);
+static fl_group *get_group(fl_conn *conn, const char *grp);
+
+/* functions for trying to deliver/return msgs to the user */
+static gc_buff_mess *deliver(fl_conn *c, gc_recv_mess *um, gc_buff_mess *bm, int al);
+static gc_buff_mess *deliver_trans_sig(fl_conn *conn, gc_recv_mess *um, char *group);
+static gc_buff_mess *deliver_flush_req(fl_conn *conn, gc_recv_mess *um, char *group);
+
+/* converting back and forth between user's variables and buffered messages (dst, src) */
+static int buffm_to_userm(gc_recv_mess *um, const gc_buff_mess *bm);
+static void userm_to_buffm(gc_buff_mess *bm, const gc_recv_mess *um);
+
+/* complex "low level" internal fcns that deal with SP layer */
+static int FL_int_flush(fl_conn *conn, fl_group *group);
+static int FL_int_scat_multicast(mailbox m, service s, const char *g,
+ int nr, char r[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat);
+static int FL_int_receive(gc_recv_mess *m);
+
+/* state machine fcns: handler functions */
+static int recv_and_handle(fl_conn *c, gc_recv_mess *m);
+static void handle_recv_flush_ok(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_flush_recv(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_reg_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_next_memb_change(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_trans_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+static void handle_recv_self_leave_memb_mess(fl_conn *c, fl_group *g, gc_recv_mess *m);
+
+/* helper functions for handler functions */
+static void install_new_view(fl_conn *c, const fl_group *g, gc_recv_mess *m);
+static void update_fl_view(fl_group *group);
+static char *determine_leavers(stdhash *leavers, view *last_sp_view, gc_recv_mess *m);
+static void age_and_invalidate_pmembs(fl_group *group);
+static sp_memb_change *collapse_memberships(fl_group *group, stdhash *leavers,
+ sp_memb_change *c);
+/* miscellaneous functions */
+/* where is full data for a msg (ie - groups or new_grps?) due to DROP_RECV considerations */
+static void get_groups_info(const gc_recv_mess *m, int *num_groups,
+ char (**groups)[MAX_GROUP_NAME]);
+static void get_scat_info(const gc_recv_mess *m, int *mess_len, scatter **scat_mess);
+
+static int is_private_group(const char *group);
+static int is_vulnerable_mess(const fl_group *group, service serv_type);
+
+static const char *state_str(fl_view_state state);
+
+static float FL_SP_version(void);
+
+/* group name hash fcns: used by stdhashs where the keys are group_names */
+static stdbool group_name_cmp(const void *str1, const void *str2);
+static size_t group_name_hashcode(const void *str);
+static stdbool group_name_ptr_cmp(const void *str_ptr1, const void *str_ptr2);
+static size_t group_name_ptr_hashcode(const void *str_ptr);
+static stdbool group_id_cmp(const void *gid1, const void *gid2);
+static size_t group_id_hashcode(const void *gid);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
Added: trunk/flush/fl_time_memb.c
===================================================================
--- trunk/flush/fl_time_memb.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/fl_time_memb.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,261 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char sender[MAX_GROUP_NAME];
+int num_groups;
+char groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16 mess_type;
+int endian_mismatch;
+int mess_len;
+char mess[MY_MAX_MESS_SIZE];
+int more_messes;
+
+char daemon_name[MAX_GROUP_NAME] = "4803@localhost";
+char user_name[MAX_GROUP_NAME] = "1";
+char priv_name[MAX_GROUP_NAME];
+char group_name[MAX_GROUP_NAME] = "test";
+
+char *exe = 0; /* executable's name */
+int user_type = -1; /* user type of application */
+int num_joins_leaves = 0; /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0; /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1; /* should there be sleeps between memberships? Ususally yes. */
+int pretty_print = 1; /* should the output be verbose and labeled for human consumption? */
+
+/* the results of the scenarios timings */
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+stats_results fl_join_stats, fl_leave_stats, fl_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+ return fprintf(outstream,
+ "Usage: %s\r\n"
+ "\t[-S <group size>] : # members in group (for stats; also login name)\r\n"
+ "\t[-s <address>] : spread daemon name - either port or port@machine\r\n"
+ "\t[-g <group name>] : group name to join\r\n"
+ "\t[-t <user type> <# joins/leaves>] : type of user "
+ "(LOAD_MEMBER = %d, DELTA = %d), # of join/leave events\r\n"
+ "\t[-f] : don't sleep between memberships (default is to sleep)\r\n"
+ "\t[-r] : print raw stats w/ no pretty headings\r\n",
+ exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+ for (++argv, --argc; argc > 0; ++argv, --argc) {
+ if (!strcmp(*argv, "-S") && --argc) {
+ strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-s") && --argc) {
+ strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-g") && --argc) {
+ strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+ user_type = atoi(*++argv);
+ num_joins_leaves = atoi(*++argv);
+
+ } else if (!strcmp(*argv, "-f")) {
+ should_sleep = 0;
+
+ } else if (!strcmp(*argv, "-r")) {
+ pretty_print = 0;
+
+ } else {
+ fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+ exit(printUsage(stderr));
+ }
+ }
+ num_members = atoi(user_name);
+}
+
+int main(int argc, char **argv) {
+ double t;
+
+ exe = *argv;
+ usage(argc, argv);
+
+ if ((err = FL_connect(daemon_name, user_name, 0, &mbox, priv_name)) != ACCEPT_SESSION) {
+ fprintf(stderr, "FL_connect failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ if (user_type == DELTA) {
+ double *fl_join_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *fl_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *fl_cmbo_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+ if (!fl_join_times || !fl_leave_times || !fl_cmbo_times) {
+ exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+ }
+
+ for (i = 0; i < num_joins_leaves; ++i) {
+ t = get_time_timeofday();
+
+ if ((err = FL_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_join failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_reg_memb_mess(service_type));
+
+ fl_join_times[i] = get_time_timeofday() - t;
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+ }
+
+ /* send a kill message if we are done */
+ if (i == num_joins_leaves - 1) {
+ if ((mess_len = FL_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+ fprintf(stderr, "FL_multicast failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ }
+
+ t = get_time_timeofday();
+
+ if ((err = FL_leave(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_leave failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = FL_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_self_leave(service_type));
+
+ fl_leave_times[i] = get_time_timeofday() - t;
+ fl_cmbo_times[i] = fl_join_times[i] + fl_leave_times[i];
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * fl_join_times[i] * 1000));
+ }
+ }
+ /* compute timing statistics */
+ comp_stats(&fl_join_stats, fl_join_times, num_joins_leaves);
+ comp_stats(&fl_leave_stats, fl_leave_times, num_joins_leaves);
+ comp_stats(&fl_cmbo_stats, fl_cmbo_times, num_joins_leaves);
+
+ /* output timing statistics */
+ if (pretty_print) {
+ printf("Flush Membership Timings (includes Spread): Group Size: %d, # Joins/Leaves: %d\r\n\r\n",
+ num_members, num_joins_leaves);
+
+ printf("\tFlush Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_join_stats.total90, sp_join_stats.total);
+
+ printf("\tFlush Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_leave_stats.total90, sp_leave_stats.total);
+
+ printf("\tFlush Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+ printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+ "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+ "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+ }
+ printf("\tFL_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_join_stats.quart1, fl_join_stats.median,
+ fl_join_stats.quart3, fl_join_stats.min5, fl_join_stats.mean90,
+ fl_join_stats.stddev90, fl_join_stats.max95, fl_join_stats.min,
+ fl_join_stats.mean, fl_join_stats.stddev, fl_join_stats.max);
+
+ printf("\tFL_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_leave_stats.quart1, fl_leave_stats.median,
+ fl_leave_stats.quart3, fl_leave_stats.min5, fl_leave_stats.mean90,
+ fl_leave_stats.stddev90, fl_leave_stats.max95, fl_leave_stats.min,
+ fl_leave_stats.mean, fl_leave_stats.stddev, fl_leave_stats.max);
+
+ printf("\tFL_Join + FL_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, fl_cmbo_stats.quart1, fl_cmbo_stats.median,
+ fl_cmbo_stats.quart3, fl_cmbo_stats.min5, fl_cmbo_stats.mean90,
+ fl_cmbo_stats.stddev90, fl_cmbo_stats.max95, fl_cmbo_stats.min,
+ fl_cmbo_stats.mean, fl_cmbo_stats.stddev, fl_cmbo_stats.max);
+
+ } else if (user_type == LOAD_MEMBER) {
+ if ((err = FL_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_join failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ while (1) {
+ if ((mess_len = FL_receive(mbox, &service_type, sender,
+ MY_MAX_NUM_GROUPS, &num_groups, groups,
+ &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess, &more_messes)) < 0) {
+ fprintf(stderr, "FL_receive failure: ");
+ FL_error(mess_len);
+ exit(1);
+ }
+ if (Is_flush_req_mess(service_type)) {
+ if ((err = FL_flush(mbox, group_name)) < 0) {
+ fprintf(stderr, "FL_flush failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ } else if (Is_reg_memb_mess(service_type)) {
+ if (Is_caused_leave_mess(service_type)) {
+ if (about_to_die) {
+ break;
+ }
+ } else if (!Is_caused_join_mess(service_type)) {
+ exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+ }
+ } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+ about_to_die = 1;
+ }
+ }
+ printf("Success!\r\n");
+
+ } else {
+ fprintf(stderr, "Unknown user type: %d\n", user_type);
+ exit(printUsage(stderr));
+ }
+
+ if ((err = FL_disconnect(mbox)) < 0) {
+ fprintf(stderr, "FL_disconnect failure: ");
+ FL_error(err);
+ exit(1);
+ }
+ return 0;
+}
Added: trunk/flush/scatp.c
===================================================================
--- trunk/flush/scatp.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,440 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <errno.h>
+#include <assert.h>
+#include <string.h>
+#include <scatp.h>
+
+long scat_capacity(const scatter *scat)
+{
+ const scat_element *curr = scat->elements, *end = scat->elements + scat->num_elements;
+ long ret = 0;
+
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ for (; curr != end; ++curr) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret += curr->len;
+ }
+ return ret;
+}
+
+int scatp_begin(scatp *pos, const scatter *scat)
+{
+ int i;
+
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ for (i = 0; i < scat->num_elements && scat->elements[i].len == 0; ++i);
+
+ if (i != scat->num_elements && scat->elements[i].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ pos->scat = (scatter*) scat;
+ pos->elem_ind = i;
+ pos->buff_ind = 0;
+ return 0;
+}
+
+int scatp_end(scatp *pos, const scatter *scat)
+{
+ if (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ pos->scat = (scatter*) scat;
+ pos->elem_ind = scat->num_elements;
+ pos->buff_ind = 0;
+ return 0;
+}
+
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence)
+{
+ int err;
+
+ if (whence == SEEK_CUR) {
+ if ((err = scatp_begin(pos, scat)))
+ return err;
+ } else
+ pos->scat = (scatter*) scat;
+
+ return scatp_seek(pos, offset, whence);
+}
+
+int scatp_is_legal(const scatp *pos)
+{
+ const scatter *scat = pos->scat;
+
+ return (scat->num_elements >= 0 && scat->num_elements <= MAX_CLIENT_SCATTER_ELEMENTS &&
+ (scatp_is_end(pos) ||
+ (pos->elem_ind >= 0 && pos->elem_ind < scat->num_elements &&
+ pos->buff_ind >= 0 && pos->buff_ind < scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_not_legal(const scatp *pos)
+{
+ const scatter *scat = pos->scat;
+
+ return (scat->num_elements < 0 || scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS ||
+ (!scatp_is_end(pos) &&
+ (pos->elem_ind < 0 || pos->elem_ind >= scat->num_elements ||
+ pos->buff_ind < 0 || pos->buff_ind >= scat->elements[pos->elem_ind].len)));
+}
+
+int scatp_is_end(const scatp *pos)
+{
+ if (pos->scat->num_elements < 0 || pos->scat->num_elements > MAX_CLIENT_SCATTER_ELEMENTS)
+ return ILLEGAL_MESSAGE;
+
+ return pos->elem_ind == pos->scat->num_elements && pos->buff_ind == 0;
+}
+
+int scatp_equals(const scatp *pos1, const scatp *pos2)
+{
+ if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+ return ILLEGAL_MESSAGE;
+
+ if (pos1->scat != pos2->scat)
+ return ILLEGAL_SERVICE;
+
+ return pos1->elem_ind == pos2->elem_ind && pos1->buff_ind == pos2->buff_ind;
+}
+
+long scatp_comp(const scatp *pos1, const scatp *pos2)
+{
+ const scat_element *curr, *end;
+ const scatter *scat = pos1->scat;
+ long ret;
+
+ if (scatp_is_not_legal(pos1) || scatp_is_not_legal(pos2))
+ return ILLEGAL_MESSAGE;
+
+ if (pos1->scat != pos2->scat)
+ return ILLEGAL_SERVICE;
+
+ if (pos1->elem_ind == pos2->elem_ind)
+ return pos1->buff_ind - pos2->buff_ind;
+
+ if (pos1->elem_ind < pos2->elem_ind) {
+ ret = pos1->buff_ind - scat->elements[pos1->elem_ind].len;
+ curr = scat->elements + pos1->elem_ind;
+ end = scat->elements + pos2->elem_ind;
+ while (++curr != end) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret -= curr->len;
+ }
+ } else {
+ ret = scat->elements[pos2->elem_ind].len - pos2->buff_ind;
+ curr = scat->elements + pos2->elem_ind;
+ end = scat->elements + pos1->elem_ind;
+ while (++curr != end) {
+ if (curr->len < 0)
+ return ILLEGAL_MESSAGE;
+ ret += curr->len;
+ }
+ }
+ return ret;
+}
+
+/* This fcn moves a scat_pos forward num_bytes bytes in a scatter. On
+ success, this fcn returns num_bytes and pos is modified
+ appropriately. Otherwise, if the jump was so big that it would have
+ jumped off the end of the scatter, the number of bytes that _could_
+ have been successfully jumped is returned, and pos is unchanged.
+*/
+
+long scatp_jforward(scatp *pos, long num_bytes)
+{
+ long elem_ind, skip_bytes, tmp;
+ const scatter *scat = pos->scat;
+
+ if (scatp_is_not_legal(pos))
+ return ILLEGAL_MESSAGE;
+
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ if (scatp_is_end(pos)) /* can't move forward from end */
+ return 0;
+
+ /* jump stays within current element buffer */
+ if ((tmp = scat->elements[pos->elem_ind].len - pos->buff_ind) > num_bytes) {
+ pos->buff_ind += num_bytes;
+ return num_bytes;
+ }
+ /* incorporate what was left in current element buffer into jump */
+ elem_ind = pos->elem_ind + 1;
+ skip_bytes = num_bytes - tmp; /* how many bytes left to skip */
+
+ for (; elem_ind < scat->num_elements; ++elem_ind) {
+ if (scat->elements[elem_ind].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ /* use < 0 because it jumps over any zero length buffers */
+ if ((skip_bytes -= scat->elements[elem_ind].len) < 0) {
+ skip_bytes += scat->elements[elem_ind].len; /* restore to positive */
+ break;
+ }
+ }
+ /* jump forward jumped past end of scatter */
+ if (elem_ind == scat->num_elements && skip_bytes != 0)
+ return num_bytes - skip_bytes;
+
+ pos->elem_ind = elem_ind;
+ pos->buff_ind = skip_bytes;
+ return num_bytes;
+}
+
+/* same as scat_jforward, except it moves the position backwards in the scatter */
+
+long scatp_jbackward(scatp *pos, long num_bytes)
+{
+ long elem_ind, e_ind, skip_bytes;
+ const scatter *scat = pos->scat;
+
+ if (scatp_is_not_legal(pos))
+ return ILLEGAL_MESSAGE;
+
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ /* jump stays within current element buffer */
+ if (pos->buff_ind >= num_bytes) {
+ pos->buff_ind -= num_bytes;
+ return num_bytes;
+ }
+ /* incorporate what was left in current element buffer into jump */
+ elem_ind = pos->elem_ind;
+ skip_bytes = num_bytes - pos->buff_ind;
+
+ for (e_ind = pos->elem_ind - 1; e_ind >= 0; --e_ind) {
+ if (scat->elements[e_ind].len < 0)
+ return ILLEGAL_MESSAGE;
+
+ /* again we want to ignore any zero length buffers */
+ if (scat->elements[e_ind].len > 0) {
+ elem_ind = e_ind; /* elem_ind must reference a non-empty element buffer */
+ if ((skip_bytes -= scat->elements[e_ind].len) <= 0)
+ break;
+ }
+ }
+ if (e_ind < 0)
+ return num_bytes - skip_bytes;
+
+ pos->elem_ind = elem_ind;
+ pos->buff_ind = -skip_bytes; /* restore to positive */
+ return num_bytes;
+}
+
+int scatp_seek(scatp *pos, long offset, int whence)
+{
+ scatp set;
+ long err;
+
+ switch (whence) {
+ case SEEK_CUR:
+ set = *pos;
+ break;
+ case SEEK_SET:
+ if ((err = scatp_begin(&set, pos->scat)))
+ return (int) err;
+ break;
+ case SEEK_END:
+ if ((err = scatp_end(&set, pos->scat)))
+ return (int) err;
+ break;
+ default:
+ return EINVAL;
+ }
+ if (offset >= 0) {
+ if ((err = scatp_jforward(&set, offset)) != offset)
+ return err < 0 ? (int) err : -1;
+ } else {
+ offset = -offset;
+ if ((err = scatp_jbackward(&set, offset)) != offset)
+ return err < 0 ? (int) err : -1;
+ }
+ *pos = set;
+ return 0;
+}
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes)
+{
+ return scatp_adv_cpy0((scatp*) dst, (scatp*) src, num_bytes, 0, 0);
+}
+
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes)
+{
+ scatter dscat;
+ scatp dscatp;
+ long err;
+
+ dscat.num_elements = 1;
+ dscat.elements[0].len = num_bytes;
+ dscat.elements[0].buf = dst;
+
+ err = scatp_begin(&dscatp, &dscat);
+ assert(err == 0);
+
+ return scatp_cpy0(&dscatp, src, num_bytes);
+}
+
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes)
+{
+ scatter sscat;
+ scatp sscatp;
+ long err;
+
+ sscat.num_elements = 1;
+ sscat.elements[0].len = num_bytes;
+ sscat.elements[0].buf = src;
+
+ err = scatp_begin(&sscatp, &sscat);
+ assert(err == 0);
+
+ return scatp_cpy0(dst, &sscatp, num_bytes);
+}
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter *dscat = dst->scat, *sscat = src->scat;
+ long dst_elem, src_elem, bytes_left, copy_size, dst_left, src_left;
+ char *dst_curr, *dst_end, *src_curr, *src_end;
+
+ if (scatp_is_not_legal(dst) || scatp_is_not_legal(src)) {
+ printf("illegal scatp! dst: %d src: %d\n", scatp_is_not_legal(dst), scatp_is_not_legal(src));
+ return ILLEGAL_MESSAGE;
+ }
+ if (num_bytes < 0)
+ return ILLEGAL_SERVICE;
+
+ if (scatp_is_end(dst) || scatp_is_end(src))
+ return 0;
+
+ dst_elem = dst->elem_ind;
+ dst_curr = dscat->elements[dst->elem_ind].buf + dst->buff_ind;
+ dst_end = dscat->elements[dst->elem_ind].buf + dscat->elements[dst->elem_ind].len;
+
+ src_elem = src->elem_ind;
+ src_curr = sscat->elements[src->elem_ind].buf + src->buff_ind;
+ src_end = sscat->elements[src->elem_ind].buf + sscat->elements[src->elem_ind].len;
+
+ bytes_left = num_bytes;
+
+ while (dst_elem < dscat->num_elements && src_elem < sscat->num_elements && bytes_left) {
+ dst_left = dst_end - dst_curr;
+ src_left = src_end - src_curr;
+ copy_size = (dst_left < src_left) ? dst_left : src_left;
+ copy_size = (copy_size < bytes_left) ? copy_size : bytes_left;
+
+ if (copy_size < 0) { /* ensure no empty or negative copies */
+ printf("scatp_cpy: buffer size negative!\n");
+ return ILLEGAL_MESSAGE;
+ }
+
+ memcpy(dst_curr, src_curr, copy_size);
+ bytes_left -= copy_size;
+
+ if (copy_size != dst_left)
+ dst_curr += copy_size;
+ else {
+ while (++dst_elem < dscat->num_elements && dscat->elements[dst_elem].len == 0);
+ if (dst_elem < dscat->num_elements) {
+ dst_curr = dscat->elements[dst_elem].buf;
+ dst_end = dscat->elements[dst_elem].buf + dscat->elements[dst_elem].len;
+ }
+ }
+ if (copy_size != src_left)
+ src_curr += copy_size;
+ else {
+ while (++src_elem < sscat->num_elements && sscat->elements[src_elem].len == 0);
+ if (src_elem < sscat->num_elements) {
+ src_curr = sscat->elements[src_elem].buf;
+ src_end = sscat->elements[src_elem].buf + sscat->elements[src_elem].len;
+ }
+ }
+ }
+ if (bytes_left != 0) /* couldn't do the entire copy */
+ return num_bytes - bytes_left;
+
+ /* success! now, update the scatp's if requested to */
+ if (adv_dst) {
+ dst->elem_ind = dst_elem;
+ if (dst_elem != dscat->num_elements)
+ dst->buff_ind = dst_curr - dscat->elements[dst_elem].buf;
+ else
+ dst->buff_ind = 0;
+ }
+ if (adv_src) {
+ src->elem_ind = src_elem;
+ if (src_elem != sscat->num_elements)
+ src->buff_ind = src_curr - sscat->elements[src_elem].buf;
+ else
+ src->buff_ind = 0;
+ }
+ return num_bytes;
+}
+
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter dscat;
+ scatp dscatp;
+ long ret;
+
+ dscat.num_elements = 1;
+ dscat.elements[0].len = num_bytes;
+ dscat.elements[0].buf = *dst;
+
+ ret = scatp_begin(&dscatp, &dscat);
+ assert(ret == 0);
+
+ if ((ret = scatp_adv_cpy0(&dscatp, src, num_bytes, 0, adv_src)) == num_bytes && adv_dst)
+ *dst += num_bytes;
+
+ return ret;
+}
+
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src)
+{
+ scatter sscat;
+ scatp sscatp;
+ long ret;
+
+ sscat.num_elements = 1;
+ sscat.elements[0].len = num_bytes;
+ sscat.elements[0].buf = *src;
+
+ ret = scatp_begin(&sscatp, &sscat);
+ assert(ret == 0);
+
+ if ((ret = scatp_adv_cpy0(dst, &sscatp, num_bytes, adv_dst, 0)) == num_bytes && adv_src)
+ *src += num_bytes;
+
+ return ret;
+}
+
Added: trunk/flush/scatp.h
===================================================================
--- trunk/flush/scatp.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/scatp.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,78 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+#define scatp_h_2000_05_21_19_14_47_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+#include <stdio.h>
+
+/* don't try to initialize this structure by yourself, instead call
+ scatp_begin, scatp_set, or scatp_end -- there are special cases
+ where the initial values are not obvious
+*/
+
+typedef struct scatp /* represents a logical position within a scatter structure */
+{
+ scatter *scat;
+ long elem_ind; /* use signed because scatter uses signed on some archs */
+ long buff_ind;
+
+} scatp;
+
+long scat_capacity(const scatter *scat);
+
+/* initializers: need to call one to init a scatp properly */
+
+int scatp_begin(scatp *pos, const scatter *scat);
+int scatp_end(scatp *pos, const scatter *scat);
+int scatp_set(scatp *pos, const scatter *scat, long offset, int whence);
+
+/* some information about a scatp */
+
+int scatp_is_legal(const scatp *pos);
+int scatp_is_not_legal(const scatp *pos);
+int scatp_is_end(const scatp *pos);
+int scatp_equals(const scatp *pos1, const scatp *pos2);
+long scatp_comp(const scatp *pos1, const scatp *pos2);
+
+/* move current position of a scatp */
+
+long scatp_jforward(scatp *pos, long num_bytes);
+long scatp_jbackward(scatp *pos, long num_bytes);
+int scatp_seek(scatp *pos, long offset, int whence);
+
+/* copy from a src to a dst like memcpy does */
+
+long scatp_cpy0(const scatp *dst, const scatp *src, long num_bytes);
+long scatp_cpy1(char *dst, const scatp *src, long num_bytes);
+long scatp_cpy2(const scatp *dst, char *src, long num_bytes);
+
+/* copy from a src to a dst like memcpy does, and advance the positions if so requested */
+
+long scatp_adv_cpy0(scatp *dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy1(char **dst, scatp *src, long num_bytes, int adv_dst, int adv_src);
+long scatp_adv_cpy2(scatp *dst, char **src, long num_bytes, int adv_dst, int adv_src);
+
+#endif
Added: trunk/flush/sp_time_memb.c
===================================================================
--- trunk/flush/sp_time_memb.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/sp_time_memb.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,253 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <float.h>
+
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include "fl.h"
+#include "stats.h"
+
+#define MY_MAX_NUM_GROUPS 1000
+#define MY_MAX_MESS_SIZE 102400
+
+enum { LOAD_MEMBER = 0, DELTA = 1, DIE_MESS = 2048 };
+
+mailbox mbox;
+service service_type;
+char sender[MAX_GROUP_NAME];
+int num_groups;
+char groups[MY_MAX_NUM_GROUPS][MAX_GROUP_NAME];
+int16 mess_type;
+int endian_mismatch;
+int mess_len;
+char mess[MY_MAX_MESS_SIZE];
+int more_messes;
+
+char daemon_name[MAX_GROUP_NAME] = "4803@localhost";
+char user_name[MAX_GROUP_NAME] = "1";
+char priv_name[MAX_GROUP_NAME];
+char group_name[MAX_GROUP_NAME] = "test";
+
+char *exe = 0; /* executable's name */
+int user_type = -1;
+int num_joins_leaves = 0; /* i: tmp var, num_joins_leaves: # of memberships to cause */
+int about_to_die = 0; /* boolean: am I dieing after next membership? */
+int num_members;
+
+int should_sleep = 1; /* should there be sleeps between memberships? ususally yes */
+int pretty_print = 1; /* should the output be verbose and human readable? */
+
+stats_results sp_join_stats, sp_leave_stats, sp_cmbo_stats;
+
+int i, err, num_joins;
+
+static int printUsage(FILE *outstream) {
+ return fprintf(outstream,
+ "Usage: %s\r\n"
+ "\t[-S <group size>] : group size (for stats; also login name, defaults to 1)\r\n"
+ "\t[-s <address>] : spread daemon name - either port or port@machine\r\n"
+ "\t[-g <group name>] : group name to join\r\n"
+ "\t[-t <user type> <# joins/leaves>] : type of user "
+ "(LOAD_MEMBER = %d, DELTA = %d) + # of join/leave events\r\n"
+ "\t[-f] : don't sleep between memberships (default is to sleep)\r\n"
+ "\t[-r] : print raw scores w/ no pretty headings\r\n",
+ exe, LOAD_MEMBER, DELTA);
+}
+
+static void usage(int argc, char **argv) {
+ for (++argv, --argc; argc > 0; ++argv, --argc) {
+ if (!strcmp(*argv, "-S") && --argc) {
+ strncpy(user_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-s") && --argc) {
+ strncpy(daemon_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-g") && --argc) {
+ strncpy(group_name, *++argv, MAX_GROUP_NAME);
+
+ } else if (!strcmp(*argv, "-t") && (argc -= 2) > 0) {
+ user_type = atoi(*++argv);
+ num_joins_leaves = atoi(*++argv);
+
+ } else if (!strcmp(*argv, "-f")) {
+ should_sleep = 0;
+
+ } else if (!strcmp(*argv, "-r")) {
+ pretty_print = 0;
+
+ } else {
+ fprintf(stderr, "Unknown cmd line param: %s\r\n", *argv);
+ exit(printUsage(stderr));
+ }
+ }
+ num_members = atoi(user_name);
+}
+
+int main(int argc, char **argv) {
+ exe = *argv;
+
+ usage(argc, argv);
+
+ if ((err = SP_connect(daemon_name, user_name, 0, 1, &mbox, priv_name)) != ACCEPT_SESSION) {
+ fprintf(stderr, "SP_connect failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ if (user_type == DELTA) {
+ double *sp_join_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *sp_leave_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+ double *sp_cmbo_times = (double*) malloc(sizeof(double) * num_joins_leaves);
+
+ if (!sp_join_times || !sp_leave_times || !sp_cmbo_times) {
+ exit(fprintf(stderr, "Couldn't mallocate tracking arrays!\r\n"));
+ }
+
+ for (i = 0; i < num_joins_leaves; ++i) {
+ double t = get_time_timeofday();
+
+ if ((err = SP_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_join failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_reg_memb_mess(service_type));
+
+ sp_join_times[i] = get_time_timeofday() - t;
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+ }
+
+ /* send a kill message if we are done */
+ if (i == num_joins_leaves - 1) {
+ if ((mess_len = SP_multicast(mbox, SAFE_MESS, group_name, DIE_MESS, 0, 0)) < 0) {
+ fprintf(stderr, "SP_multicast failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ }
+ t = get_time_timeofday();
+
+ if ((err = SP_leave(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_leave failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ do {
+ if ((mess_len = SP_receive(mbox, &service_type, sender, MY_MAX_NUM_GROUPS,
+ &num_groups, groups, &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ } while (!Is_self_leave(service_type));
+
+ sp_leave_times[i] = get_time_timeofday() - t;
+ sp_cmbo_times[i] = sp_join_times[i] + sp_leave_times[i];
+
+ if (should_sleep) {
+ /* sleep for 4 times how long the join membership took */
+ /* allows membership to stabilize */
+ usleep((unsigned long) (4 * sp_join_times[i] * 1000));
+ }
+ }
+ /* compute statistics */
+ comp_stats(&sp_join_stats, sp_join_times, num_joins_leaves);
+ comp_stats(&sp_leave_stats, sp_leave_times, num_joins_leaves);
+ comp_stats(&sp_cmbo_stats, sp_cmbo_times, num_joins_leaves);
+
+ /* output statistics */
+ if (pretty_print) {
+ printf("Spread Membership Timings: Group Size: %d, # Joins/Leaves: %d\r\n\r\n",
+ num_members, num_joins_leaves);
+
+ printf("\tSpread Join: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_join_stats.total90, sp_join_stats.total);
+
+ printf("\tSpread Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_leave_stats.total90, sp_leave_stats.total);
+
+ printf("\tSpread Join/Leave: ind. total (90%%): %.6fms, ind. total: %.6fms\r\n",
+ sp_cmbo_stats.total90, sp_cmbo_stats.total);
+
+ printf("\r\n\t\tGroup Size\t# Trials\tQ1\tMEDIAN\tQ3"
+ "\tMIN (5%%)\tMEAN90\tSTDDEV90\tMAX (95%%)"
+ "\tMIN\tMEAN\tSTDDEV\tMAX\r\n");
+ }
+ printf("\tSP_Join:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_join_stats.quart1, sp_join_stats.median,
+ sp_join_stats.quart3, sp_join_stats.min5, sp_join_stats.mean90,
+ sp_join_stats.stddev90, sp_join_stats.max95, sp_join_stats.min,
+ sp_join_stats.mean, sp_join_stats.stddev, sp_join_stats.max);
+
+ printf("\tSP_Leave:\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_leave_stats.quart1, sp_leave_stats.median,
+ sp_leave_stats.quart3, sp_leave_stats.min5, sp_leave_stats.mean90,
+ sp_leave_stats.stddev90, sp_leave_stats.max95, sp_leave_stats.min,
+ sp_leave_stats.mean, sp_leave_stats.stddev, sp_leave_stats.max);
+
+ printf("\tSP_Join + SP_Leave\t%d\t%d\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\t%.6f\r\n",
+ num_members, num_joins_leaves, sp_cmbo_stats.quart1, sp_cmbo_stats.median,
+ sp_cmbo_stats.quart3, sp_cmbo_stats.min5, sp_cmbo_stats.mean90,
+ sp_cmbo_stats.stddev90, sp_cmbo_stats.max95, sp_cmbo_stats.min,
+ sp_cmbo_stats.mean, sp_cmbo_stats.stddev, sp_cmbo_stats.max);
+
+ } else if (user_type == LOAD_MEMBER) {
+ if ((err = SP_join(mbox, group_name)) < 0) {
+ fprintf(stderr, "SP_join failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ while (1) {
+ if ((mess_len = SP_receive(mbox, &service_type, sender,
+ MY_MAX_NUM_GROUPS, &num_groups, groups,
+ &mess_type, &endian_mismatch,
+ MY_MAX_MESS_SIZE, mess)) < 0) {
+ fprintf(stderr, "SP_receive failure: ");
+ SP_error(mess_len);
+ exit(1);
+ }
+ if (Is_reg_memb_mess(service_type)) {
+ if (Is_caused_leave_mess(service_type)) {
+ if (about_to_die) {
+ break;
+ }
+ } else if (!Is_caused_join_mess(service_type)) {
+ exit(fprintf(stderr, "Unexpected membership type: %d\n", service_type));
+ }
+ } else if (Is_safe_mess(service_type) && mess_type == DIE_MESS) {
+ about_to_die = 1;
+ }
+ }
+ printf("Success!\r\n");
+
+ } else {
+ fprintf(stderr, "Unknown user type: %d\n", user_type);
+ exit(printUsage(stderr));
+ }
+
+ if ((err = SP_disconnect(mbox)) < 0) {
+ fprintf(stderr, "SP_disconnect failure: ");
+ SP_error(err);
+ exit(1);
+ }
+ return 0;
+}
+
+
Added: trunk/flush/stats.c
===================================================================
--- trunk/flush/stats.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,81 @@
+#include <stdlib.h>
+#include <math.h>
+#include <sys/time.h>
+
+#include "stats.h"
+
+static int dbl_cmp(const void *arg1, const void *arg2)
+{
+ double a1 = *(double*) arg1, a2 = *(double*) arg2;
+
+ if (a1 < a2) {
+ return -1;
+
+ } else if (a1 > a2) {
+ return 1;
+ }
+ return 0;
+}
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples)
+{
+ size_t ind5 = num_samples / 20, ind95 = 19 * num_samples / 20, num_samples90 = ind95 - ind5 + 1;
+ size_t i;
+
+ /* total90, mean90, stddev90, min5, max95 are all computed from samples [5%, 95%] range */
+
+ if (num_samples == 0) {
+ return -1;
+ }
+ qsort(samples, num_samples, sizeof(double), dbl_cmp);
+
+ ans->min = samples[0];
+ ans->min5 = samples[ind5];
+ ans->max95 = samples[ind95];
+ ans->max = samples[num_samples - 1];
+
+ ans->quart1 = samples[num_samples / 4];
+ ans->median = samples[num_samples / 2];
+ ans->quart3 = samples[3 * num_samples / 4];
+
+ for (i = 0, ans->total = 0, ans->total90 = 0; i < ind5; ++i) {
+ ans->total += samples[i];
+ }
+ for (i = ind5; i <= ind95; ++i) {
+ ans->total90 += samples[i];
+ }
+ for (i = ind95 + 1; i < num_samples; ++i) {
+ ans->total += samples[i];
+ }
+ ans->mean90 = ans->total90 / num_samples90;
+ ans->mean = (ans->total + ans->total90) / num_samples;
+
+ if (num_samples != 1) {
+ for (i = 0, ans->stddev = 0, ans->stddev90 = 0; i < ind5; ++i) {
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ for (i = ind5; i <= ind95; ++i) {
+ ans->stddev90 += (ans->mean90 - samples[i]) * (ans->mean90 - samples[i]);
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ for (i = ind95 + 1; i < num_samples; ++i) {
+ ans->stddev += (ans->mean - samples[i]) * (ans->mean - samples[i]);
+ }
+ ans->stddev90 = sqrt(ans->stddev90 / (num_samples90 - 1));
+ ans->stddev = sqrt(ans->stddev / (num_samples - 1));
+
+ } else {
+ ans->stddev90 = 0;
+ ans->stddev = 0;
+ }
+ return 0;
+}
+
+double get_time_timeofday(void)
+{
+ struct timeval used;
+
+ gettimeofday(&used, 0);
+
+ return used.tv_sec * 1000.0 + used.tv_usec / 1000.0;
+}
Added: trunk/flush/stats.h
===================================================================
--- trunk/flush/stats.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/stats.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,19 @@
+#ifndef stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+#define stats_h_2001_07_06_14_19_22_jschultz_at_dfusion_net
+
+#include <stddef.h>
+
+typedef struct {
+ double total, total90;
+ double mean, mean90;
+ double stddev, stddev90;
+ double min, min5, max, max95;
+ double quart1, median, quart3;
+
+} stats_results;
+
+int comp_stats(stats_results *ans, double *samples, size_t num_samples);
+
+double get_time_timeofday(void);
+
+#endif
Added: trunk/flush/user.c
===================================================================
--- trunk/flush/user.c 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/flush/user.c 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,423 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fl.h>
+
+#ifdef USE_DMALLOC
+#include <dmalloc.h>
+#endif
+
+static char User[80];
+static char Spread_name[80];
+static char Private_group[MAX_GROUP_NAME];
+static mailbox Mbox;
+static int Num_sent;
+static int Previous_len;
+static int To_exit = 0;
+
+static void Print_menu();
+static void User_command();
+static void Read_message();
+static void Usage( int argc, char *argv[] );
+static void Bye();
+
+int main(int argc, char *argv[]) {
+ int ret, major, minor, patch;
+
+ Usage(argc, argv);
+
+ FL_version(&major, &minor, &patch);
+ printf("Flush library version is %d.%d.%d\n", major, minor, patch);
+
+ ret = FL_connect( Spread_name, User, LOW_PRIORITY, &Mbox, Private_group );
+ if( ret < 0 ) {
+ FL_error( ret );
+ Bye();
+ }
+ printf("User: connected to %s with private group %s\n", Spread_name, Private_group );
+
+ Print_menu();
+
+ printf("\nUser> ");
+ fflush(stdout);
+
+ Num_sent = 0;
+
+ for(;;)
+ User_command();
+
+ return 0;
+}
+
+static void User_command()
+{
+ char command[130];
+ char mess[102400];
+ char group[80];
+ char groups[10][MAX_GROUP_NAME];
+ int num_groups;
+ int mess_len;
+ int ret;
+ int i;
+
+ for (i = 0; i < sizeof(command); i++)
+ command[i] = 0;
+
+ if (fgets(command, 130, stdin) == 0)
+ Bye();
+
+ switch(command[0]) {
+ case 'j':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret < 1 ) {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_join( Mbox, group );
+ if( ret < 0 ) FL_error( ret );
+ break;
+
+ case 'l':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_leave( Mbox, group );
+ if( ret < 0 ) FL_error( ret );
+
+ break;
+
+ case 'g': {
+ int i;
+
+ if ((num_groups = sscanf(&command[2], "%s", groups[0])) < 1) {
+ printf("invalid group\n");
+ break;
+ }
+ printf("enter number of receivers: ");
+ fflush(stdout);
+ if (fgets(mess, 200, stdin) == 0 ||
+ sscanf(mess, "%d", &num_groups) != 1 ||
+ num_groups < 1 || num_groups > 9) {
+ printf("invalid number of receivers, must be an integer in the range [1, 9]\n");
+ break;
+ }
+ for (i = 1; i <= num_groups; ++i) {
+ printf("enter recvr %d: ", i);
+ fflush(stdout);
+ if (fgets(mess, MAX_GROUP_NAME, stdin) == 0 || sscanf(mess, "%s", groups[i]) != 1)
+ Bye();
+ }
+ printf("enter message: ");
+ fflush(stdout);
+ if (fgets(mess, 200, stdin) == 0) Bye();
+ mess_len = strlen(mess);
+
+ printf("user.c: subgroupcasting message of size %d to group %s with receivers:\n", mess_len, groups[0]);
+ for (i = 1; i <= num_groups; ++i)
+ printf("%s\n", groups[i]);
+
+ if ((ret = FL_subgroupcast(Mbox, SAFE_MESS, groups[0], num_groups, groups+1, 0, mess_len, mess)) != mess_len) {
+ if (ret < 0) {
+ FL_error(ret);
+ Bye();
+ } else
+ printf("Wierd return from FL_subgroupcast %d, should be %d\n", ret, mess_len);
+ }
+ Num_sent++;
+ break;
+ }
+
+ case 's':
+ num_groups = sscanf(&command[2], "%s", groups[0]);
+ if( num_groups < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ printf("enter message: ");
+ ret = (int) fgets( mess, 200, stdin );
+ if( ret==0 ) Bye();
+ mess_len = strlen( mess );
+
+ printf("user.c : multicasting message of size %d:\n'%s'\n", mess_len, mess);
+
+ ret = FL_multicast( Mbox, SAFE_MESS, groups[0], 1, mess_len, mess );
+ if( ret < 0 )
+ {
+ FL_error( ret );
+ Bye();
+ }
+ Num_sent++;
+ break;
+
+ case 'b':
+ ret = sscanf( &command[2], "%s", group );
+ if( ret != 1 ) strcpy( group, "dummy_group_name" );
+ printf("enter size of each message: ");
+ ret = (int) fgets( mess, 200, stdin );
+ if( ret==0 ) Bye();
+ ret = sscanf(mess, "%d", &mess_len );
+ if( ret !=1 ) mess_len = Previous_len;
+ if( mess_len < 0 ) mess_len = 0;
+ Previous_len = mess_len;
+ printf("sending 10 messages of %d bytes\n", mess_len );
+ for( i=0; i<10; i++ )
+ {
+ Num_sent++;
+ sprintf( mess, "mess num %d", Num_sent );
+ ret= FL_multicast( Mbox, FIFO_MESS, group, 2, mess_len, mess );
+
+ if( ret < 0 )
+ {
+ FL_error( ret );
+ Bye();
+ }
+ printf("sent message %d (total %d)\n", i+1, Num_sent );
+ }
+ break;
+ case 'r':
+
+ Read_message();
+ break;
+
+ case 'p':
+
+ ret = FL_poll( Mbox );
+ printf("Polling says: %d\n", ret );
+ break;
+
+ /*
+ case 'e':
+
+ E_attach_fd( Mbox, Read_message, 0, HIGH_PRIORITY );
+ break;
+
+ case 'd':
+
+ E_detach_fd( Mbox );
+ break;
+
+ */
+ case 'f':
+ ret = sscanf( &command[2], "%s", group);
+ if( ret < 1 )
+ {
+ printf(" invalid group \n");
+ break;
+ }
+ ret = FL_flush(Mbox, group);
+
+ printf("Sent a FLUSH_OK message to group '%s'\n", group);
+
+ if (ret < 0) {
+ FL_error(ret);
+ Bye();
+ }
+ break;
+
+ case 'q':
+ Bye();
+ break;
+
+ default:
+ printf("\nUnknown commnad\n");
+ Print_menu();
+
+ break;
+ }
+ printf("\nUser> ");
+ fflush(stdout);
+}
+
+static void Print_menu()
+{
+ printf("\n");
+ printf("==========\n");
+ printf("User Menu:\n");
+ printf("----------\n");
+ printf("\n");
+ printf("\tj <group> -- join a group\n");
+ printf("\tl <group> -- leave a group\n");
+ printf("\n");
+ printf("\ts <group> -- send a message\n");
+ printf("\tg <group> -- send a subgroup message\n");
+ printf("\tb <group> -- send a burst of messages\n");
+ printf("\n");
+ printf("\tr -- receive a message (stuck) \n");
+ printf("\tp -- poll for a message \n");
+ /*
+ printf("\te -- enable asynchonous read (default)\n");
+ printf("\td -- disable asynchronous read \n");
+ */
+ printf("\tf <group> -- send a FLUSH_OK message\n");
+ printf("\n");
+ printf("\tq -- quit\n");
+ fflush(stdout);
+}
+
+static void Read_message()
+{
+
+ static char mess[102400];
+ char sender[MAX_GROUP_NAME];
+ char target_groups[100][MAX_GROUP_NAME];
+ group_id *grp_id;
+ int32 *num_vs;
+ char *vs_members;
+ int num_groups;
+ int num_bytes;
+ int service_type;
+ int16 mess_type;
+ int endian_mismatch;
+ int i;
+ int ret;
+ int more_messes;
+
+ ret = FL_receive( Mbox, &service_type, sender, 100, &num_groups, target_groups,
+ &mess_type, &endian_mismatch, sizeof(mess), mess, &more_messes);
+ printf("\n============================\n");
+ if( ret < 0 )
+ {
+ if( ! To_exit )
+ {
+ FL_error( ret );
+ printf("\n============================\n");
+ printf("\nBye.\n");
+ }
+ exit( 0 );
+ }
+ if( Is_regular_mess( service_type ) )
+ {
+ mess[ret] = 0;
+ if ( Is_unreliable_mess( service_type ) ) printf("received UNRELIABLE ");
+ else if( Is_reliable_mess( service_type ) ) printf("received RELIABLE ");
+ else if( Is_fifo_mess( service_type ) ) printf("received FIFO ");
+ else if( Is_causal_mess( service_type ) ) printf("received CAUSAL ");
+ else if( Is_agreed_mess( service_type ) ) printf("received AGREED ");
+ else if( Is_safe_mess( service_type ) ) printf("received SAFE ");
+
+ printf("message from %s, of type %d, (endian %d) to %d groups \n(%d bytes): %s\n",
+ sender, mess_type, endian_mismatch, num_groups, ret, mess );
+ printf("Message is:\n'%s'\n", mess);
+ if (Is_subgroup_mess(service_type)) {
+ int i;
+
+ printf("Received a subgroupcast message to group %s: receivers:\n", target_groups[num_groups - 1]);
+ for (i = 0; i < num_groups - 1; ++i)
+ printf("\t%s\n", target_groups[i]);
+ }
+ } else if( Is_membership_mess( service_type ) ){
+ if ( Is_reg_memb_mess( service_type ) )
+ {
+ num_bytes = 0;
+ grp_id = (group_id *)&mess[num_bytes];
+ num_bytes += sizeof( group_id );
+ num_vs = (int32 *)&mess[num_bytes];
+ num_bytes += sizeof( int32 );
+ vs_members = &mess[num_bytes];
+
+ printf("Received REGULAR membership for group %s with %d members, where I am member %d:\n",
+ sender, num_groups, mess_type );
+
+ for( i=0; i < num_groups; i++ )
+ printf("\t%s\n", &target_groups[i][0]);
+
+ printf("grp id is %d %d %d\n", grp_id->id[0], grp_id->id[1], grp_id->id[2]);
+
+ if( Is_caused_join_mess( service_type ) )
+ {
+ printf("Due to the JOIN of %s\n", vs_members );
+ }else if( Is_caused_leave_mess( service_type ) ){
+ printf("Due to the LEAVE of %s\n", vs_members);
+ }else if( Is_caused_disconnect_mess( service_type ) ){
+ printf("Due to the DISCONNECT of %s\n", vs_members );
+ }else if( Is_caused_network_mess( service_type ) ){
+ printf("Due to NETWORK change. ");
+ printf("VS set has %d members:\n", *num_vs );
+ for( i=0; i < *num_vs; i++, vs_members+= MAX_GROUP_NAME )
+ printf("\t%s\n", vs_members );
+ }
+ }else if( Is_transition_mess( service_type ) ) {
+ printf("received TRANSITIONAL membership for group %s\n", sender );
+ }else if( Is_caused_leave_mess( service_type ) ){
+ printf("received membership message that left group %s\n", sender );
+ }else printf("received incorrecty membership message of type %d\n", service_type );
+ }else if (Is_flush_req_mess(service_type)) {
+ printf("received a FLUSH_REQ message for group %s\n", sender);
+ }else printf("received message of unknown message type %d with ret %d\n", service_type, ret);
+
+ printf("There are %d buffered messages waiting to be delivered\n", more_messes);
+
+ printf("\n");
+ printf("User> ");
+ fflush(stdout);
+}
+
+static void Usage(int argc, char *argv[])
+{
+
+ sprintf( User, "user" );
+ sprintf( Spread_name, "3333@localhost");
+ while( --argc > 0 )
+ {
+ argv++;
+
+ if( !strncmp( *argv, "-u", 2 ) )
+ {
+ strcpy( User, argv[1] );
+ argc--; argv++;
+ }else if( !strncmp( *argv, "-s", 2 ) ){
+ strcpy( Spread_name, argv[1] );
+ argc--; argv++;
+ }else{
+ printf( "Usage: user\n%s\n%s\n",
+ "\t[-u <user name>] : unique (in this machine) user name",
+ "\t[-s <address>] : either port or port@machine");
+ exit( 0 );
+ }
+ }
+}
+
+static void Bye()
+{
+ To_exit = 1;
+
+ printf("\nBye.\n");
+
+ printf("Calling FL_disconnect(mbox = %d)\n", Mbox);
+
+ FL_disconnect( Mbox );
+ exit( 0 );
+}
+
+
+
+
+
Added: trunk/include/fl.h
===================================================================
--- trunk/include/fl.h 2005-07-28 14:50:56 UTC (rev 252)
+++ trunk/include/fl.h 2005-07-31 10:51:40 UTC (rev 253)
@@ -0,0 +1,114 @@
+/*
+ * The contents of this file are subject to the FLUSH SPREAD Non-Commercial
+ * License, Version 1.0 (the ``License''); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at:
+ *
+ * http://www.cnds.jhu.edu/research/group/flush_spread/FLUSH_LICENSE
+ *
+ * or in the file ``FLUSH_LICENSE'' found in the root of this distribution.
+ *
+ * Software distributed under the License is distributed on an AS IS basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Software is:
+ * The Flush Spread Library
+ *
+ * The Initial Developers of the Original Software are:
+ * Yair Amir, John Schultz and Jonathan Stanton
+ *
+ * All Rights Reserved.
+ *
+ */
+
+#ifndef fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+#define fl_h_2000_03_20_14_36_26_jschultz_at_cnds_jhu_edu
+
+#include <sp.h>
+
+/* FL service types */
+#define DONT_BLOCK 0x10000000
+#define FLUSH_REQ_MESS 0x20000000
+#define SUBGROUP_CAST 0x40000000
+
+/* FL service query macros */
+#define Is_flush_req_mess(serv) (((serv) & FLUSH_REQ_MESS) != 0)
+#define Is_subgroup_mess(serv) (((serv) & SUBGROUP_CAST) != 0)
+
+/* FL error codes */
+#define ILLEGAL_PARAM -24
+#define WOULD_BLOCK -25
+#define ILLEGAL_MESSAGE_TYPE -26
+#define ILLEGAL_STATE -27
+#define ILLEGAL_RECEIVERS -28
+
+/* maximum # of usable scatter elements for FL msgs */
+#define FL_MAX_SCATTER_ELEMENTS (MAX_CLIENT_SCATTER_ELEMENTS - 1)
+
+/* minimum message type a user is allowed to use - less than this is illegal */
+#define FL_MIN_LEGAL_MESS_TYPE ((int16) -32765)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int FL_lib_init(void);
+
+void FL_version(int *major_ver, int *minor_ver, int *patch_ver);
+
+int FL_connect(const char *daemon_name, const char *user_name, int priority,
+ mailbox *mbox, char *private_name);
+
+int FL_disconnect(mailbox mbox);
+
+int FL_join(mailbox mbox, const char *group_name);
+
+int FL_leave(mailbox mbox, const char *group_name);
+
+int FL_flush(mailbox mbox, const char *group_name);
+
+int FL_unicast(mailbox mbox, service serv_type, const char *group_name,
+ const char *recvr_name, int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_unicast(mailbox mbox, service serv_type, const char *group_name,
+ const char *recvr_name, int16 mess_type, const scatter *scat);
+
+int FL_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+ int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+ int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_subgroupcast(mailbox mbox, service serv_type, const char *group_name,
+ int num_recvrs, char recvr_names[][MAX_GROUP_NAME],
+ int16 mess_type, const scatter *scat);
+
+int FL_multicast(mailbox mbox, service serv_type, const char *group_name,
+ int16 mess_type, int mess_len, const char *mess);
+
+int FL_scat_multicast(mailbox mbox, service serv_type, const char *group_name,
+ int16 mess_type, const scatter *scat_mess);
+
+int FL_receive(mailbox mbox, service *serv_type, char *sender_name,
+ int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+ int16 *mess_type, int *endian_mismatch, int max_mess_len,
+ char *mess, int *more_msgs);
+
+int FL_scat_receive(mailbox mbox, service *serv_type, char *sender_name,
+ int max_groups, int *num_groups, char group_names[][MAX_GROUP_NAME],
+ int16 *mess_type, int *endian_mismatch, scatter *scat_mess,
+ int *more_msgs);
+
+int FL_more_msgs(mailbox mbox);
+
+int FL_poll(mailbox mbox);
+
+void FL_error(int error_code);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
+
+
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.