[Patch] Resend due to build error

Index: linux/hpc/kcomd.c
===================================================================
--- linux.orig/hpc/kcomd.c	2006-09-18 11:11:58.000000000 +0200
+++ linux/hpc/kcomd.c	2006-09-18 17:59:51.000000000 +0200
@@ -34,27 +34,30 @@
 
 	ret = sock_create(saddr->sa_family, SOCK_STREAM, IPPROTO_TCP, &sock);
 	if (ret < 0)
-		return -1;
+		goto err_fd;
 
 	fd = sock_map_fd(sock);
 	if (fd < 0)
-		goto err;
+		goto err_fd;
 
 	ret = sock->ops->bind(sock, saddr, sizeof(*saddr));
 	if (ret < 0)
-		goto err_fd;
+		goto err_bind;
 
 	ret = sock->ops->listen(sock, SOMAXCONN);
 	if (ret < 0)
-		goto err_fd;
+		goto err_listen;
 	*res = sock;
 	return fd;
-err_fd:
-	sys_close(fd);
-err:
+
+err_listen:
 	sock_release(sock);
+err_bind:
+	sys_close(fd);
+err_fd:
 	*res = NULL;
 	return -1;
+
 }
 
 /**
@@ -260,33 +263,45 @@
 {
 	struct socket *sock;
 	int ret, fd;
+	int len;
+	struct sockaddr_in address;
 
 	sock = sock_alloc();
-	if (!sock)
+	if (!sock) {
+		printk(KERN_ERR "openMosix: Unable to allocate socket.\n");
 		return -1;
+	}
+
+	sock->type = lsock->type;
+	sock->ops = lsock->ops;
 
 	ret = lsock->ops->accept(lsock, sock, 0);
-	if (ret)
-		goto err;
-	/*
-	if (!sock->ops || !sock->ops->getname)
-		goto err;
+	if (ret) {
+		printk(KERN_ERR "openMosix: Error accepting connection\n");
+		goto err_accept;
+	}
 
-	ret = sock->ops->getname
-	check if it's already in node list.
-	*/
+	ret = sock->ops->getname(sock, (struct sockaddr *)&address, &len, 1);
+	if (ret)
+		goto err_accept;
 
 	fd = sock_map_fd(sock);
-	if (fd < 0)
-		goto err;
+	if (fd < 0) {
+		printk(KERN_ERR "openMosix: Error mapping socket to file descriptor\n");
+		goto err_accept;
+	}
 
 	ret = kcom_node_add(fd, sock);
-	if (ret < 0)
+	if ( ret < 0 ) {
+		printk(KERN_ERR "openMosix: Error adding new node\n");
 		goto errfd;
+	}
+
 	return fd;
+
 errfd:
 	sys_close(fd);
-err:
+err_accept:
 	sock_release(sock);
 	return -1;
 }
Index: linux/include/hpc/hpc.h
===================================================================
--- linux.orig/include/hpc/hpc.h	2006-09-18 12:00:40.000000000 +0200
+++ linux/include/hpc/hpc.h	2006-09-18 12:08:04.000000000 +0200
@@ -71,4 +71,5 @@
 				char __user *__user *envp,
 				struct pt_regs * regs);
 
+int remote_handle_user(task_t *p, int endtype);
 #endif /* _HPC_HPC_H */
Index: linux/include/hpc/mig.h
===================================================================
--- linux.orig/include/hpc/mig.h	2006-09-18 14:33:12.000000000 +0200
+++ linux/include/hpc/mig.h	2006-09-18 14:34:27.000000000 +0200
@@ -25,6 +25,10 @@
 #include <hpc/comm.h>
 
 #define REMOTE_DAEMON_PORT  0x3412
+/* Define some linux functions (remove warnings) */
+
+
+void reparent_to_init(void);
 
 /* PROTOTYPES */
 int openmosix_mig_daemon(void *);
Index: linux/hpc/migrecv.c
===================================================================
--- linux.orig/hpc/migrecv.c	2006-09-18 16:20:13.000000000 +0200
+++ linux/hpc/migrecv.c	2006-09-18 16:44:12.000000000 +0200
@@ -408,4 +408,7 @@
 		if (error < 0)
 			comm_close(mlink);
 	}
+
+	/* Not reached, just to prevent warning on recent gcc: */
+	return 0;
 }
Index: linux/include/net/sock.h
===================================================================
--- linux.orig/include/net/sock.h	2006-09-18 16:23:07.000000000 +0200
+++ linux/include/net/sock.h	2006-09-18 16:23:51.000000000 +0200
@@ -774,6 +774,7 @@
 						     unsigned long size,
 						     int noblock,
 						     int *errcode);
+extern struct socket 		*sock_alloc(void);
 extern void *sock_kmalloc(struct sock *sk, int size,
 			  gfp_t priority);
 extern void sock_kfree_s(struct sock *sk, void *mem, int size);
Index: linux/include/linux/compiler.h
===================================================================
--- linux.orig/include/linux/compiler.h	2006-09-18 17:54:29.000000000 +0200
+++ linux/include/linux/compiler.h	2006-09-18 17:59:51.000000000 +0200
@@ -1,6 +1,8 @@
 #ifndef __LINUX_COMPILER_H
 #define __LINUX_COMPILER_H
 
+#include <linux/config.h>
+
 #ifndef __ASSEMBLY__
 
 #ifdef __CHECKER__
Index: linux/net/socket.c
===================================================================
--- linux.orig/net/socket.c	2006-09-18 18:13:09.000000000 +0200
+++ linux/net/socket.c	2006-09-18 18:13:13.000000000 +0200
@@ -511,7 +511,7 @@
  *	NULL is returned.
  */
 
-KCOMD_NSTATIC struct socket *sock_alloc(void)
+struct socket *sock_alloc(void)
 {
 	struct inode * inode;
 	struct socket * sock;

Index: linux/hpc/copyuser.c
===================================================================
--- linux.orig/hpc/copyuser.c	2006-09-18 21:24:38.000000000 +0200
+++ linux/hpc/copyuser.c	2006-09-18 23:29:14.000000000 +0200
@@ -21,7 +21,13 @@
 /********** DEPUTY PART **********/
 
 /**
- * deputy_copy_from_user - Copy from remote when running on deputy
+ * deputy_copy_from_user -
+ * @to:     kernelspace address to copy to
+ * @from:   userspace address to copy from
+ * @n:      size of data to copy
+ *
+ * Description:
+ *    Copy from remote when running on deputy
  **/
 unsigned long deputy_copy_from_user(void *to, const void __user *from, unsigned long n)
 {
@@ -52,7 +58,13 @@
 EXPORT_SYMBOL(deputy_copy_from_user);
 
 /**
- * deputy_strncpy_from_user - strncpy on remote when running on deputy
+ * deputy_strncpy_from_user
+ * @dst:     kernelspace address to copy to
+ * @src:   userspace address to copy from
+ * @count:      size of data to copy
+ *
+ * Description:
+ *    strncpy on remote when running on deputy
  **/
 unsigned long deputy_strncpy_from_user(char *dst, const char __user *src,
 							long count)
@@ -79,7 +91,13 @@
 }
 
 /**
- * deputy_copy_to_user - copy to remote when running on deputy
+ * deputy_copy_to_user -
+ * @to:     userspace address to copy to
+ * @from:   kernelspace address to copy from
+ * @count:      size of data to copy
+ *
+ * Description:
+ *    copy to remote when running on deputy
  **/
 unsigned long deputy_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
@@ -110,7 +128,12 @@
 EXPORT_SYMBOL(deputy_copy_to_user);
 
 /**
- * deputy_strnlen_user - strnlen on remote when running on deputy
+ * deputy_strnlen_user -
+ * @s:     string address
+ * @n:     size
+ *
+ * Description:
+ *    strnlen on remote when running on deputy
  **/
 unsigned long deputy_strnlen_user(const char *s, long n)
 {
@@ -138,7 +161,13 @@
 EXPORT_SYMBOL(deputy_strnlen_user);
 
 /**
- * deputy_put_userX - put a value of 64 bit or less to remote
+ * deputy_put_userX
+ * @value:
+ * @addr:
+ * @size:
+ *
+ * Description:
+ *    put a value of 64 bit or less to remote
  **/
 static inline long deputy_put_userX(s64 value, const void *addr, size_t size)
 {
@@ -161,7 +190,13 @@
 }
 
 /**
- * deputy_put_user - put a long value to remote
+ * deputy_put_user
+ * @value:
+ * @addr:
+ * @size:
+ *
+ * Description:
+ *    put a long value to remote
  **/
 long deputy_put_user(long value, const void *addr, size_t size)
 {
@@ -172,7 +207,13 @@
 
 #if BITS_PER_LONG < 64
 /**
- * deputy_put_user - put a 64 bit value to remote
+ * deputy_put_user
+ * @value:
+ * @addr:
+ * @size:
+ *
+ * Description:
+ *    put a long value to remote
  **/
 long deputy_put_user64(s64 value, const void *addr)
 {
Index: linux/hpc/kcomd.c
===================================================================
--- linux.orig/hpc/kcomd.c	2006-09-18 21:24:38.000000000 +0200
+++ linux/hpc/kcomd.c	2006-09-18 23:28:27.000000000 +0200
@@ -21,11 +21,17 @@
 #include <net/sock.h>
 #include <net/tcp.h>
 
+/**
+ * socket_listen
+ *
+ * Description:
+ *    Creates the network socket and maps it to a file descriptor
+ **/
 static int socket_listen(struct sockaddr *saddr, struct socket **res)
 {
 	struct socket *sock;
 	int ret, fd;
-	
+
 	ret = sock_create(saddr->sa_family, SOCK_STREAM, IPPROTO_TCP, &sock);
 	if (ret < 0)
 		return -1;
@@ -51,6 +57,12 @@
 	return -1;
 }
 
+/**
+ * socket_listen_ip4
+ *
+ * Description:
+ *    IPv4
+ **/
 static int socket_listen_ip4(int port, struct socket **res)
 {
 	struct sockaddr_in saddr4 = {
@@ -62,6 +74,12 @@
 	return socket_listen((struct sockaddr *) &saddr4, res);
 }
 
+/**
+ * socket_listen_ip6
+ *
+ * Description:
+ *    IPv6
+ **/
 static int socket_listen_ip6(int port, struct socket **res)
 {
 	struct sockaddr_in6 saddr6 = {
@@ -96,7 +114,7 @@
 	pid_t pid;              /* pid of the process owning this struct */
 	struct kcom_node *node; /* node of the process to send/recv */
 	struct list_head list;  /* list of process using some node */
-	
+
 	struct list_head out_packs;
 	struct kcom_pkt in_packs;
 };
@@ -193,7 +211,7 @@
 	ret = sock->ops->getname
 	check if it's already in node list.
 	*/
-	
+
 	spin_lock(&kcom_nodes_lock);
 	list_add(&node->list, &kcom_nodes);
 	spin_unlock(&kcom_nodes_lock);
@@ -229,6 +247,15 @@
 int comm_iovec(void);
 int comm_iovec_ack(void);
 
+/**
+ * accept_connection
+ *
+ * Description:
+ *    Once kcomd's sockets receive a new connection attempt,
+ *    the connection is accepted, the remote IP address is
+ *    retrieved, the file descriptor is mapped and the
+ *    kcom node is created with this information.
+ **/
 static int accept_connection(struct socket *lsock)
 {
 	struct socket *sock;
@@ -269,6 +296,14 @@
 	return 0;
 }
 
+/**
+ * data_write
+ *
+ * Description:
+ *    Loops through all tasks that have processes on the node that
+ *    has data to send, and sends the pkts.
+ *    Once the pkt has been sent, its memory is freed.
+ **/
 int data_write(struct kcom_node *node)
 {
 	return 0;
@@ -288,7 +323,7 @@
 		kctask->pid = pid;
 		kctask->node = node;
 		INIT_LIST_HEAD(&kctask->list);
-		
+
 		list_add(&kctask->list, &node->tasks);
 	}
 	return kctask;
@@ -337,7 +372,7 @@
 	tsk = kcom_task_find(pid);
 	if (!tsk)
 		return -ENODEV;
-	
+
 	/* put pkt in kcom_task */
 	pkt = kcom_pkt_create(0, 0, NULL);
 	if (!pkt)
@@ -346,11 +381,24 @@
 
 	/* go to sleep */
 	/* wait reply */
-	
+
 	return 0;
 }
 
 
+/**
+ * kcomd_thread
+ *
+ * Description:
+ *    kcomd - kernel thread that handles the communications.
+ *    Creates the memory slabs.
+ *    Once the pkt has been sent, its memory is freed.
+ *    Maps new connections to file descriptors.
+ *    Waits for incoming data, signals from processes
+ *    or any data that is ready to be sent.
+ *    Also cleans up memory and any open sockets and
+ *    file descriptors on exit.
+ **/
 static int kcomd_thread(void *nothing)
 {
 	int ret;
@@ -381,7 +429,7 @@
 		zero_fd_set(n, sockets_fds.in);
 		zero_fd_set(n, sockets_fds.out);
 		zero_fd_set(n, sockets_fds.ex);
-	
+
 		/* add listening sockets to the set */
 		set_bit(fd4, sockets_fds.in);
 		set_bit(fd6, sockets_fds.in);
Index: linux/hpc/migctrl.c
===================================================================
--- linux.orig/hpc/migctrl.c	2006-09-18 21:24:38.000000000 +0200
+++ linux/hpc/migctrl.c	2006-09-18 23:28:25.000000000 +0200
@@ -31,8 +31,11 @@
 #include <hpc/protocol.h>
 
 /**
- * task_remote_expel - call from REMOTE to send a task to DEPUTY
+ * task_remote_expel
  * @p:		task which will come back
+ *
+ * Description:
+ *    Call from REMOTE to send a task to DEPUTY
  **/
 int task_remote_expel(task_t *p)
 {
@@ -58,8 +61,11 @@
 }
 
 /**
- * task_remote_wait_expel - call from REMOTE to send a task to DEPUTY
+ * task_remote_wait_expel
  * @p:		task which will come back
+ *
+ * Description:
+ *    Call from REMOTE to send a task to DEPUTY
  **/
 int task_remote_wait_expel(task_t *p)
 {
@@ -80,10 +86,13 @@
 }
 
 /**
- * task_local_send - Send a local task to remote
+ * task_local_send
  * @p:		task to send
  * @whereto:	destination sockaddr
  * @reason:	reason to send there (if any)
+ *
+ * Description:
+ *    Send a local task to remote
  **/
 static int task_local_send(task_t *p, struct sockaddr *whereto, int reason)
 {
@@ -127,9 +136,12 @@
 
 
 /**
- * task_local_bring - Receive task back in the deputy stub
+ * task_local_bring
  * @p:		deputy task to receive
  * @reason:	reason to send (if any)
+ *
+ * Description:
+ *    Receive task back in the deputy stub
  **/
 static int task_local_bring(task_t *p, int reason)
 {
@@ -166,10 +178,13 @@
 }
 
 /**
- * task_move_remote2remote - migrate a task from remote to remote
+ * task_move_remote2remote
  * @p:		task to send
  * @whereto:	whereto
  * @reason:	reason to send (if any)
+ *
+ * Description:
+ *    Migrate a task from remote to remote
  **/
 static int task_move_remote2remote(task_t *p, struct sockaddr * whereto,
 								int reason)
@@ -179,10 +194,13 @@
 }
 
 /**
- * task_move_to_node - send a task to a node
+ * task_move_to_node
  * @p:		task to send
  * @whereto:	destination sockaddr
  * @reason:	why
+ *
+ * Description:
+ *    Send a task to a node
  **/
 static int __task_move_to_node(struct task_struct *p,
                                struct sockaddr * whereto,
@@ -217,8 +235,14 @@
 }
 
 /**
- * task_go_home - Migrate task to home
- **/
+ * task_go_home
+ * @p:		task to send
+ * @whereto:	destination sockaddr
+ * @reason:	why
+ *
+ * Description:
+ *    Migrate task to home
+**/
 int task_go_home(task_t *p)
 {
 	if (!task_test_dflags(p, DMIGRATED)) {
@@ -235,7 +259,12 @@
 }
 
 /**
- * task_go_home_for_reason - Migrate back a task for a reason
+ * task_go_home_for_reason
+ * @p:		task to send
+ * @reason:	why
+ *
+ * Description:
+ *    Migrate back a task for a reason
  **/
 int task_go_home_for_reason(task_t *p, int reason)
 {
Index: linux/hpc/migsend.c
===================================================================
--- linux.orig/hpc/migsend.c	2006-09-18 21:24:38.000000000 +0200
+++ linux/hpc/migsend.c	2006-09-18 23:28:26.000000000 +0200
@@ -68,6 +68,12 @@
 	return 0;
 }
 
+/**
+ * mig_send_fp
+ *
+ * Description:
+ *    Sends the process floating point information(?) to the other node.
+ **/
 static int mig_send_fp(task_t *p)
 {
 	struct omp_mig_fp m;
@@ -80,6 +86,13 @@
 }
 
 
+/**
+ * mig_send_mm
+ *
+ * Description:
+ *    Sends the process memory map information to the other node.
+ *    Wait for an acknowledgement
+ **/
 static int mig_send_mm(task_t *p)
 {
 	struct omp_mig_mm s;
@@ -89,6 +102,13 @@
 	return comm_send_hd(p->om.contact, MIG_MM, &s, sizeof(s));
 }
 
+/**
+ * mig_send_vma_file
+ *
+ * Description:
+ *    Called by mig_send_vmas.
+ *    FIXME:  file vmas ??
+ **/
 static inline void mig_send_vma_file(task_t *p, struct vm_area_struct *vma,
 							struct omp_mig_vma *m)
 {
@@ -108,6 +128,14 @@
 	}
 }
 
+/**
+ * mig_send_vmas
+ *
+ * Description:
+ *    loops through and sends all process vmas to the other node.
+ *    vma's are the virtual memory structs.  They hold the lists of
+ *    mapped pages and page permissions.
+ **/
 static int mig_send_vmas(task_t *p)
 {
 	struct vm_area_struct *vma;
@@ -135,6 +163,13 @@
 }
 
 
+/**
+ * mig_send_pages
+ *
+ * Description:
+ *    loops through and sends all process pages to the other node.
+ *    All the process's memory space is sent, one page at a time.
+ **/
 static int mig_send_pages(task_t *p)
 {
 	struct vm_area_struct * vma;
@@ -165,6 +200,13 @@
 }
 
 
+/**
+ * mig_send_proc_context
+ *
+ * Description:
+ *    Sends the 'important' part of the process context.
+ *
+ **/
 static int mig_send_proc_context(task_t *p)
 {
 	struct omp_mig_task m;
@@ -220,6 +262,13 @@
 	return -1;
 }
 
+/**
+ * mig_do_send
+ *
+ * Description:
+ *    Main loop for sending the process to the other node.
+ *
+ **/
 int mig_do_send(task_t *p)
 {
 	arch_mig_send_pre(p);

Index: linux/hpc/kcomd.c
===================================================================
--- linux.orig/hpc/kcomd.c	2006-09-18 23:29:28.000000000 +0200
+++ linux/hpc/kcomd.c	2006-09-20 17:06:58.000000000 +0200
@@ -21,6 +21,12 @@
 #include <net/sock.h>
 #include <net/tcp.h>
 
+#include <linux/inet.h>
+#include <hpc/kcom.h>
+#include <hpc/prototype.h>
+
+static int kcomd_done=0;
+
 /**
  * socket_listen
  *
@@ -93,162 +99,6 @@
 	return socket_listen((struct sockaddr *) &saddr6, res);
 }
 
-struct kcom_pkt
-{
-	pid_t pid;    /* pid of the process */
-	int len;      /* len of data */
-	int type;     /* type of data */
-	char *data;   /* ptr of data */
-	struct list_head list;
-};
-
-struct kcom_node
-{
-	int fd;                 /* fd to send packet */
-	struct socket *sock;    /* socket */
-	struct sockaddr addr;   /* addr of this node */
-	spinlock_t tasks_lock;  /* lock for the list */
-	struct list_head tasks; /* list of task */
-	struct list_head list; /* list of nodes */
-};
-
-struct kcom_task
-{
-	pid_t pid;              /* pid of the process owning this struct */
-	struct kcom_node *node; /* node of the process to send/recv */
-	struct list_head list;  /* list of process using some node */
-
-	struct list_head out_packs;
-	struct kcom_pkt in_packs;
-};
-
-static DEFINE_SPINLOCK(kcom_nodes_lock);
-struct list_head kcom_nodes = LIST_HEAD_INIT(kcom_nodes);
-
-fd_set_bits sockets_fds;
-char *sockets_fds_bitmap = NULL;
-int maxfds = -1;
-
-static int alloc_fd_bitmap(int fd4, int fd6)
-{
-	struct kcom_node *node;
-	int n, size;
-
-	n = max(fd4, fd6);
-
-	spin_lock(&kcom_nodes_lock);
-	list_for_each_entry(node, &kcom_nodes, list)
-		n = max(node->fd, n);
-	spin_unlock(&kcom_nodes_lock);
-
-	/* we don't need to reallocate the bitmap */
-	if (n <= maxfds)
-		return 0;
-	maxfds = n;
-
-	kfree(sockets_fds_bitmap);
-
-	size = FDS_BYTES(n);
-	sockets_fds_bitmap = kmalloc(6 * size, GFP_KERNEL);
-	if (!sockets_fds_bitmap)
-		return ENOMEM;
-
-	sockets_fds.in      = (unsigned long *)  sockets_fds_bitmap;
-	sockets_fds.out     = (unsigned long *) (sockets_fds_bitmap +   size);
-	sockets_fds.ex      = (unsigned long *) (sockets_fds_bitmap + 2*size);
-	sockets_fds.res_in  = (unsigned long *) (sockets_fds_bitmap + 3*size);
-	sockets_fds.res_out = (unsigned long *) (sockets_fds_bitmap + 4*size);
-	sockets_fds.res_ex  = (unsigned long *) (sockets_fds_bitmap + 5*size);
-
-	return 0;
-}
-
-struct kcom_pkt *kcom_pkt_create(int len, int type, char *data)
-{
-	struct kcom_pkt *pkt;
-	pkt = kzalloc(sizeof(struct kcom_pkt), GFP_KERNEL);
-	if (pkt) {
-		pkt->len = len;
-		pkt->type = type;
-		pkt->data = data;
-	}
-	return pkt;
-}
-
-struct kcom_node *__kcom_node_find(struct sockaddr *saddr)
-{
-	struct kcom_node *tmp;
-
-	list_for_each_entry(tmp, &kcom_nodes, list) {
-		/* FIXME compare fields, no memcmp */
-		if (memcmp(saddr, tmp, sizeof(struct sockaddr)) == 0)
-			return tmp;
-	}
-	return NULL;
-}
-
-struct kcom_node *kcom_node_find(struct sockaddr *saddr)
-{
-	struct kcom_node *node;
-
-	spin_lock(&kcom_nodes_lock);
-	node = __kcom_node_find(saddr);
-	spin_unlock(&kcom_nodes_lock);
-	return node;
-}
-
-int kcom_node_add(int fd, struct socket *sock)
-{
-	struct kcom_node *node;
-
-	node = kzalloc(sizeof(struct kcom_node), GFP_KERNEL);
-	if (!node)
-		return -ENOMEM;
-	INIT_LIST_HEAD(&node->list);
-	node->sock = sock;
-	node->fd = fd;
-	/*
-	if (!sock->ops || !sock->ops->getname)
-		goto err;
-
-	ret = sock->ops->getname
-	check if it's already in node list.
-	*/
-
-	spin_lock(&kcom_nodes_lock);
-	list_add(&node->list, &kcom_nodes);
-	spin_unlock(&kcom_nodes_lock);
-	return 0;
-}
-
-int kcom_node_del(struct sockaddr *addr)
-{
-	struct kcom_node *node;
-
-	/* remove the node from the list */
-	spin_lock(&kcom_nodes_lock);
-	node = __kcom_node_find(addr);
-	if (!node) {
-		spin_unlock(&kcom_nodes_lock);
-		return -ENOENT;
-	}
-	list_del(&node->list);
-	spin_unlock(&kcom_nodes_lock);
-
-	/* release and free structure */
-	sys_close(node->fd);
-	sock_release(node->sock);
-	kfree(node);
-	return 0;
-}
-
-int comm_simple(int type, char * data)
-{
-	return 0;
-}
-int comm_ack(void);
-int comm_iovec(void);
-int comm_iovec_ack(void);
 
 /**
  * accept_connection
@@ -262,6 +112,7 @@
 static int accept_connection(struct socket *lsock)
 {
 	struct socket *sock;
+	struct kcom_node *node;
 	int ret, fd;
 	int len;
 	struct sockaddr_in address;
@@ -291,11 +142,16 @@
 		goto err_accept;
 	}
 
-	ret = kcom_node_add(fd, sock);
-	if ( ret < 0 ) {
+	node = kcom_node_add(sock);
+	if (node==NULL) {
 		printk(KERN_ERR "openMosix: Error adding new node\n");
 		goto errfd;
 	}
+	// Store the IP addr.
+	memcpy(&node->addr, &address, sizeof(address));
+	node->fd=fd;
+	// Allocated file descriptor bitmap for do_select
+	alloc_fd_bitmap(fd);
 
 	return fd;
 
@@ -306,12 +162,262 @@
 	return -1;
 }
 
-int data_read(struct kcom_node *node)
+/**
+ * data_send
+ *
+ * Description:
+ *    Sends the kcom pkt header and the data, if any.
+ **/
+int data_send(struct socket *sock, void *data, int len)
+{
+	struct iovec iov;
+	int i=-1;
+	struct msghdr msg = { NULL, 0, &iov, 1, NULL, 0, MSG_WAITALL | MSG_NOSIGNAL };
+	mm_segment_t oldfs;
+	struct kcom_pkt *send_pkt;
+	char buf[32];
+	struct timeval start,stop;
+
+	do_gettimeofday(&start);
+	send_pkt=data;
+
+
+   	/* Send kcom_pkt header */
+	iov.iov_base = send_pkt;
+	iov.iov_len = sizeof(*send_pkt);
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+
+	while (iov.iov_len > 0) {
+		printk("sock_sendmsg hdr\n");
+		i = sock_sendmsg(sock, &msg, iov.iov_len);
+		if ((i == -ENOSPC) || (i == -EAGAIN)) {
+			printk("Retrying hdr...error %d\n", i);
+			schedule_timeout(HZ/1000);
+			continue;
+		}
+		if (i == -EFAULT) {
+			printk("Error %d sending data. Unable to access data.\n", i);
+			printk("Data may need to be copied into a temporary buffer to be sent.\n");
+		}
+
+		if (i < 0) {
+			set_fs(oldfs);
+			return -1;
+		}
+		iov.iov_base += i;
+		iov.iov_len -= i;
+ 	}
+	set_fs(oldfs);
+
+	/* Sending too small a data packet, delays.  */
+	if ((send_pkt->len > 0) && (send_pkt->len < 32)) {
+		memset(&buf, 0, 32);
+		memcpy(&buf, send_pkt->data, send_pkt->len);
+		iov.iov_base = &buf;
+		iov.iov_len = 32;
+	} else {
+		iov.iov_base = send_pkt->data;
+		iov.iov_len = send_pkt->len;
+	}
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	while (iov.iov_len > 0) {
+		printk("sock_sendmsg data\n");
+		i = sock_sendmsg(sock, &msg, iov.iov_len);
+
+		if ((i == -ENOSPC) || (i == -EAGAIN)) {
+
+			printk(KERN_DEBUG "Retrying data...error %d\n", i);
+			schedule_timeout(HZ/1000);
+			continue;
+		}
+		if (i < 0) {
+			printk(KERN_ERR"openMosix: ERROR %d sending data\n", i);
+			set_fs(oldfs);
+			return -1;
+		}
+		iov.iov_base += i;
+		iov.iov_len -= i;
+ 	}
+	set_fs(oldfs);
+	do_gettimeofday(&stop);
+	return i;
+
+}
+
+/**
+ * data_exception
+ *
+ * Description:
+ *    Dropped connections need to be cleaned up.  Memory freed,
+ *    file descriptors unmapped, etc.  This function does that.
+ *    FIXME:  Supposed to, but doesn't work yet.  Dropped connections
+ *    are seen as data, of length 0, is available for read.
+ **/
+int data_exception(struct kcom_node *node)
 {
+
+	sock_release(node->sock);
+	sys_close(node->fd);
+	/* kfree(node->sock);*/
+	list_del(&node->list);
+	kmem_cache_free(kcom_node_cachep, node);
+	/* kfree(node);*/
 	return 0;
 }
 
 /**
+ * append_in_packs
+ *
+ * Description:
+ *    Packets are either new pkts, or responses or (n)acks to new pkts.
+ *    If a pkt isn't new, a function is waiting on it (wait_for_ack/response),
+ *    so we can just add this pkt to the task in_packs list.
+ **/
+int append_in_packs(struct kcom_pkt *recv_kcom_pkt)
+{
+	struct kcom_task *tsk;
+	task_t *sltsk;
+
+	/* FIXME : should spinlock/unlock */
+
+	if ((recv_kcom_pkt->type & NODE_MASK) == DEP_FLG) {
+		/* command from dep to remote? */
+
+		tsk=kcom_remote_task_find(recv_kcom_pkt->rpid);
+		if (tsk) {
+			/* spin_lock(&tsk->spinlock); */
+			list_add_tail(&recv_kcom_pkt->list, &tsk->in_packs);
+			/* spin_unlock(&tsk->spinlock); */
+		} else {
+			printk(KERN_ERR "unable to find remote pid %u\n", recv_kcom_pkt->rpid);
+			return -1;
+
+		}
+		/* read_lock(&tasklist_lock); */
+		sltsk=find_task_by_pid(recv_kcom_pkt->rpid);
+		/* read_unlock(&tasklist_lock); */
+		if (sltsk) {
+			wake_up_process(sltsk);
+		} else {
+			printk(KERN_ERR "Unable to find remote pid %u to wake up\n", recv_kcom_pkt->rpid);
+			return -1;
+		}
+
+	} else {
+		tsk=kcom_home_task_find(recv_kcom_pkt->hpid);
+		if (tsk) {
+			/* spin_lock(&tsk->spinlock); */
+			list_add_tail(&recv_kcom_pkt->list, &tsk->in_packs);
+			/* spin_unlock(&tsk->spinlock); */
+		} else {
+			printk(KERN_ERR "unable to find home pid %u\n", recv_kcom_pkt->hpid);
+			return -1;
+		}
+		/* read_lock(&tasklist_lock); */
+		sltsk=find_task_by_pid(recv_kcom_pkt->hpid);
+		/* read_unlock(&tasklist_lock); */
+		if (sltsk) {
+			wake_up_process(sltsk);
+		} else {
+			printk(KERN_ERR "Unable to find home pid %u to wake up\n", recv_kcom_pkt->hpid);
+			return -1;
+		}
+	}
+	return 0;
+}
+
+/**
+ * pkt_read
+ *
+ * Description:
+ *    Reads the packet header and, if exists, data,  and put in appropriate task's
+ *    in_pack list.
+ *    All but 3 pkts an be handled by the task itself.
+ *    MIG_INIT creates a new process and task
+ *    MIG_GO/COME_HOME - migration command.
+ **/
+int pkt_read(struct kcom_node *node)
+{
+	struct kcom_pkt *recv_kcom_pkt;
+	int i=0;
+	task_t *sltsk;
+
+	// read in hdr
+	recv_kcom_pkt=pkt_hdr_read(node);
+	if (recv_kcom_pkt==NULL) {
+		printk("ERROR: incomplete header pkt\n");
+		goto error_recv;
+	}
+
+	// read in any data
+	if (recv_kcom_pkt->len > 0) {
+		if ((recv_kcom_pkt->type & MSG_MASK) == PKT_NEW_MSG) {
+			recv_kcom_pkt->data=kzalloc(recv_kcom_pkt->len, GFP_KERNEL);
+			i=pkt_data_read(node, recv_kcom_pkt, recv_kcom_pkt->len, recv_kcom_pkt->data);
+		} else {
+			i=pkt_data_read(node, recv_kcom_pkt, recv_kcom_pkt->len, recv_kcom_pkt->resp);
+		}
+		if (i<recv_kcom_pkt->len) {
+			printk("ERROR: incomplete data pkt\n");
+			goto error_recv;
+		}
+	}
+
+
+	if ((recv_kcom_pkt->type & MSG_MASK) == PKT_NEW_MSG) {
+		switch (recv_kcom_pkt->type & MIG_MASK) {
+			case MIG_INIT:
+				mig_do_receive_init(node, recv_kcom_pkt);
+				break;
+			case MIG_GO_HOME:
+				mig_do_receive_home(node, recv_kcom_pkt);
+				break;
+			case MIG_COME_HOME:
+				sltsk=find_task_by_pid(recv_kcom_pkt->rpid);
+				task_register_migration(sltsk);
+				break;
+			default:
+				append_in_packs(recv_kcom_pkt);
+
+				break;
+		}
+	} else { // PKT_ACK and PKT_RESP go straight to in_packs
+		append_in_packs(recv_kcom_pkt);
+	}
+
+	return 0;
+
+error_recv:
+	printk("sock_close\n");
+	sys_close(node->fd);
+	printk("sock_release\n");
+	sock_release(node->sock);
+
+	#if 0
+	node->fd=-1;
+	printk("kfree(data)\n");
+	kfree(pkt);
+	printk("kfree(node->sock)\n");
+	kfree(node->sock);
+	printk("list_del\n");
+	#endif
+
+	list_del(&node->list);
+	kmem_cache_free(kcom_node_cachep, node);
+
+	#if 0
+	kfree(node);
+	#endif
+	return -1;
+
+}
+
+
+
+/**
  * data_write
  *
  * Description:
@@ -321,6 +427,19 @@
  **/
 int data_write(struct kcom_node *node)
 {
+	struct kcom_task *task, *task_next;
+	struct kcom_pkt *pkt, *pkt_next;
+
+	list_for_each_entry_safe(task, task_next, &node->tasks, list)
+		list_for_each_entry_safe(pkt, pkt_next, &task->out_packs, list) {
+			data_send(node->sock, (void *)pkt, pkt->len);
+
+			list_del(&pkt->list);
+			if (((pkt->type|PKT_ACK)==PKT_ACK) && (pkt->len > 0))
+				kmem_cache_free(kcom_data_cachep, pkt->data);
+			kmem_cache_free(kcom_pkt_cachep, pkt);
+		}
+
 	return 0;
 }
 
@@ -417,28 +536,53 @@
 static int kcomd_thread(void *nothing)
 {
 	int ret;
-	struct socket *lsock4, *lsock6;
-	int fd4, fd6;
-
+	struct kcom_node *node, *node_next;
+	struct kcom_task *task, *task_next;
+	struct kcom_pkt *pkt, *pkt_next;
+	s64 timeout = -1;
+	int n = -1;
+	siginfo_t info; /* matt*/
+	int sig;
+	int outpkt_cnt, inpkt_cnt, tsk_cnt;
+	int err;
+	struct timeval start,stop;
+
+	fd4=-1;
+	fd6=-1;
+	kcomd_done=0;
 	printk(KERN_INFO "kcomd: init\n");
+
+	kcom_data_cachep=kmem_cache_create("kcom_data_cache", 1024, 0, 0, NULL, NULL); /* for now help chase down memory leaks*/
+	kcom_pkt_cachep=kmem_cache_create("kcom_pkt_cache", sizeof(struct kcom_pkt), 0, 0, NULL, NULL);
+	kcom_task_cachep=kmem_cache_create("kcom_task_cache", sizeof(struct kcom_task), 0, 0, NULL, NULL);
+	kcom_node_cachep=kmem_cache_create("kcom_node_cache", sizeof(struct kcom_node), 0, 0, NULL, NULL);
+	/* kcom_saddr_cachep=kmem_cache_create("kcom_saddr_cache", sizeof(struct sockaddr), 0, 0, NULL, NULL);*/
+
 	daemonize("kcomd", 0);
+	sigfillset(&current->blocked);
+
+	/* kcom_pid=current->pid;*/
+	/* read_lock(&tasklist_lock);*/
+	/* kcomd_task = find_task_by_pid(current->pid);*/
+	/* read_unlock(&tasklist_lock);*/
+	kcomd_task=current;
 
 retry_listen:
-	fd4 = socket_listen_ip4(0xb55, &lsock4);
-	fd6 = socket_listen_ip6(0xb56, &lsock6);
+	fd4 = socket_listen_ip4(DAEMON_IP4_PORT, &lsock4);
+	fd6 = socket_listen_ip6(DAEMON_IP6_PORT, &lsock6);
 
 	if (fd4 == -1 && fd6 == -1) {
 		schedule_timeout_interruptible(HZ);
 		goto retry_listen;
 	}
 
-	while (1)
+
+	n=alloc_fd_bitmap(max(fd4, fd6));
+
+	while (kcomd_done==0)
 	{
-		s64 timeout = -1;
-		int n = -1;
-		struct kcom_node *node;
 
-		alloc_fd_bitmap(fd4, fd6);
+
 		n = maxfds;
 
 		zero_fd_set(n, sockets_fds.in);
@@ -446,61 +590,221 @@
 		zero_fd_set(n, sockets_fds.ex);
 
 		/* add listening sockets to the set */
-		set_bit(fd4, sockets_fds.in);
-		set_bit(fd6, sockets_fds.in);
+		if (fd4!=-1)
+			set_bit(fd4, sockets_fds.in);
+		if (fd6!=-1)
+			set_bit(fd6, sockets_fds.in);
 
 		/* for each nodes (set fds.in && fds.out) */
-		spin_lock(&kcom_nodes_lock);
+		/* printk("spin_lock\n");*/
+		/* spin_lock(&kcom_nodes_lock);*/
+		/* printk("Setting bits for select (n=%d, maxfds=%d).\n", n, maxfds);*/
+	do_gettimeofday(&start);
 		list_for_each_entry(node, &kcom_nodes, list) {
-			struct kcom_task *task;
+			/* printk("node->fd=%d\n", node->fd);*/
 
-			if (node->fd == -1 || node->fd > maxfds)
+			if (node->fd == -1 || node->fd > maxfds) {
+				/* spin_unlock(&kcom_nodes_lock);*/
 				continue;
+			}
+
+			/* if a file descriptor is open, we want select to pay attention.*/
+			/* printk("Setting in bit for fd%d\n", node->fd);*/
 			set_bit(node->fd, sockets_fds.in);
+			set_bit(node->fd, sockets_fds.ex);
+
+			/* if there are packets to be sent, select should pay attention.*/
 			list_for_each_entry(task, &node->tasks, list)
-				if (!list_empty(&task->out_packs))
+				if (!list_empty(&task->out_packs)) {
+					/* printk("Setting out bit for fd%d\n", node->fd);*/
 					set_bit(node->fd, sockets_fds.out);
+				}
 		}
-		spin_unlock(&kcom_nodes_lock);
+	do_gettimeofday(&stop);
+	if (stop.tv_sec==start.tv_sec)
+		printk("node_loopB: time %lu usecs\n", stop.tv_usec-start.tv_usec);
+	else
+		printk("node_loopB: time %lu usecs\n", ((stop.tv_sec*USEC_PER_SEC)+stop.tv_usec)-((start.tv_sec*USEC_PER_SEC)+start.tv_usec));
+		/* spin_unlock(&kcom_nodes_lock);*/
+		/* printk("Done setting bits for select.\n");*/
+
+
 
 		zero_fd_set(n, sockets_fds.res_in);
 		zero_fd_set(n, sockets_fds.res_out);
 		zero_fd_set(n, sockets_fds.res_ex);
 
+	do_gettimeofday(&start);
+		allow_signal (SIGHUP);
+		timeout=-1;
+		printk("............DO_SELECT...........");
 		ret = do_select(n + 1, &sockets_fds, &timeout);
-		if (ret < 0)
+		printk(" returned %d.\n", ret);
+		/* SIGHUP is sent by kcom_send to wake up kcomd so it can send the packet*/
+		while (signal_pending (current)) {
+			printk("[kcomd] dequeueing signal\n");
+			sig = dequeue_signal (current, &current->blocked, &info);
+		}
+		disallow_signal (SIGHUP);
+		if (ret == 0) { /* -1=error; 0=signal*/
+			/* New kernel security, prohibits sharing file descriptors between kernel threads.*/
+			/* We have to allocate them here.*/
+			/* We'll do that for the signal so we don't have to do it every iteration.*/
+
+			list_for_each_entry(node, &kcom_nodes, list)
+				if (node->fd==0) { /* unmapped*/
+					/* printk("Found unmapped socket\n");*/
+					node->fd = sock_map_fd(node->sock);
+					alloc_fd_bitmap(node->fd);
+				}
 			continue;
+		} else if (ret < 0) {/* -1=error; 0=signal*/
+			printk("do_select returned an error. \n");
+			schedule_timeout_interruptible(HZ);
+		}
+	do_gettimeofday(&stop);
+	if (stop.tv_sec==start.tv_sec)
+		printk("do_select: time %lu usecs\n", stop.tv_usec-start.tv_usec);
+	else
+		printk("do_select: time %lu usecs\n", ((stop.tv_sec*USEC_PER_SEC)+stop.tv_usec)-((start.tv_sec*USEC_PER_SEC)+start.tv_usec));
 
 		/* test listening sockets */
-		if (fd4 != -1 && test_bit(fd4, sockets_fds.res_in))
+		if (fd4 != -1 && test_bit(fd4, sockets_fds.res_in)) {
 			accept_connection(lsock4);
-		if (fd6 != -1 && test_bit(fd6, sockets_fds.res_in))
+			continue;
+		}
+
+		if (fd6 != -1 && test_bit(fd6, sockets_fds.res_in)) {
 			accept_connection(lsock6);
+			continue;
+		}
 
 		/* for each nodes { test bit, in, out and do stuff } */
-		spin_lock(&kcom_nodes_lock);
-		list_for_each_entry(node, &kcom_nodes, list) {
-			if (test_bit(node->fd, sockets_fds.res_in))
-				data_read(node);
-			if (test_bit(node->fd, sockets_fds.res_out))
+		/* spin_lock(&kcom_nodes_lock);*/
+restart:
+		list_for_each_entry_safe(node, node_next, &kcom_nodes, list) {
+			/*
+			if (test_bit(node->fd, sockets_fds.res_ex)) {
+				data_exception(node);
+				goto restart;
+			}
+			*/
+			if (test_bit(node->fd, sockets_fds.res_in)) {
+				do_gettimeofday(&start);
+				err=pkt_read(node);
+				do_gettimeofday(&stop);
+				if (stop.tv_sec==start.tv_sec)
+					printk("pkt_read: time %lu usecs\n", stop.tv_usec-start.tv_usec);
+				else
+					printk("pkt_read: time %lu usecs\n", ((stop.tv_sec*USEC_PER_SEC)+stop.tv_usec)-((start.tv_sec*USEC_PER_SEC)+start.tv_usec));
+				if (err!=0) {
+					kcomd_done=1;
+					printk("ERROR receiving data.\nQuitting.\n");
+					continue;
+				}
+			}
+			if (node->fd!=-1 && test_bit(node->fd, sockets_fds.res_out)) {
+				do_gettimeofday(&start);
 				data_write(node);
+				do_gettimeofday(&stop);
+				if (stop.tv_sec==start.tv_sec)
+					printk("data_write: time %lu usecs\n", stop.tv_usec-start.tv_usec);
+				else
+					printk("data_write: time %lu usecs\n", ((stop.tv_sec*USEC_PER_SEC)+stop.tv_usec)-((start.tv_sec*USEC_PER_SEC)+start.tv_usec));
+			}
 		}
-		spin_unlock(&kcom_nodes_lock);
+		/* spin_unlock(&kcom_nodes_lock);*/
 	}
 
-	return -1;
+/* DONE*/
+	printk(KERN_INFO "kcomd: cleaning up.\n");
+	/* spin_lock(&kcom_nodes_lock);*/
+	if (fd4!=-1) {
+		printk("fd4=%d\n", fd4);
+		printk("sys_close(fd4)\n");
+		sys_close(fd4);
+		if (lsock4) {  /* doesn't like this.  Help????????*/
+			printk("sock_release(lsock4)\n");
+			sock_release(lsock4);
+		}
+	}
+
+	if (fd6!=-1) {
+		printk("fd6=%d\n", fd6);
+		if (lsock6) { /* doesn't like this.  Help????????*/
+			printk("sock_release(lsock6)\n");
+			sock_release(lsock6);
+		}
+		printk("sys_close(fd6)\n");
+		sys_close(fd6);
+	}
+	printk("Stopped accepting new connections.\n");
+
+	list_for_each_entry_safe(node, node_next, &kcom_nodes, list) {
+		list_for_each_entry_safe(task, task_next, &node->tasks, list) {
+			list_for_each_entry_safe(pkt, pkt_next, &task->in_packs, list) {
+				if ((pkt->len) > 0)
+					kfree(pkt->data);
+				kmem_cache_free(kcom_pkt_cachep, pkt);
+				list_del(&pkt->list);
+			}
+			list_for_each_entry_safe(pkt, pkt_next, &task->out_packs, list) {
+				if ((pkt->len) > 0)
+					kfree(pkt->data);
+				kmem_cache_free(kcom_pkt_cachep, pkt);
+				list_del(&pkt->list);
+			}
+			list_del(&task->list);
+			kmem_cache_free(kcom_task_cachep, task);
+		}
+
+
+		printk("sock_release, %d\n", node->fd);
+		sock_release(node->sock);
+		printk("sys_close\n");
+		sys_close(node->fd);
+		/* kfree(node->sock);*/
+		printk("list_del\n");
+		list_del(&node->list);
+		printk("kmem_cache_free\n");
+		kmem_cache_free(kcom_node_cachep, node);
+		/* kfree(node);*/
+	}
+	/* spin_unlock(&kcom_nodes_lock);*/
+	kfree(sockets_fds_bitmap);
+
+	kmem_cache_destroy(kcom_data_cachep);
+	kmem_cache_destroy(kcom_pkt_cachep);
+	kmem_cache_destroy(kcom_task_cachep);
+	kmem_cache_destroy(kcom_node_cachep);
+
+	kcomd_task=NULL;
+	printk(KERN_INFO "kcomd: exit\n");
+	return 0;
 }
 
 static int __init kcomd_init(void)
 {
 	long ret;
 
-	ret = kernel_thread(kcomd_thread, NULL, 0);
+	ret = kernel_thread(kcomd_thread, NULL, CLONE_FS | CLONE_FILES);
 	return ret;
 }
 
 static void __exit kcomd_exit(void)
 {
+
+	/*
+	 * Set the exit condition and signal kcomd
+	 * kcomd cleans up after itself
+	 */
+	kcomd_done=1;
+
+	if (kcomd_task)
+		send_sig(SIGHUP, kcomd_task, 0);
+	else
+		printk(KERN_WARNING "Unable to to find kcomd daemon\n");
+
 }
 
 module_init(kcomd_init);
Index: linux/hpc/Makefile
===================================================================
--- linux.orig/hpc/Makefile	2006-09-18 23:28:26.000000000 +0200
+++ linux/hpc/Makefile	2006-09-18 23:29:28.000000000 +0200
@@ -2,7 +2,7 @@
 obj-$(CONFIG_KCOMD)     += kcomd.o
 
 # core part
-obj-$(CONFIG_OPENMOSIX)		+= kernel.o task.o comm.o
+obj-$(CONFIG_OPENMOSIX)		+= kernel.o task.o comm.o kcom.o
 obj-$(CONFIG_OPENMOSIX)		+= remote.o deputy.o copyuser.o files.o syscalls.o
 obj-$(CONFIG_OPENMOSIX)		+= migrecv.o migsend.o migctrl.o
 obj-$(CONFIG_OPENMOSIX)		+= service.o
Index: linux/include/hpc/protocol.h
===================================================================
--- linux.orig/include/hpc/protocol.h	2006-09-18 23:28:26.000000000 +0200
+++ linux/include/hpc/protocol.h	2006-09-18 23:29:28.000000000 +0200
@@ -34,12 +34,12 @@
 	int personality;	/* of process to be sent */
 };
 /* handshake types */
+
 #define HSHAKE_MIG_REQUEST	0x01
 #define HSHAKE_DEPUTY_PROBE	0x02
 #define HSHAKE_REPLY		0x04
 #define HSHAKE_NOTOK		0x08
 
-
 /* main structure for passing messages between
  * DEPUTY and REMOTE. Denotes the type of request,
  * and the length of it */
@@ -48,30 +48,68 @@
 	int type;
 	int dlen;
 };
+// pkt type
 
-#define DEP_FLG		0x100
-#define MIG_FLG		0x200
-#define REM_FLG		0x400
-#define REPLY		0x800
+// bits 3:0 - ACK
+#define MSG_MASK 		0xF
+#define PKT_NEW_MSG	0x0
+#define PKT_ACK		0x1
+#define PKT_RESP		0x2
+#define PKT_NACK		0x3
+
+#define NODE_MASK		0xF000
+#define DEP_FLG		0x1000
+#define MIG_FLG		0x2000
+#define REM_FLG		0x4000
+#define REPLY			0x8000
+
+#define DEP_PKT_NEW_MSG	(DEP_FLAG | PKT_NEW_MSG)
+#define DEP_PKT_ACK		(DEP_FLAG | PKT_ACK)
+#define DEP_PKT_RESP		(DEP_FLAG | PKT_RESP)
+#define DEP_PKT_NACK		(DEP_FLAG | PKT_NACK)
+
+#define REM_PKT_NEW_MSG	(REM_FLAG | PKT_NEW_MSG)
+#define REM_PKT_ACK		(REM_FLAG | PKT_ACK)
+#define REM_PKT_RESP		(REM_FLAG | PKT_RESP)
+#define REM_PKT_NACK		(REM_FLAG | PKT_NACK)
 
 /* commands sent during migration */
-#define MIG_MM		(MIG_FLG | 0x10)
-#define MIG_VMA		(MIG_FLG | 0x11)
-#define MIG_PAGE	(MIG_FLG | 0x12)
-#define MIG_FP		(MIG_FLG | 0x13)
-#define MIG_ARCH	(MIG_FLG | 0x14)
-#define MIG_TASK	(MIG_FLG | 0x15)
-#define MIG_ABORT	(MIG_FLG | 0x16)
+// #define MIG_MM		(MIG_FLG | 0x10)
+// #define MIG_VMA		(MIG_FLG | 0x11)
+// #define MIG_PAGE	(MIG_FLG | 0x12)
+// #define MIG_FP		(MIG_FLG | 0x13)
+// #define MIG_ARCH	(MIG_FLG | 0x14)
+// #define MIG_TASK	(MIG_FLG | 0x15)
+// #define MIG_ABORT	(MIG_FLG | 0x16)
+
+// PKT_TYPES
+#define MIG_MASK	0xF0
+#define MIG_PING	0x10
+#define MIG_INIT	0x20
+#define MIG_MM		0x30
+#define MIG_VMA	0x40
+#define MIG_PAGE	0x50
+#define MIG_FP		0x60
+#define MIG_ARCH	0x70
+#define MIG_TASK	0x80
+#define MIG_GO_HOME	0x90
+#define MIG_COME_HOME	0xA0
+#define MIG_ABORT	0xB0
+#define MIG_SYSCALL	0xC0
+
+#define MIG_HOME	MIG_GO_HOME
 
 /* commands sent by deputy to remote */
-#define DEP_COPY_FROM_USER	(DEP_FLG | 0x01)
-#define DEP_COPY_TO_USER	(DEP_FLG | 0x02)
-#define DEP_STRNCPY_FROM_USER	(DEP_FLG | 0x03)
-#define DEP_STRNLEN_USER	(DEP_FLG | 0x04)
-#define DEP_GET_USER		(DEP_FLG | 0x05)
-#define DEP_PUT_USER		(DEP_FLG | 0x06)
-#define DEP_SIGNAL		(DEP_FLG | 0x07)
-#define DEP_COMING_HOME		(DEP_FLG | 0x10)
+#define SYSCALL_MASK 0xF00
+#define DEP_COPY_FROM_USER	0x100
+#define DEP_COPY_TO_USER	0x200
+#define DEP_STRNCPY_FROM_USER	0x300
+#define DEP_STRNLEN_USER	0x400
+#define DEP_GET_USER		0x500
+#define DEP_PUT_USER		0x600
+#define DEP_SIGNAL		0x700
+#define DEP_COMING_HOME		0x800
+#define SYSCALL_DONE		0x900
 
 /* commands sent by remote to deputy */
 #define REM_BRING_HOME	(REM_FLG | 0x10)
@@ -107,7 +145,7 @@
 	struct pt_regs regs;
 
 	struct omp_mig_arch_task arch;
-	char comm[TASK_COMM_LEN]; 
+	char comm[TASK_COMM_LEN];
 };
 
 /* mm_struct values */
Index: linux/include/hpc/prototype.h
===================================================================
--- linux.orig/include/hpc/prototype.h	2006-09-18 23:28:26.000000000 +0200
+++ linux/include/hpc/prototype.h	2006-09-20 17:07:33.000000000 +0200
@@ -28,6 +28,8 @@
 
 #define OMBUG(f, a...)	printk(KERN_ERR "[OMBUG] %s: " f, __FUNCTION__, ## a)
 
+#include <linux/in.h>
+#include <hpc/kcom.h>
 
 NORET_TYPE void		deputy_die_on_communication(void);
 void			deputy_main_loop(void);
@@ -83,4 +85,39 @@
 				struct vm_area_struct *vma);
 int		remote_readpage(struct file *file, struct page *page);
 
+int remote_handle_user(task_t *, int);
+
+int alloc_fd_bitmap(int);
+
+struct kcom_pkt *kcom_pkt_create(int, int, int, char *);
+
+int kcom_send(int, int, char *, unsigned long, struct sockaddr_in *);
+int kcom_send_with_ack(int, int, char *, unsigned long, struct sockaddr_in *);
+int kcom_send_with_response(int, int, char *, unsigned long, char *, struct sockaddr_in *);
+int wait_for_ack(struct kcom_task *, unsigned int);
+int wait_for_response(struct kcom_task *, unsigned int);
+int kcom_send_ack(task_t *, struct kcom_pkt *);
+int kcom_send_resp(task_t *, int , char *, struct kcom_pkt *);
+
+struct kcom_node *kcom_node_add(struct socket *);
+struct kcom_node *kcom_node_find(struct sockaddr *);
+
+struct kcom_task *kcom_task_create(struct kcom_node *, int);
+struct kcom_task *kcom_task_find(int);
+int kcom_task_send(struct kcom_task *, int, int, char *, char *, unsigned long);
+int kcom_task_delete(int);
+
+int mig_handle_migration(int *);
+int user_thread(int (*fn)(void *), void * arg, unsigned long flags);
+int mig_init(struct kcom_node *,struct kcom_pkt *);
+
+struct kcom_pkt* pkt_hdr_read(struct kcom_node *);
+int pkt_data_read(struct kcom_node *, struct kcom_pkt *, int, char *);
+
+struct kcom_task *kcom_remote_task_find(int);
+struct kcom_task *kcom_home_task_find(int);
+int mig_do_receive_home(struct kcom_node *node, struct kcom_pkt *recv_kcom_pkt);
+int mig_do_receive_init(struct kcom_node *node, struct kcom_pkt *recv_kcom_pkt);
+
+int remote_do_signal(task_t *,  struct kcom_pkt *);
 #endif /* _HPC_PROTOTYPE_H */
Index: linux/include/hpc/task.h
===================================================================
--- linux.orig/include/hpc/task.h	2006-09-18 23:28:26.000000000 +0200
+++ linux/include/hpc/task.h	2006-09-18 23:29:28.000000000 +0200
@@ -142,7 +142,7 @@
 void task_do_request(void);
 
 struct sockaddr;
-int task_register_migration(task_t *p, struct sockaddr *);
+int task_register_migration(task_t *p);
 
 struct inode;
 int task_maps_inode(task_t *p, struct inode *);
Index: linux/net/socket.c
===================================================================
--- linux.orig/net/socket.c	2006-09-18 23:29:28.000000000 +0200
+++ linux/net/socket.c	2006-09-18 23:29:28.000000000 +0200
@@ -530,7 +530,7 @@
 	put_cpu_var(sockets_in_use);
 	return sock;
 }
-#ifdef CONFIG_KCOMD
+#if defined(CONFIG_KCOMD) || defined (CONFIG_KCOMD_MODULE)
 EXPORT_SYMBOL_GPL(sock_alloc);
 #endif
 
Index: linux/fs/select.c
===================================================================
--- linux.orig/fs/select.c	2006-09-18 23:28:26.000000000 +0200
+++ linux/fs/select.c	2006-09-18 23:29:28.000000000 +0200
@@ -294,7 +294,7 @@
 
 	return retval;
 }
-#ifdef CONFIG_KCOMD
+#ifdef CONFIG_KCOMD_MODULE
 EXPORT_SYMBOL_GPL(do_select);
 #endif
 
Index: linux/include/linux/compiler.h
===================================================================
--- linux.orig/include/linux/compiler.h	2006-09-18 23:54:41.000000000 +0200
+++ linux/include/linux/compiler.h	2006-09-18 23:55:05.000000000 +0200
@@ -44,7 +44,7 @@
 #define OM_NSTATIC static
 #endif
 
-#ifdef CONFIG_KCOMD
+#if defined(CONFIG_KCOMD) || defined(CONFIG_KCOMD_MODULE)
 #define KCOMD_NSTATIC
 #else
 #define KCOMD_NSTATIC static

Index: linux/hpc/kcom.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/hpc/kcom.c	2006-09-20 17:48:20.000000000 +0200
@@ -0,0 +1,1061 @@
+/*
+ *	Copyright (C) 2006 Matt Dew <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <linux/sched.h>
+#include <linux/socket.h>
+#include <linux/in.h>
+#include <linux/in6.h>
+#include <linux/net.h>
+#include <linux/syscalls.h>
+#include <net/sock.h>
+#include <net/tcp.h>
+#define _HPC_KCOMC_H 1
+#include <hpc/kcom.h>
+#include <hpc/prototype.h>
+
+
+/**
+ * pkt_data_read
+ *
+ * Description:
+ * 	read the data that was send following the pkt header.
+ * 	wait until all data has been read.
+ * 	The ->len field = size of the data in bytes.
+ **/
+int pkt_data_read(struct kcom_node *node, struct kcom_pkt *pkt, int len, char *data)
+{
+	struct socket *sock=node->sock;
+	struct iovec iov;
+	struct msghdr msg = { NULL, 0, &iov, 1, NULL, 0, MSG_WAITALL | MSG_NOSIGNAL };
+	mm_segment_t oldfs;
+	int i;
+	char buf[32];
+
+	iov.iov_base = data;
+	iov.iov_len = len;
+
+	// Too small of a packet gets delayed before being sent. Even with TCP_NODELAY
+	if (len < 32) {
+		iov.iov_base = &buf;
+		iov.iov_len = 32;
+	}
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+
+	while (iov.iov_len > 0) {
+		i = sock_recvmsg(sock, &msg, iov.iov_len, msg.msg_flags);
+		if ((i == -ENOSPC) || (i == -EAGAIN)) {
+			schedule_timeout(HZ/1000);
+			continue;
+		}
+		if (i < 0) {
+			printk("ERROR %d receiving data.\n", i);
+			set_fs(oldfs);
+			return -1;
+		}
+		iov.iov_base += i;
+ 	}
+
+	set_fs(oldfs);
+
+	if (len < 32) {
+		memcpy(data, buf, len);
+	}
+
+	return len;
+}
+EXPORT_SYMBOL_GPL(pkt_data_read);
+
+/**
+ * pkt_hdr_read
+ *
+ * Description:
+ * 	read the pkt header of the data transmission.
+ * 	The hdr indicates the type and size of the data.
+ * 	All packet headers are the same size.
+ **/
+struct kcom_pkt* pkt_hdr_read(struct kcom_node *node)
+{
+	struct iovec iov;
+	struct msghdr msg = { NULL, 0, &iov, 1, NULL, 0, MSG_WAITALL | MSG_NOSIGNAL };
+	struct kcom_pkt *recv_kcom_pkt;
+	mm_segment_t oldfs;
+	struct socket *sock=node->sock;
+	int i;
+
+	recv_kcom_pkt=kmem_cache_alloc(kcom_pkt_cachep, SLAB_KERNEL);
+	iov.iov_base = recv_kcom_pkt;
+	iov.iov_len = sizeof(*recv_kcom_pkt);
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+	while ( iov.iov_len > 0 ) {
+		i = sock_recvmsg(sock, &msg, iov.iov_len, msg.msg_flags);
+		if ((i == -ENOSPC) || (i == -EAGAIN)) {
+			schedule_timeout(HZ/1000);
+			continue;
+		}
+		if (i < 0) {
+			printk("ERROR %d receiving header.\n", i);
+			set_fs(oldfs);
+			return NULL;
+		}
+		iov.iov_base += i;
+ 	}
+	set_fs(oldfs);
+
+	if (iov.iov_len==0) // all expected data received.
+		return recv_kcom_pkt;
+	else
+		return NULL;
+}
+EXPORT_SYMBOL_GPL(pkt_hdr_read);
+
+
+/**
+ * alloc_fd_bitmap
+ *
+ * Description:
+ * 	Allocate a large enough file descriptor bitmap
+ * 	for the do_select function to operate correctly
+ * 	on all open sockets.
+ *    If more space needs to be allocated, it will be
+ **/
+int alloc_fd_bitmap(int fd)
+{
+	struct kcom_node *node;
+	int size;
+	int n=fd;
+
+	if (fd <= maxfds)
+		return 0;
+
+	n = max(fd,max(fd4, fd6));
+	// spin_lock(&kcom_nodes_lock);
+	list_for_each_entry(node, &kcom_nodes, list)
+		n = max(node->fd, n);
+	// spin_unlock(&kcom_nodes_lock);
+
+
+	maxfds = n;
+
+	kfree(sockets_fds_bitmap);
+
+	size = FDS_BYTES(n);
+	sockets_fds_bitmap = kmalloc(6 * size, GFP_KERNEL);
+	if (!sockets_fds_bitmap)
+		return ENOMEM;
+
+	sockets_fds.in      = (unsigned long *)  sockets_fds_bitmap;
+	sockets_fds.out     = (unsigned long *) (sockets_fds_bitmap +   size);
+	sockets_fds.ex      = (unsigned long *) (sockets_fds_bitmap + 2*size);
+	sockets_fds.res_in  = (unsigned long *) (sockets_fds_bitmap + 3*size);
+	sockets_fds.res_out = (unsigned long *) (sockets_fds_bitmap + 4*size);
+	sockets_fds.res_ex  = (unsigned long *) (sockets_fds_bitmap + 5*size);
+
+	return 0;
+
+}
+EXPORT_SYMBOL_GPL(alloc_fd_bitmap);
+
+/**
+ * kcom_pkt_create
+ *
+ * Description:
+ * 	Allocate a kcom pkt from the slab.
+ * 	Initialize the pkt len,type and data ptr.
+ *    If this is a new packet, increment the global msgid counter.
+ **/
+struct kcom_pkt *kcom_pkt_create(int len, int type, int ack, char *data)
+{
+	struct kcom_pkt *pkt;
+
+	pkt=kmem_cache_alloc(kcom_pkt_cachep, SLAB_KERNEL);
+	if (pkt) {
+		pkt->len = len;
+		pkt->type = type;
+
+		if (len > 0)
+			pkt->data=data;
+		else
+			pkt->data=NULL;
+
+		// acks and responses don't get new a msgid
+		// FIXME:  is this inc SMP safe?
+		if ((type & MSG_MASK) == PKT_NEW_MSG)
+			pkt->msgid = kcom_msgid++;
+
+		return pkt;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(kcom_pkt_create);
+
+/**
+ * __kcom_node_find
+ *
+ * Description:
+ * 	Does the actual work of finding, if it exists,
+ * 	an existing node connection.
+ *    The IP address is the determiner.
+ **/
+struct kcom_node *__kcom_node_find(struct sockaddr *saddr)
+{
+	struct kcom_node *tmp;
+	struct sockaddr_in *saddr_tmp;
+	struct sockaddr_in *saddr_in=(struct sockaddr_in *)saddr;
+	unsigned long int node_addr, find_addr;
+
+	find_addr=saddr_in->sin_addr.s_addr;
+
+	list_for_each_entry(tmp, &kcom_nodes, list) {
+		saddr_tmp=(struct sockaddr_in *)&tmp->addr;
+		node_addr=saddr_tmp->sin_addr.s_addr;
+
+		if ((saddr_in->sin_family == saddr_tmp->sin_family) &&
+			(find_addr == node_addr))
+			return tmp;
+	}
+	return NULL;
+}
+
+
+/**
+ * kcom_node_find
+ *
+ * Description:
+ * 	calls __kcom_node_find
+ *    ,  which searches for a node
+ **/
+struct kcom_node *kcom_node_find(struct sockaddr *saddr)
+{
+	task_t *p=current;
+	struct kcom_node *node;
+	struct sockaddr_in *addr_tmp;
+	unsigned int addr;
+
+	addr_tmp=(struct sockaddr_in *)p->om.whereto;
+	addr=addr_tmp->sin_addr.s_addr;
+	// spin_lock(&kcom_nodes_lock);
+	node = __kcom_node_find(saddr);
+	// spin_unlock(&kcom_nodes_lock);
+	return node;
+}
+EXPORT_SYMBOL_GPL(kcom_node_find);
+
+/**
+ * kcom_node_add
+ *
+ * Description:
+ * 	adds a new node connection.
+ *    signals kcomd that it needs to watch this node's socket.
+ *    the file descriptor belonging to this socket is mapped in
+ *    kcomd, but the function sock_map_fd. Due to a kernel security
+ *    check file descriptors are not shared between kernel threads.
+ **/
+struct kcom_node *kcom_node_add(struct socket *sock)
+{
+	struct kcom_node *node;
+
+	node=kmem_cache_alloc(kcom_node_cachep, SLAB_KERNEL);
+	if (!node) {
+		printk("Unable to allocate node space.\n");
+		return NULL; //-ENOMEM;
+	}
+	INIT_LIST_HEAD(&node->list);
+	INIT_LIST_HEAD(&node->tasks);
+	node->sock=sock;
+	node->fd = 0; // kcomd will see this and assign a fd properly.
+
+	/*
+	if (!sock->ops || !sock->ops->getname)
+		goto err;
+
+	ret = sock->ops->getname
+	check if it's already in node list.
+	*/
+
+	spin_lock(&kcom_nodes_lock);
+	list_add_tail(&node->list, &kcom_nodes);
+	spin_unlock(&kcom_nodes_lock);
+
+	if (kcomd_task)
+		send_sig(SIGHUP,kcomd_task,0);
+	else {
+		printk("Unable to find kcomd daemon.\n");
+		return NULL;
+	}
+	printk("leaving kcom_node_add\n");
+	return node;
+
+}
+EXPORT_SYMBOL_GPL(kcom_node_add);
+
+/**
+ * kcom_node_del
+ *
+ * Description:
+ * 	removes this node from the list of connected nodes.
+ *    releases the corresponding socket and file descriptor.
+ **/
+int kcom_node_del(struct sockaddr *addr)
+{
+	struct kcom_node *node;
+
+	/* remove the node from the list */
+	spin_lock(&kcom_nodes_lock);
+	node = __kcom_node_find(addr);
+	if (!node) {
+		spin_unlock(&kcom_nodes_lock);
+		return -ENOENT;
+	}
+	list_del(&node->list);
+	spin_unlock(&kcom_nodes_lock);
+
+	/* release and free structure */
+	sys_close(node->fd);
+	sock_release(node->sock);
+	kfree(node);
+	return 0;
+}
+
+/**
+ * set_sockopts
+ *
+ * Description:
+ * 	sets the socket options.  TCP_NODELAY (send tcp packet immediately),
+ *    keepalive intervals, retries, etc.
+ **/
+int set_sockopts(struct socket *sock)
+{
+	int val;
+	int ret;
+	char __user *pval;
+	mm_segment_t oldfs;
+
+	oldfs = get_fs();
+	set_fs(KERNEL_DS);
+
+	pval = (char __user *) &val;
+
+	val = 1;
+	ret = sock_setsockopt(sock, SOL_SOCKET, SO_KEEPALIVE,
+		pval, sizeof(val));
+	if (ret) {
+		printk("unable to setsock SO_KEEPALIVE ERROR %d\n", ret);
+		return -1;
+	}
+
+	/* FIXME: check on these, old COMM_MIGD */
+	val = OPENMOSIX_CONNECTION_KEEPALIVE_INTERVAL;
+	ret = sock->ops->setsockopt(sock, IPPROTO_TCP, TCP_KEEPINTVL,
+		pval, sizeof(val));
+	if (ret) {
+		printk("Unable to setsock TCP_KEEPINTVL ERROR %d\n", ret);
+		return -1;
+	}
+
+	val = OPENMOSIX_CONNECTION_KEEPALIVE_MAXTRIES;
+	ret = sock->ops->setsockopt(sock, IPPROTO_TCP, TCP_KEEPCNT,
+		pval, sizeof(val));
+	if (ret) {
+		printk("unable to setsock TCP_KEEPCNT ERROR %d\n", ret);
+		return -1;
+	}
+
+	val = OPENMOSIX_CONNECTION_KEEPALIVE_TOTAL;
+	ret = sock->ops->setsockopt(sock, IPPROTO_TCP, TCP_KEEPIDLE,
+		pval, sizeof(val));
+	if (ret) {
+		printk("unable to setsock TCP_KEEPIDLE ERROR %d\n", ret);
+		return -1;
+	}
+
+	val=1;
+	ret = sock->ops->setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, pval, sizeof(val));
+	if (ret < 0) {
+		printk("Unable to setsockopt ERROR: %d\n", ret);
+		return -1;
+	}
+	set_fs(oldfs);
+	return 0;
+
+}
+
+/**
+ * create_connection
+ *
+ * Description:
+ * 	creates the network connection to other nodes and creates
+ *    a node that holds this connection. Stores the corresponding
+ *    IP address in the node as well.
+ **/
+struct kcom_node *create_connection(struct sockaddr *saddr)
+{
+
+	struct socket *sock;
+	// int fd;
+	int ret;
+	int error;
+	DECLARE_WAITQUEUE(wait, current);
+	unsigned long timo=MAX_SCHEDULE_TIMEOUT;
+
+	struct kcom_node *node;
+
+	ret = sock_create(saddr->sa_family, SOCK_STREAM, IPPROTO_TCP, &sock);
+	if (ret < 0) {
+		printk("Unable to sock_create. ret=%d\n", ret);
+		return NULL;
+	}
+
+	error = sock->ops->connect(sock, saddr, sizeof(struct sockaddr_in), O_NONBLOCK);
+	add_wait_queue(sock->sk->sk_sleep, &wait);
+	while (sock->state != SS_CONNECTED) {
+		set_current_state(TASK_INTERRUPTIBLE);
+		error = sock->ops->connect(sock, saddr, sizeof(struct sockaddr_in), O_NONBLOCK);
+		if (error != -EALREADY || (error = sock_error(sock->sk)))
+			break;
+		timo = schedule_timeout(timo);
+		if (timo <= 0) {
+			error = -EAGAIN;
+			break;
+		}
+	}
+	remove_wait_queue(sock->sk->sk_sleep, &wait);
+	set_current_state(TASK_RUNNING);
+
+	if (error < 0) {
+		printk("Unable to create connection. Err %d\n", error);
+		return NULL;
+	}
+
+	node = kcom_node_find(saddr);
+	if (!node) {
+		node = kcom_node_add(sock);
+		if (!node) {
+			printk("Unable to add node\n");
+			return NULL;
+		}
+		memcpy(&node->addr, saddr, sizeof(*saddr));
+		ret=set_sockopts(node->sock);
+		if (ret) {
+			printk("Unable to set socket options.\n");
+			return NULL;
+		}
+	} else
+		printk("Connection already exists. Not creating new node.\n");
+
+	return node;
+
+}
+EXPORT_SYMBOL_GPL(create_connection);
+
+/**
+ * kcom_task_create
+ *
+ * Description:
+ * 	creates the kcom task related to this process.
+ *    initializes the linked lists for incoming and outgoing
+ *    pkts.  Also links this task to the corresponding node
+ *    since only migrated(ing) processes need kcom tasks.
+ **/
+struct kcom_task *kcom_task_create(struct kcom_node *node, int pid)
+{
+	struct kcom_task *kctask;
+	task_t *p;
+
+	kctask=kmem_cache_alloc(kcom_task_cachep, SLAB_KERNEL);
+
+	if (kctask) {
+		if (pid==0) { // this only occurs when remote node is creating new proc.
+				kctask->rpid = 0;
+				kctask->hpid = pid;
+		} else {
+			// read_lock(&tasklist_lock);
+			p = find_task_by_pid(pid);
+			// read_unlock(&tasklist_lock);
+			if (p) {
+				if (task_test_dflags(p, DREMOTE)) {
+					kctask->rpid = pid;
+					kctask->hpid = 0;
+				} else {
+					kctask->hpid = pid;
+					kctask->rpid = 0;
+				}
+			} else {
+				printk("ERROR creating kcom task.\n");
+				return NULL;
+			}
+		}
+
+		kctask->node = node;
+		INIT_LIST_HEAD(&kctask->list);
+		INIT_LIST_HEAD(&kctask->out_packs);
+		INIT_LIST_HEAD(&kctask->in_packs);
+		spin_lock_init(&kctask->spinlock);
+
+		list_add_tail(&kctask->list, &node->tasks);
+	} else
+		return NULL;
+	return kctask;
+}
+EXPORT_SYMBOL_GPL(kcom_task_create);
+
+/**
+ * kcom_task_delete
+ *
+ * Description:
+ * 	deletes the kcom task and frees the memory.
+ *    Lookins that dflags to see if this is a process is a
+ *    remote process corresponding to a migrated process or
+ *    if its a 'home' process.
+ *    This is needed since both process ID's are kept in the
+ *    kcom task.
+ **/
+int kcom_task_delete(pid_t pid)
+{
+	struct kcom_node *tmp;
+	struct kcom_task *tmp2;
+	struct kcom_task *tsk;
+
+	// Can't delete task until all pkts are sent.
+	tsk=kcom_task_find(pid);
+	// FIXME:  SMP safety
+	while (!list_empty(&tsk->out_packs))
+			schedule_timeout(HZ/1000);
+
+
+	list_for_each_entry(tmp, &kcom_nodes, list)
+		list_for_each_entry(tmp2, &tmp->tasks, list)
+			if (task_test_dflags(current, DREMOTE)) {
+				if (tmp2->rpid == pid) {
+					printk("Deleting remote kcom task %u.\n", pid);
+					list_del(&tmp2->list);
+					kfree(tmp2);
+					break;
+				}
+			 } else {
+				if (tmp2->hpid == pid) {
+					printk("Deleting home kcom task %u.\n", pid);
+					list_del(&tmp2->list);
+					kfree(tmp2);
+					break;
+				}
+			}
+	return 0;
+}
+
+/**
+ * __kcom_task_find
+ *
+ * Description:
+ * 	Searches all nodes for a specific task, by pid.
+ *    Lookins that dflags to see if this is a process is a
+ *    remote process corresponding to a migrated process or
+ *    if its a 'home' process.
+ *    This is needed since both process ID's are kept in the
+ *    kcom task.
+ **/
+struct kcom_task *__kcom_task_find(pid_t pid, int where)
+{
+	struct kcom_node *tmp;
+	struct kcom_task *tmp2;
+	task_t *p;
+
+	// read_lock(&tasklist_lock);
+	p = find_task_by_pid(pid);
+	// read_unlock(&tasklist_lock);
+	if (!p) {
+		printk("Unable to find pid %u\n", pid);
+		return NULL;
+	}
+
+	if (where == 0) {
+		// printk("FUNCTION: kcom_task_find\n");
+		list_for_each_entry(tmp, &kcom_nodes, list)
+			list_for_each_entry(tmp2, &tmp->tasks, list)
+				// One remote node, we're interested in rpid
+				// One home node, we're interested in hpid
+				if (task_test_dflags(p, DREMOTE)) {
+					// printk("kcom_task_find: dremote\n");
+					if (tmp2->rpid == pid)
+						return tmp2;
+				} else {
+					// printk("kcom_task_find: ddeputy\n");
+					if (tmp2->hpid == pid)
+						return tmp2;
+				}
+	} else if (where == 1) { // home
+		list_for_each_entry(tmp, &kcom_nodes, list)
+			list_for_each_entry(tmp2, &tmp->tasks, list)
+				if (tmp2->hpid == pid)
+					return tmp2;
+	} else if (where == 2) { // remote
+		list_for_each_entry(tmp, &kcom_nodes, list)
+			list_for_each_entry(tmp2, &tmp->tasks, list)
+				if (tmp2->rpid == pid)
+					return tmp2;
+
+	}
+	// printk("leaving FUNCTION: kcom_task_find\n");
+	return NULL;
+}
+
+/**
+ * kcom_task_find
+ *
+ * Description:
+ * 	calls __kcom_task_find, specifically looking
+ *    for a home pid.
+ **/
+struct kcom_task *kcom_home_task_find(pid_t pid)
+{
+	struct kcom_task *tmp;
+
+	tmp = __kcom_task_find(pid, 1);
+	return tmp;
+
+}
+EXPORT_SYMBOL_GPL(kcom_home_task_find);
+
+/**
+ * kcom_remote_task_find
+ *
+ * Description:
+ * 	calls __kcom_task_find, specifically looking
+ *    for a remote pid.
+ **/
+struct kcom_task *kcom_remote_task_find(pid_t pid)
+{
+	struct kcom_task *tmp;
+
+	tmp = __kcom_task_find(pid, 2);
+	return tmp;
+
+}
+EXPORT_SYMBOL_GPL(kcom_remote_task_find);
+
+
+/**
+ * kcom_task_find
+ *
+ * Description:
+ * 	calls __kcom_task_find,  which looks
+ *    at dflags to determine for itself if this is
+ *    a home or remote pid.
+ **/
+struct kcom_task *kcom_task_find(pid_t pid)
+{
+	struct kcom_task *tmp;
+
+	tmp = __kcom_task_find(pid, 0);
+	return tmp;
+}
+EXPORT_SYMBOL_GPL(kcom_task_find);
+
+/**
+ * kcom_task_send
+ *
+ * Description:
+ * 	Creates a packet to send and adds it to the task's outbound list.
+ *    No acknowledgement or reply is expected.
+ **/
+int kcom_task_send(struct kcom_task *tsk, int type, int datasize, char *data, char *resp, unsigned long addr)
+{
+	// struct kcom_task *tsk;
+	struct kcom_pkt *pkt;
+
+	// printk("kcom_task_send:\n");
+	// tsk = kcom_task_find(pid);
+	if (!tsk)
+		return -ENODEV;
+
+	/* put pkt in kcom_task */
+	pkt = kcom_pkt_create(datasize, type, PKT_NEW_MSG, data);
+	if (!pkt)
+		return -1;
+
+	if (!resp)
+		pkt->resp=data;
+	else
+		pkt->resp=resp;
+
+	pkt->hpid=tsk->hpid;
+	pkt->rpid=tsk->rpid;
+	pkt->addr=addr; // used by vma_pages
+
+	// spin_lock(&tsk->spinlock);
+	list_add_tail(&pkt->list, &tsk->out_packs);
+	// spin_unlock(&tsk->spinlock);
+
+	return pkt->msgid;
+}
+
+/**
+ * kcom_send
+ *
+ * Description:
+ *    Tell this function what type of pkt, the data and where to send it and
+ *    it finds the corresponding node, creating a new connection if necessary
+ *    Finds the corresponding tsk, creating it if necessary.
+ * 	creates a packet to send and adds it to the task's outbound list.
+ *    No acknowledgement or reply is expected.
+ **/
+int kcom_send(int type, int datasize, char *data, unsigned long addr, struct sockaddr_in *saddr)
+{
+
+	struct kcom_node *node;
+	struct kcom_task *tsk;
+	task_t *p = current;
+	unsigned int msgid;
+
+	node=kcom_node_find((struct sockaddr *)saddr);
+	if (node==NULL) {
+		printk("node not found, creating new connection.\n");
+		node=create_connection((struct sockaddr *)saddr);
+		if (node==NULL)
+			return -1;
+	}
+
+	tsk=kcom_task_find(p->pid);
+	if (tsk==NULL) {
+		printk("Task not found. creating new.\n");
+		tsk=kcom_task_create(node, p->pid);
+		if (tsk==NULL)
+			return -1;
+	}
+
+	msgid=kcom_task_send(tsk, type, datasize, data, NULL, addr);
+
+	if (kcomd_task != NULL)
+		send_sig(SIGHUP,kcomd_task,0);
+
+	return 0;
+}
+/**
+ * kcom_send_nack
+ *
+ * Description:
+ *    Send an ack to the other node.  This is the matching function for kcom_send_with_ack.
+ *    An ack is the acknowledgement that the kcom pkt was received correctly.
+ **/
+int kcom_send_nack(task_t *p, struct kcom_pkt *recv_pkt)
+{
+	// task_t *kcomd_task;
+	int mig_flag;
+	int syscall_flag;
+	int node_flag;
+	struct kcom_pkt *send_pkt;
+	struct kcom_task *send_tsk;
+
+	mig_flag=recv_pkt->type & MIG_MASK;
+	syscall_flag=recv_pkt->type & SYSCALL_MASK;
+
+	send_tsk=kcom_task_find(p->pid);
+
+	if (task_test_dflags(p, DREMOTE))
+		node_flag = REM_FLG;
+	else
+		node_flag = DEP_FLG;
+	send_pkt=kcom_pkt_create(0, mig_flag | syscall_flag | PKT_NACK | node_flag, PKT_NACK, NULL);
+
+	send_pkt->msgid=recv_pkt->msgid; // responses have same msg id as pkt they are responding to.
+	send_pkt->hpid=recv_pkt->hpid;
+	send_pkt->rpid=recv_pkt->rpid;
+	send_pkt->resp=recv_pkt->resp;
+
+	// spin_lock(&send_tsk->spinlock); // done outside this function
+	list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+	// spin_unlock(&send_tsk->spinlock);
+
+	if (kcomd_task)
+		send_sig(SIGHUP,kcomd_task,0);
+	else {
+		printk("Unable to signal kcomd\n");
+		return -1;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(kcom_send_nack);
+
+
+/**
+ * kcom_send_ack
+ *
+ * Description:
+ *    Send an ack to the other node.  This is the matching function for kcom_send_with_ack.
+ *    An ack is the acknowledgement that the kcom pkt was received correctly.
+ **/
+int kcom_send_ack(task_t *p, struct kcom_pkt *recv_pkt)
+{
+	// task_t *kcomd_task;
+	int mig_flag;
+	int syscall_flag;
+	int node_flag;
+	struct kcom_pkt *send_pkt;
+	struct kcom_task *send_tsk;
+
+	mig_flag=recv_pkt->type & MIG_MASK;
+	syscall_flag=recv_pkt->type & SYSCALL_MASK;
+
+	send_tsk=kcom_task_find(p->pid);
+
+	if (task_test_dflags(p, DREMOTE))
+		node_flag = REM_FLG;
+	else
+		node_flag = DEP_FLG;
+	send_pkt=kcom_pkt_create(0, mig_flag | syscall_flag | PKT_ACK | node_flag, PKT_ACK, NULL);
+
+	send_pkt->msgid=recv_pkt->msgid; // responses have same msg id as pkt they are responding to.
+	send_pkt->hpid=recv_pkt->hpid;
+	send_pkt->rpid=recv_pkt->rpid;
+	send_pkt->resp=recv_pkt->resp;
+
+	// spin_lock(&send_tsk->spinlock); // done outside this function
+	list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+	// spin_unlock(&send_tsk->spinlock);
+
+	if (kcomd_task)
+		send_sig(SIGHUP,kcomd_task,0);
+	else {
+		printk("Unable to signal kcomd\n");
+		return -1;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(kcom_send_ack);
+
+/**
+ * kcom_send_resp
+ *
+ * Description:
+ *    Send an response to the other node.  This is the matching function for kcom_send_with_response.
+ *    An response is both the acknowledgement that the kcom pkt was received correctly (or not) and
+ *    the expected response data.
+ **/
+int kcom_send_resp(task_t *p, int len, char *buf, struct kcom_pkt *recv_pkt)
+{
+	int mig_flag;
+	int syscall_flag;
+	int node_flag;
+	struct kcom_pkt *send_pkt;
+	struct kcom_task *send_tsk;
+
+	mig_flag=recv_pkt->type & MIG_MASK;
+	syscall_flag=recv_pkt->type & SYSCALL_MASK;
+
+	send_tsk=kcom_task_find(p->pid);
+
+	if (task_test_dflags(p, DREMOTE))
+		node_flag = REM_FLG;
+	else
+		node_flag = DEP_FLG;
+
+	send_pkt=kcom_pkt_create(len, mig_flag | syscall_flag | PKT_ACK | node_flag, PKT_ACK, buf);
+
+	send_pkt->msgid=recv_pkt->msgid;
+	send_pkt->hpid=send_tsk->hpid;
+	send_pkt->rpid=send_tsk->rpid;
+	send_pkt->resp=recv_pkt->resp;
+
+	list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+	if (kcomd_task) {
+		printk("Signaling kcomd\n");
+		send_sig(SIGHUP, kcomd_task, 0);
+	} else {
+		printk("Unable to signal kcomd\n");
+		return -1;
+	}
+	return 0;
+
+}
+EXPORT_SYMBOL(kcom_send_resp);
+
+/**
+ * wait_for_ack
+ *
+ * Description:
+ *    This actually checks the task's incoming pkt list for the matching msgid.
+ *    If the matching pkt is found, it is removed from the list and the function
+ *    returns success, else returns failure.
+ **/
+int wait_for_ack(struct kcom_task *task, unsigned int msgid)
+{
+	struct kcom_pkt *pkt, *pkt_next;
+
+	// spin_lock(&task->spinlock);
+	if (!list_empty(&task->in_packs)) {
+		list_for_each_entry_safe(pkt, pkt_next, &task->in_packs, list) {
+			if ((msgid==pkt->msgid) && ((pkt->type & MSG_MASK) == PKT_ACK)) {
+				list_del(&pkt->list);
+				kmem_cache_free(kcom_pkt_cachep, pkt);
+
+				// FIXME:  this doesn't belong here.
+				// init packet has rpid
+				if ((pkt->type & MIG_MASK) == MIG_INIT)
+					task->rpid=pkt->rpid;
+
+				// spin_unlock(&task->spinlock);
+
+				return 0;
+			}  // FIXME:  what about nacks?
+
+		}
+	}
+
+	// spin_unlock(&task->spinlock);
+	return -1;
+
+}
+
+/**
+ * wait_for_response
+ *
+ * Description:
+ *    This actually checks the task's incoming pkt list for the matching msgid.
+ *    If the matching pkt is found, it is removed from the list and the function
+ *    returns success, else returns failure.
+ *    FIXME: very similar to wait_for_ack,  merge??
+ **/
+int wait_for_response(struct kcom_task *task, unsigned int msgid)
+{
+	struct kcom_pkt *pkt, *pkt_next;
+	int i=-1;
+
+	printk("wait_for_response...for msgid %u\n", msgid);
+		list_for_each_entry_safe(pkt, pkt_next, &task->in_packs, list)
+			// FIXME:  check for resp or ack, too.
+			if (msgid==pkt->msgid) {
+				i=pkt->len;
+				list_del(&pkt->list);
+				kmem_cache_free(kcom_pkt_cachep, pkt);
+				printk("wait_for_response: i=%d, addr=%lu pkt->resp=%s\n", i, pkt->addr, pkt->resp);
+			}
+	return i;
+
+}
+
+/**
+ * kcom_send_with_ack
+ *
+ * Description:
+ *    Same as kcom_send except this function waits until an (n)ack is received.
+ **/
+int kcom_send_with_ack(int type, int datasize, char *data, unsigned long addr, struct sockaddr_in *saddr)
+{
+	struct kcom_node *node;
+	struct kcom_task *tsk;
+	task_t *p = current;
+	unsigned int msgid;
+
+	node=kcom_node_find((struct sockaddr *)saddr);
+	if (node==NULL) {
+		printk("Node not found, creating new connection.\n");
+		node=create_connection((struct sockaddr *)saddr);
+		if (!node) {
+			printk("ERROR: Unable to create new connection.\n");
+			return -1;
+		}
+	}
+
+	tsk=kcom_task_find(p->pid);
+	if (tsk==NULL) {
+		printk("Task not found. creating new.\n");
+		tsk=kcom_task_create(node, p->pid);
+		if (!tsk) {
+			printk("ERROR: Unable to create task.\n");
+			return -1;
+		}
+	}
+
+	// FIXME:  what if tsk is deleted between kcom_task_send and wait_for_ack ?
+	msgid=kcom_task_send(tsk, type, datasize, data, NULL, addr);
+
+	if (kcomd_task)
+		send_sig(SIGHUP,kcomd_task,0);
+	else {
+		printk("Unable to find kcomd daemon.\n");
+		return -1;
+	}
+
+	set_current_state(TASK_INTERRUPTIBLE);
+	while (wait_for_ack(tsk, msgid)!=0) {
+		schedule();
+		set_current_state(TASK_INTERRUPTIBLE);
+	}
+	set_current_state(TASK_RUNNING);
+	printk("Done waiting. msgid %d received...\n", msgid);
+
+
+	return 0;
+}
+
+/**
+ * kcom_send_with_response
+ *
+ * Description:
+ *    Same as kcom_send except this function waits until an response is received.
+ **/
+int kcom_send_with_response(int type, int datasize, char *data, unsigned long addr, char *resp, struct sockaddr_in *saddr)
+{
+	struct kcom_node *node;
+	struct kcom_task *tsk;
+	task_t *p = current;
+	unsigned int msgid;
+	int i=-1;
+
+	node=kcom_node_find((struct sockaddr *)saddr);
+	if (node==NULL) {
+		printk("node not found, creating new connection.\n");
+		node=create_connection((struct sockaddr *)saddr);
+		if (node==NULL) {
+			return -1;
+		}
+	}
+
+	tsk=kcom_task_find(p->pid);
+	if (!tsk) {
+		printk("Task not found. Creating new.\n");
+		tsk=kcom_task_create(node, p->pid);
+		if (!tsk) {
+			printk("Unable to create task\n");
+			return -1;
+		}
+	}
+
+	msgid=kcom_task_send(tsk, type, datasize, data, resp, addr);
+
+	if (kcomd_task)
+		send_sig(SIGHUP,kcomd_task,0);
+	else {
+		printk("Unable to find kcomd daemon.\n");
+		return -1;
+	}
+
+	set_current_state(TASK_INTERRUPTIBLE);
+retry:
+	i=wait_for_response(tsk, msgid);
+	if (i < 0) {
+		set_current_state(TASK_INTERRUPTIBLE); // Not the correct packet. go back to sleep
+		schedule();
+		goto retry;
+	}
+	printk("Done waiting. msgid %d received...\n", msgid);
+
+	return i;
+}
Index: linux/include/hpc/kcom.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/include/hpc/kcom.h	2006-09-20 17:50:41.000000000 +0200
@@ -0,0 +1,140 @@
+/*
+ *	Copyright (C) 2006 Matt Dew <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#ifndef _HPC_KCOM_H
+#define _HPC_KCOM_H
+
+#include <linux/sched.h>
+#include <net/sock.h>
+#include <hpc/comm.h>
+
+#include <linux/in.h>
+
+#include <hpc/protocol.h>
+
+#define DAEMON_IP4_PORT	0xB55 // 2901
+#define DAEMON_IP6_PORT	0xB56 // 2902
+
+/* PROTOTYPES */
+#ifdef _HPC_KCOMC_H
+	DEFINE_SPINLOCK(kcom_nodes_lock);
+	EXPORT_SYMBOL(kcom_nodes_lock);
+
+	struct list_head kcom_nodes = LIST_HEAD_INIT(kcom_nodes);
+	EXPORT_SYMBOL(kcom_nodes);
+
+	fd_set_bits sockets_fds;
+	EXPORT_SYMBOL(sockets_fds);
+
+	char *sockets_fds_bitmap = NULL;
+	EXPORT_SYMBOL(sockets_fds_bitmap);
+
+	int maxfds = -1;
+	EXPORT_SYMBOL(maxfds);
+
+	struct socket *lsock4=NULL, *lsock6=NULL;
+	EXPORT_SYMBOL(lsock4);
+	EXPORT_SYMBOL(lsock6);
+
+	int fd4, fd6;
+	EXPORT_SYMBOL(fd4);
+	EXPORT_SYMBOL(fd6);
+
+	// pid_t kcom_pid;
+	// EXPORT_SYMBOL(kcom_pid);
+
+	task_t *kcomd_task=NULL;
+	EXPORT_SYMBOL(kcomd_task);
+
+	unsigned int kcom_msgid=0;
+	EXPORT_SYMBOL(kcom_msgid);
+
+	kmem_cache_t *kcom_data_cachep;
+	EXPORT_SYMBOL(kcom_data_cachep);
+
+	kmem_cache_t *kcom_pkt_cachep;
+	EXPORT_SYMBOL(kcom_pkt_cachep);
+
+	kmem_cache_t *kcom_task_cachep;
+	EXPORT_SYMBOL(kcom_task_cachep);
+
+	kmem_cache_t *kcom_node_cachep;
+	EXPORT_SYMBOL(kcom_node_cachep);
+
+	kmem_cache_t *kcom_saddr_cachep;
+	EXPORT_SYMBOL(kcom_saddr_cachep);
+
+
+#else /* _HPC_KCOMC_H */
+	extern int maxfds;
+	extern spinlock_t kcom_nodes_lock;
+	extern struct list_head kcom_nodes;
+
+	extern fd_set_bits sockets_fds;
+	extern char *sockets_fds_bitmap;
+	extern struct socket *lsock4;
+	extern struct socket *lsock6;
+	extern int fd4;
+	extern int fd6;
+
+	//extern pid_t kcom_pid;
+	extern task_t *kcomd_task;
+	extern kmem_cache_t *kcom_data_cachep;
+	extern kmem_cache_t *kcom_pkt_cachep;
+	extern kmem_cache_t *kcom_task_cachep;
+	extern kmem_cache_t *kcom_node_cachep;
+	extern kmem_cache_t *kcom_saddr_cachep;
+#endif /* _HPC_KCOMC_H */
+
+
+struct kcom_pkt
+{
+	pid_t hpid;     /* home pid of the this process */
+	pid_t rpid;     /* remote pid of 'that' other node process */
+	int len;      /* len of data */
+	int type;     /* type of data */
+	// int ack;      /* new msg, ack or response */
+	unsigned long addr; /* used by mm pages */
+	unsigned int msgid;
+	struct list_head list;
+	char *data;   /* ptr of data */
+	char *resp;   /* ptr of response */
+};
+
+struct kcom_node
+{
+	int fd;                 /* fd to send packet */
+	struct socket *sock;    /* socket */
+	struct sockaddr addr;   /* addr of this node */
+	spinlock_t tasks_lock;  /* lock for the list */
+	struct list_head tasks; /* list of task */
+	struct list_head list; /* list of nodes */
+	spinlock_t spinlock;
+};
+
+struct kcom_task
+{
+        pid_t hpid;          /* pid of the home node process owning this struct */
+        pid_t rpid;          /* pid of remote node process */
+        struct kcom_node *node; /* node of the process to send/recv */
+        struct list_head list;  /* list of process using some node */
+
+        struct list_head out_packs;
+        // struct kcom_pkt in_packs;
+        struct list_head in_packs;
+			spinlock_t spinlock; // FIXME:   ?? two spinlocks?  in_packs, out_packs
+};
+
+extern int kcom_send_nack(task_t *p, struct kcom_pkt *recv_pkt);
+#endif /* _HPC_KCOM_H */

Index: linux/hpc/arch-i386.c
===================================================================
--- linux.orig/hpc/arch-i386.c	2006-09-20 18:04:32.000000000 +0200
+++ linux/hpc/arch-i386.c	2006-09-20 18:04:59.000000000 +0200
@@ -245,10 +245,12 @@
 	syscall_func_t fct;
 	extern void * sys_call_table[];
 
-	OMDEBUG_SYS(4, "exec_sys[%d](%lx, %lx, %lx, %lx, %lx, %lx)\n", n,
+	/* OMDEBUG_SYS(4, "exec_sys[%d](%lx, %lx, %lx, %lx, %lx, %lx)\n", n,*/
+	printk("exec_sys[%d](%lx, %lx, %lx, %lx, %lx, %lx)\n", n,
 			args->arg[0], args->arg[1], args->arg[2],
 			args->arg[3], args->arg[4], args->arg[5]);
 	fct = (syscall_func_t) sys_call_table[n];
+	printk("arch_exec_syscall: debug1\n");
 	return fct(*((syscall_parameter_t *) args));
 }
 
Index: linux/hpc/comm.c
===================================================================
--- linux.orig/hpc/comm.c	2006-09-20 18:04:32.000000000 +0200
+++ linux/hpc/comm.c	2006-09-20 18:04:59.000000000 +0200
@@ -42,6 +42,7 @@
  **/
 static void comm_shutdown(struct socket *sock)
 {
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	if (sock && sock->ops)
 		sock->ops->shutdown(sock, SEND_SHUTDOWN);
 }
@@ -56,6 +57,7 @@
 {
 	int val, ret;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	val = sizeof(struct sockaddr);
 	if (!sock->ops || !sock->ops->getname)
 		return -1;
@@ -72,6 +74,7 @@
  **/
 void comm_data_ready(struct sock *sk, int len)
 {
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	wake_up_interruptible(sk->sk_sleep);
 }
 
@@ -82,6 +85,7 @@
 	int error;
 	mm_segment_t oldfs;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	oldfs = get_fs();
 	set_fs(KERNEL_DS);
 
@@ -129,6 +133,7 @@
 	int error;
 	struct socket *sock;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	error = sock_create(family, type, proto, &sock);
 	if (error < 0)
 		return NULL;
@@ -140,6 +145,7 @@
 {
 	int error;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	error = sock->ops->bind(sock, saddr, sizeof(*saddr));
 	if (error == -EADDRINUSE)
 		printk("comm_bind() Already in use\n");
@@ -151,6 +157,7 @@
 {
 	int error;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	error = sock->ops->listen(sock, SOMAXCONN);
 
 	return error;
@@ -162,6 +169,7 @@
 	int error;
 	DECLARE_WAITQUEUE(wait, current);
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	if (!timo)
 		timo = MAX_SCHEDULE_TIMEOUT;
 
@@ -207,6 +215,7 @@
 {
 	BUG_ON(!sock);
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	sock_release(sock);
 }
 
@@ -217,6 +226,7 @@
 {
 	int mask;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	mask = sock->ops->poll(NULL, sock, NULL);
 	return (mask & POLLIN_SET) ? 1 : 0;
 }
@@ -238,6 +248,7 @@
 	static struct file sighfile = {.f_count = ATOMIC_INIT(1)};
 	DECLARE_WAITQUEUE(wait, current);
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	/*
 	 * sighfile: we are required to supply a file to "hold" while we poll.
 	 * a bit ridiculous in this context, but nobody will notice because
@@ -262,6 +273,7 @@
 	remove_wait_queue(sock->sk->sk_sleep, &wait);
 	set_current_state(TASK_RUNNING);
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return (pollmask & mask) ? 1 : 0;
 }
 
@@ -293,6 +305,7 @@
 
 	*mlp = NULL;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	sock = sock_alloc();
 	if (!sock)
 		return error;
@@ -315,6 +328,7 @@
 		goto failed;
 
 	*mlp = sock;
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return error;
 
 failed:
@@ -338,6 +352,7 @@
 	int n = 0;
 	int left = len;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	do {
 		n = sock_recvmsg(sock, msg, left, msg->msg_flags);
 		if (n <= 0) {
@@ -367,6 +382,7 @@
 			}
 		}
 	} while (left);
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return len;
 }
 
@@ -387,6 +403,7 @@
 	BUG_ON(len > PAGE_SIZE);
 	BUG_ON(!mlink);
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	iov.iov_base = data;
 	iov.iov_len = len;
 
@@ -403,6 +420,7 @@
 
 	set_fs(oldfs);
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return error;
 }
 
@@ -421,6 +439,7 @@
 	mm_segment_t oldfs;
 	int error;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	BUG_ON(!mlink);
 
 	iov.iov_base = data;
@@ -435,6 +454,7 @@
 
 	set_fs(oldfs);
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return error;
 }
 
@@ -465,6 +485,7 @@
 	struct socket *link;
 	int error;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	link = comm_socket(sa->sa_family, SOCK_STREAM, IPPROTO_TCP);
 	if (!link)
 		return NULL;
@@ -477,6 +498,7 @@
 	if (error < 0)
 		goto fail;
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return link;
 
 fail:
@@ -488,6 +510,7 @@
 {
 	struct socket *link;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	link = comm_socket(sa->sa_family, SOCK_STREAM, IPPROTO_TCP);
 	if (!link)
 		return NULL;
@@ -495,6 +518,7 @@
 	if (comm_connect(link, sa, timo))
 		goto fail;
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return link;
 
 fail:
@@ -507,6 +531,7 @@
 	struct omp_req req;
 	int error;
 
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	req.type = type;
 	req.dlen = dlen;
 
@@ -518,6 +543,7 @@
 	if (error < 0)
 		return -1;
 
+	printk("ERROR: COMM FUNCTION %s leaving\n", __FUNCTION__);
 	return 0;
 }
 
@@ -526,5 +552,6 @@
 {
 	struct omp_req req;
 	req.type = type;
+	printk("ERROR: COMM FUNCTION %s\n", __FUNCTION__);
 	return comm_send(link, &req, sizeof(req));
 }
Index: linux/hpc/copyuser.c
===================================================================
--- linux.orig/hpc/copyuser.c	2006-09-20 18:04:59.000000000 +0200
+++ linux/hpc/copyuser.c	2006-09-20 18:04:59.000000000 +0200
@@ -177,6 +177,7 @@
 long deputy_put_user(long value, const void *addr, size_t size)
 {
 	BUG_ON(size > sizeof(long));
+	printk(KERN_DEBUG "FUNCTION: deputy_put_user\n");
 	return deputy_put_userX((s64) value, addr, size);
 }
 EXPORT_SYMBOL(deputy_put_user);
@@ -193,6 +194,7 @@
  **/
 long deputy_put_user64(s64 value, const void *addr)
 {
+	printk(KERN_DEBUG "FUNCTION: deputy_put_user64\n");
 	return deputy_put_userX(value, addr, 8);
 }
 EXPORT_SYMBOL(deputy_put_user64);
@@ -226,6 +228,7 @@
 long deputy_get_user(long *value, const void *addr, size_t size)
 {
 	BUG_ON(size > sizeof(long));
+	printk(KERN_DEBUG "FUNCTION: deputy_get_user\n");
 	return deputy_get_userX((u64 *) value, addr, size);
 }
 EXPORT_SYMBOL(deputy_get_user);
@@ -236,6 +239,7 @@
  **/
 long deputy_get_user64(s64 *value, const void *addr)
 {
+	printk(KERN_DEBUG "FUNCTION: deputy_get_user64\n");
 	return deputy_get_userX(value, addr, 8);
 }
 EXPORT_SYMBOL(deputy_get_user64);
@@ -308,7 +312,7 @@
 /**
  * remote_strnlen_from_user - strnlen from user for deputy
  **/
-static int remote_strnlen_user(task_t *p, struct kcom_pkt *pkt)
+/*static*/ int remote_strnlen_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_req u;
 	long *ret_ptr;
@@ -360,6 +364,7 @@
 	memcpy(&u, pkt->data, sizeof(struct omp_usercopy_req));
 	ret=kzalloc(sizeof(*ret), GFP_KERNEL);
 
+	printk("KERN_DEBUG remote_get_user; addr=%lu, len=%lu\n", u.addr, u.len);
 
 	switch (u.len) {
 	case 1: get_user(*ret, (u8 *) u.addr); break;
@@ -390,6 +395,9 @@
 		set_current_state(TASK_INTERRUPTIBLE);
 		list_for_each_entry_safe(pkt, pkt_next, &task->in_packs, list) {
 
+			printk(KERN_DEBUG "openMosix %s Checking pkt list for pid %u, type:0x%x\n"
+			      , __FUNCTION__, p->pid, pkt->type);
+
 			switch ( pkt->type & SYSCALL_MASK ) {
 
 				case DEP_STRNCPY_FROM_USER :
@@ -422,6 +430,7 @@
 					list_add_tail(&send_pkt->list, &task->out_packs);
 
 					if (kcomd_task) {
+						printk(KERN_DEBUG "Signaling kcomd\n");
 						send_sig(SIGHUP, kcomd_task, 0);
 					} else
 						printk("Unable to signal kcomd\n");
@@ -431,6 +440,7 @@
 					kmem_cache_free(kcom_pkt_cachep, pkt);
 					set_current_state(TASK_RUNNING);
 					schedule();
+					printk(KERN_DEBUG "leaving FUNCTION: remote_handle_user\n");
 					return ret;
 					break;
 
@@ -441,12 +451,14 @@
 		}
 
 		if (list_empty(&task->in_packs)) {
+			printk(KERN_DEBUG "remote_handle_user, sleeping\n");
 			schedule();
+			printk(KERN_DEBUG "remote_handle_user, waking\n");
 		}
 
 	}
 
-	printk( KERN_ERR "openMosix: %s is not supposed to end that way ! (%s:%n)\n"
+	printk( KERN_ERR "openMosix: %s is not supposed to end that way ! (%s:%d)\n"
 	      , __FUNCTION__, __FILE__, __LINE__);
 
 	return -1;
Index: linux/hpc/deputy.c
===================================================================
--- linux.orig/hpc/deputy.c	2006-09-20 18:04:59.000000000 +0200
+++ linux/hpc/deputy.c	2006-09-20 18:04:59.000000000 +0200
@@ -69,7 +69,7 @@
 		send_sig(SIGHUP,kcomd_task,0);
 
 	if (sizeof(struct omp_syscall_req) != pkt->len)
-		printk("ERROR in deputy_do_syscall.  data size of %d does not match expected %d\n", pkt->len, sizeof(struct omp_syscall_req));
+		printk("ERROR in deputy_do_syscall.  data size of %d does not match expected %lu\n", pkt->len, sizeof(struct omp_syscall_req));
 	memcpy(&s, pkt->data, pkt->len);
 	
 	OMDEBUG_SYS(1, "[deputy] receive syscall %d\n", s.n);
@@ -79,6 +79,7 @@
 	
 	kcom_send_with_ack(MIG_SYSCALL | PKT_NEW_MSG | SYSCALL_DONE | DEP_FLG, 0, NULL, r.ret, dest_ptr);
 
+	printk(KERN_DEBUG "[deputy] replied to syscall %d with %ld\n", s.n, r.ret);
 
 	return 0;
 }
@@ -94,6 +95,8 @@
 	task_t *child;
 	struct socket *childsock;
 
+	printk(KERN_DEBUG "openmosix %s\n", __FUNCTION__);
+
 	error = comm_recv(p->om.contact, &m, sizeof(m));
 	if (error < 0)
 		return -1;
@@ -356,6 +359,7 @@
 	int sz;
 	char *data = NULL;
 
+	printk(KERN_DEBUG "openmosix %s\n", __FUNCTION__);
 	error = comm_recv(p->om.contact, &m, sizeof(m));
 	if (error < 0)
 		goto error;
@@ -508,8 +512,11 @@
 
 			list_for_each_entry_safe(pkt, pkt_next, &mytsk->in_packs, list) {
 
+				printk(KERN_DEBUG "packet found.\n");
+
 				if ((pkt->type & MIG_MASK) == MIG_SYSCALL) {
 
+					printk(KERN_DEBUG "SYSCALL pkt found.\n");
 
 					error = deputy_do_syscall(p, pkt);
 					list_del(&pkt->list);
@@ -521,6 +528,9 @@
 		deputy_process_misc(current);
 		schedule();
 	}
+
+	printk(KERN_DEBUG "leaving FUNCTION: deputy_main_loop\n");
+	printk(KERN_DEBUG "Process %u has fully migrated home.\n", p->pid);
 }
 
 void exit_mm(task_t *);
Index: linux/hpc/kcomd.c
===================================================================
--- linux.orig/hpc/kcomd.c	2006-09-20 18:04:49.000000000 +0200
+++ linux/hpc/kcomd.c	2006-09-20 18:06:15.000000000 +0200
@@ -243,6 +243,17 @@
  	}
 	set_fs(oldfs);
 	do_gettimeofday(&stop);
+	if (stop.tv_sec==start.tv_sec)
+		printk(KERN_DEBUG"openMosix data_send: time %lu usecs\n", stop.tv_usec-start.tv_usec);
+	else
+		printk(KERN_DEBUG"openMosix data_send: time %lu usecs\n", ((stop.tv_sec*USEC_PER_SEC)+stop.tv_usec)-((start.tv_sec*USEC_PER_SEC)+start.tv_usec));
+
+	if ((send_pkt->type & MSG_MASK) ==PKT_NEW_MSG)
+		printk(KERN_DEBUG"FUNCTION: data_send: sent NEW msgid:%u type:0x%x len:%d\n", send_pkt->msgid, send_pkt->type, send_pkt->len);
+	else if ((send_pkt->type & MSG_MASK) ==PKT_ACK)
+		printk(KERN_DEBUG"FUNCTION: data_send: sent ACK msgid:%u type:0x%x len:%d\n", send_pkt->msgid, send_pkt->type, send_pkt->len);
+	else if ((send_pkt->type & MSG_MASK) ==PKT_RESP)
+		printk(KERN_DEBUG"FUNCTION: data_send: sent RESP msgid:%u type:0x%x len:%d\n", send_pkt->msgid, send_pkt->type, send_pkt->len);
 	return i;
 
 }
@@ -291,6 +302,7 @@
 			/* spin_lock(&tsk->spinlock); */
 			list_add_tail(&recv_kcom_pkt->list, &tsk->in_packs);
 			/* spin_unlock(&tsk->spinlock); */
+ 			printk("pkt (msgid: %u) added to remote pid %u in_packs list.\n", recv_kcom_pkt->msgid, tsk->rpid);
 		} else {
 			printk(KERN_ERR "unable to find remote pid %u\n", recv_kcom_pkt->rpid);
 			return -1;
@@ -300,6 +312,7 @@
 		sltsk=find_task_by_pid(recv_kcom_pkt->rpid);
 		/* read_unlock(&tasklist_lock); */
 		if (sltsk) {
+ 			printk("Waking up process %u\n", sltsk->pid);
 			wake_up_process(sltsk);
 		} else {
 			printk(KERN_ERR "Unable to find remote pid %u to wake up\n", recv_kcom_pkt->rpid);
@@ -312,6 +325,7 @@
 			/* spin_lock(&tsk->spinlock); */
 			list_add_tail(&recv_kcom_pkt->list, &tsk->in_packs);
 			/* spin_unlock(&tsk->spinlock); */
+			printk(KERN_DEBUG "pkt (msgid: %u) adding to home pid %u in_packs list.\n", recv_kcom_pkt->msgid, tsk->hpid);
 		} else {
 			printk(KERN_ERR "unable to find home pid %u\n", recv_kcom_pkt->hpid);
 			return -1;
@@ -320,6 +334,7 @@
 		sltsk=find_task_by_pid(recv_kcom_pkt->hpid);
 		/* read_unlock(&tasklist_lock); */
 		if (sltsk) {
+			printk(KERN_DEBUG "Waking up process %u\n", sltsk->pid);
 			wake_up_process(sltsk);
 		} else {
 			printk(KERN_ERR "Unable to find home pid %u to wake up\n", recv_kcom_pkt->hpid);
@@ -366,16 +381,25 @@
 		}
 	}
 
+ 	if ((recv_kcom_pkt->type & MSG_MASK) ==PKT_NEW_MSG)
+ 		printk("FUNCTION: pkt_read: received NEW msgid:%u type:0x%x len:%d rpid:%u hpid:%u\n", recv_kcom_pkt->msgid, recv_kcom_pkt->type, recv_kcom_pkt->len, recv_kcom_pkt->rpid, recv_kcom_pkt->hpid);
+ 	else if ((recv_kcom_pkt->type & MSG_MASK) ==PKT_ACK)
+ 		printk("FUNCTION: pkt_read: received ACK msgid:%u type:0x%x len:%d rpid:%u hpid:%u\n", recv_kcom_pkt->msgid, recv_kcom_pkt->type, recv_kcom_pkt->len, recv_kcom_pkt->rpid, recv_kcom_pkt->hpid);
+ 	else if ((recv_kcom_pkt->type & MSG_MASK) ==PKT_RESP)
+ 		printk("FUNCTION: pkt_read: received RESP msgid:%u type:0x%x len:%d rpid:%u hpid:%u\n", recv_kcom_pkt->msgid, recv_kcom_pkt->type, recv_kcom_pkt->len, recv_kcom_pkt->rpid, recv_kcom_pkt->hpid);
 
 	if ((recv_kcom_pkt->type & MSG_MASK) == PKT_NEW_MSG) {
 		switch (recv_kcom_pkt->type & MIG_MASK) {
 			case MIG_INIT:
+ 				printk("case MIG_INIT\n");
 				mig_do_receive_init(node, recv_kcom_pkt);
 				break;
 			case MIG_GO_HOME:
+ 				printk("case MIG_GO_HOME\n");
 				mig_do_receive_home(node, recv_kcom_pkt);
 				break;
 			case MIG_COME_HOME:
+ 				printk("case MIG_COME_HOME\n");
 				sltsk=find_task_by_pid(recv_kcom_pkt->rpid);
 				task_register_migration(sltsk);
 				break;
Index: linux/hpc/syscalls.c
===================================================================
--- linux.orig/hpc/syscalls.c	2006-09-20 18:04:32.000000000 +0200
+++ linux/hpc/syscalls.c	2006-09-20 18:04:59.000000000 +0200
@@ -27,12 +27,15 @@
 #include <hpc/syscalls.h>
 #include <hpc/debug.h>
 #include <hpc/hpc.h>
+#include <hpc/kcom.h>
+#include <hpc/prototype.h>
 
 /* generic multiplexer */
 asmlinkage long om_sys_local(struct pt_regs regs)
 {
 	long ret;
 
+	printk("[remote] local syscall %d\n", SYSNB());
 	OMDEBUG_SYS(1, "[remote] local syscall %d\n", SYSNB());
 
 	ret = arch_exec_syscall(SYSNB(), (syscall_parameter_t *) &regs);
@@ -41,17 +44,20 @@
 
 asmlinkage long om_sys_remote(struct pt_regs regs)
 {
+	printk("om_sys_gettid syscall\n");
 	return remote_do_syscall(SYSNB(), &regs);
 }
 
 /* specific remote syscalls */
 asmlinkage int om_sys_gettid(struct pt_regs regs)
 {
+	printk("om_sys_getpid syscall\n");
 	return current->om.pid;
 }
 
 asmlinkage int om_sys_getpid(struct pt_regs regs)
 {
+	printk("om_sys_execve syscall\n");
 	return current->om.tgid;
 }
 
Index: linux/hpc/migctrl.c
===================================================================
--- linux.orig/hpc/migctrl.c	2006-09-20 18:04:58.000000000 +0200
+++ linux/hpc/migctrl.c	2006-09-20 18:04:59.000000000 +0200
@@ -129,6 +129,7 @@
 	set_current_state(TASK_RUNNING);
 	schedule();
 
+	printk("leaving FUNCTION: task_local_bring\n");
 	return 0;
 failed:
 	OMBUG("failed\n");

Index: linux/hpc/migctrl.c
===================================================================
--- linux.orig/hpc/migctrl.c	2006-09-18 23:40:22.000000000 +0200
+++ linux/hpc/migctrl.c	2006-09-18 23:47:35.000000000 +0200
@@ -26,6 +26,9 @@
 #include <hpc/hpc.h>
 #include <hpc/debug.h>
 #include <hpc/service.h>
+#include <linux/in.h>
+#include <hpc/omtask.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/mig.h>
 #include <hpc/protocol.h>
@@ -39,20 +42,12 @@
  **/
 int task_remote_expel(task_t *p)
 {
-	struct socket *link;
-
 	BUG_ON(!task_test_dflags(p, DREMOTE));
 
-	if (mig_send_hshake(p, p->om.contact, HSHAKE_MIG_REQUEST))
-		goto failed;
-
 	if (mig_do_send(p)) {
 		goto failed;
 	}
 
-	link = task_set_comm(p, NULL);
-	comm_close(link);
-
 	do_exit(SIGKILL);
 	return 0;
 failed:
@@ -69,19 +64,6 @@
  **/
 int task_remote_wait_expel(task_t *p)
 {
-	int error;
-	struct omp_req req;
-
-	comm_send_req(p->om.contact, REM_BRING_HOME);
-
-	error = comm_recv(p->om.contact, &req, sizeof(req));
-	if (error < 0)
-		return -1;
-	if (req.type != DEP_COMING_HOME) {
-		OMBUG("!DEP_COMING_HOME\n");
-		return -1;
-	}
-
 	return task_remote_expel(p);
 }
 
@@ -96,27 +78,12 @@
  **/
 static int task_local_send(task_t *p, struct sockaddr *whereto, int reason)
 {
-	struct socket *mlink;
 	int error = 0;
 
 	if (task_test_dflags(p, DDEPUTY))
 		return 0;
-	sockaddr_setup_port(whereto, REMOTE_DAEMON_PORT);
-	mlink = comm_setup_connect(whereto, 0);
-	if (!mlink) {
-		OMBUG("error\n");
-
-		error = -1;
-		goto failed;
-	}
-
-	task_set_comm(p, mlink);
 	task_set_dflags(p, DDEPUTY);
 
-	/* see if other part is with on this */
-	if (mig_send_hshake(p, mlink, HSHAKE_MIG_REQUEST))
-		goto failed;
-
 	if (mig_do_send(p)) {
 		error = -1;
 		goto failed;
@@ -128,8 +95,6 @@
 failed:
 	OMBUG("failed\n");
 	task_clear_dflags(p, DDEPUTY);
-	if (mlink)
-		comm_close(mlink);
 	return error;
 }
 
@@ -146,7 +111,6 @@
 static int task_local_bring(task_t *p, int reason)
 {
 	int error;
-	struct socket *link;
 
 	if (!task_test_dflags(p, DDEPUTY))
 		return 0;
@@ -155,21 +119,15 @@
 		goto failed;
 	}
 
-	/* send remote request */
-	comm_send_req(p->om.contact, DEP_COMING_HOME);
-
-	/* see if other part is with on this */
-	if (mig_recv_hshake(p->om.contact))
-		goto failed;
-
 	/* receive the process back */
 	error = mig_do_receive(p);
 	if (error)
 		goto failed;
 
 	task_clear_dflags(p, DDEPUTY);
-	link = task_set_comm(p, NULL);
-	comm_close(link);
+
+	set_current_state(TASK_RUNNING);
+	schedule();
 
 	return 0;
 failed:
@@ -206,21 +164,29 @@
                                struct sockaddr * whereto,
                                int reason)
 {
-	int error;
 
 	task_set_dflags(p, DPASSING);
 
-	error = (task_test_dflags(p, DREMOTE))
-		? (whereto)
-			? task_move_remote2remote(p, whereto, reason)
-			: task_remote_wait_expel(p)
-		: (whereto)
-			? task_local_send(p, whereto, reason)
-			: task_local_bring(p, reason);
+	/*
+	 * Ok, if DREMOTE flag set, then this is a remote process
+	 * if DDEPUTY is set, then this is a deputy
+	 * if neither is set, then this is a home process going out.
+	 * FIXME:  remote to remote?
+	 */
+
+	if (task_test_dflags(p, DREMOTE))
+		task_remote_wait_expel(p);
+	else if (task_test_dflags(p, DDEPUTY))
+		task_local_bring(p, reason);
+	else
+		task_local_send(p, whereto, reason);	/*
+							 * if neither flag is set then
+							 * home process going out
+							 */
 
 	task_clear_dflags(p, DPASSING);
 
-	return error;
+	return 0;
 }
 
 int task_move_to_node(struct task_struct *p, struct sockaddr * whereto,

Index: linux/hpc/migrecv.c
===================================================================
--- linux.orig/hpc/migrecv.c	2006-09-20 17:06:59.000000000 +0200
+++ linux/hpc/migrecv.c	2006-09-20 17:53:11.000000000 +0200
@@ -25,6 +25,7 @@
 #include <linux/stddef.h>
 #include <linux/highmem.h>
 #include <linux/personality.h>
+#include <linux/syscalls.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
 #include <hpc/comm.h>
@@ -34,10 +35,12 @@
 #include <hpc/mig.h>
 #include <hpc/debug.h>
 #include <hpc/protocol.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/version.h>
 #include <hpc/arch.h>
 
+#include <linux/inet.h>  /* in_aton*/
 /* handshake with the remote part */
 int mig_recv_hshake(struct socket *mlink)
 {
@@ -66,110 +69,316 @@
 	return 0;
 }
 
+/**
+ * mig_do_receive_home
+ *
+ * Description:
+ *    Called by kcomd when it receives a MIG_GO_HOME pkt.
+ *    Task_register_migration is called to inform the process that the
+ *    remote process is coming home.
+ **/
+int mig_do_receive_home(struct kcom_node *node, struct kcom_pkt *recv_kcom_pkt)
+{
+	struct kcom_task *recv_tsk;
+	struct kcom_pkt *send_pkt;
+	task_t *sltsk;
+
+	printk("FUNCTION: mig_do_receive_home\n");
+
+	if ((recv_kcom_pkt->type & MSG_MASK)==PKT_NEW_MSG) {
+		printk("Received MIG_GO_HOME NEW_MSG packet.\n");
+
+		recv_tsk=kcom_task_find(recv_kcom_pkt->hpid);
+		if (!recv_tsk) {
+			printk("Unable to find home pid %u\n", recv_kcom_pkt->hpid);
+			return -1;
+		}
+
+		send_pkt=kcom_pkt_create(0, MIG_GO_HOME | PKT_ACK | DEP_FLG, 0,  NULL);
+
+		send_pkt->msgid=recv_kcom_pkt->msgid; /* responses have same msg id as pkt they are responding to.*/
+		send_pkt->hpid=recv_kcom_pkt->hpid;
+		send_pkt->rpid=recv_kcom_pkt->rpid;;
+		send_pkt->resp=recv_kcom_pkt->resp;
+		/* spin_lock(&recv_tsk->spinlock);*/
+		list_add_tail(&send_pkt->list, &recv_tsk->out_packs);
+		/* spin_unlock(&recv_tsk->spinlock);*/
+
+		/* Ok, tell task migration is coming*/
+		/* read_lock(&tasklist_lock);*/
+		sltsk=find_task_by_pid(recv_kcom_pkt->hpid); /* only home node will receive MIG_GO_HOME NEW_MSG*/
+		/* read_unlock(&tasklist_lock);*/
+		if (sltsk) {
+			printk("Registering task migration\n");
+			task_register_migration(sltsk);
+		} else {
+			printk("Failed to register task migration\n");
+			return -1;
+		}
+
+
+	} else {
+		printk("Received MIG_GO_HOME PKT_ACK packet.\n");
+
+		recv_tsk=kcom_task_find(recv_kcom_pkt->rpid);
+		if (!recv_tsk) {
+			printk("Unable to find remote pid %u\n", recv_kcom_pkt->rpid);
+			return -1;
+		}
+
+		/* spin_lock(&recv_tsk->spinlock);*/
+		list_add_tail(&recv_kcom_pkt->list, &recv_tsk->in_packs);
+		/* spin_unlock(&recv_tsk->spinlock);*/
+		sltsk=find_task_by_pid(recv_kcom_pkt->rpid); /* only remote node will receive MIG_GO_HOME ack*/
+		if (sltsk) {
+			printk("Waking up process %u\n", sltsk->pid);
+			wake_up_process(sltsk);
+		} else {
+			printk("Unable to wake up process %u\n", recv_kcom_pkt->rpid);
+			return -1;
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mig_do_receive_home);
+
+/**
+ * mig_do_receive_init
+ *
+ * Description:
+ *    Called by kcomd when it receives a MIG_INIT pkt.
+ *    Creates a new process and sets up the associated task.
+ **/
+int mig_do_receive_init(struct kcom_node *node, struct kcom_pkt *recv_kcom_pkt)
+{
+	struct kcom_task *send_tsk;
+	struct kcom_pkt *send_pkt;
+
+	pid_t rpid;
+	struct sockaddr_in *saddr;
+	const unsigned int LO_IP=in_aton("127.0.0.1");
+	printk("Received MIG_INIT packet.\n");
+
+	if ((recv_kcom_pkt->type & MSG_MASK)==PKT_NEW_MSG) {	 /* incoming process*/
+		printk("Creating new process.\n");
+		rpid=0;
+
+		send_pkt=kcom_pkt_create(0, MIG_INIT | PKT_ACK | REM_FLG, 0,  NULL);
+
+		if (!send_pkt) {
+			printk("ERROR creating pkt in mig_do_receive_init\n");
+			return -1;
+		}
+		/* responses have same msg id as pkt they are responding to.*/
+		send_pkt->msgid=recv_kcom_pkt->msgid;
+
+		send_pkt->hpid=recv_kcom_pkt->hpid;
+		send_pkt->resp=recv_kcom_pkt->resp;
+
+		/* spin_lock();*/
+		saddr=(struct sockaddr_in *)&node->addr;
+		if (saddr->sin_addr.s_addr==LO_IP) { /* this allows loopback migration to work.*/
+			printk("Loopback migration.\n"); /* both home and remote processes use same task.  better idea?*/
+			send_tsk=kcom_home_task_find(recv_kcom_pkt->hpid);
+		} else {
+			send_tsk=kcom_task_create(node, 0);
+			/* spin_unlock();*/
+			if (!send_tsk) {
+				printk("ERROR: creating new kcom_task.\n");
+				return -1;
+			}
+			send_tsk->hpid=recv_kcom_pkt->hpid;
+		}
+		/* Delete init packet before starting new process.*/
+		/* spin_lock(&send_tsk->spinlock);*/
+		list_del(&recv_kcom_pkt->list);
+		/* spin_unlock(&send_tsk->spinlock);*/
+
+		/* send_pkt->rpid=0; // set this to 0 or if same process migrates second time here, it'll keep old rpid*/
+		user_thread(mig_handle_migration, &rpid, 0);
+		while (rpid==0) {
+			schedule_timeout_interruptible(HZ/1000);
+		}
+
+		if (rpid < 0) {
+			printk("Error creating new process.\n");
+			send_pkt->type=MIG_INIT | PKT_NACK | REM_FLG;
+
+			/* spin_lock(&send_tsk->spinlock);*/
+			list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+			/* spin_unlock(&send_tsk->spinlock);*/
+			return -1;
+		} else
+			printk("New process: %u\n", rpid);
+
+		send_pkt->rpid=rpid;
+
+		/* spin_lock(&send_tsk->spinlock);*/
+		list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+		/* spin_unlock(&send_tsk->spinlock);*/
+
+		send_tsk->rpid=rpid;
+
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mig_do_receive_init);
+
+
 /*****************************************************************************/
 
 /**
- * mig_do_receive_mig_mm - Receive some parameters for a mm
+ * mig_do_receive_mm
+ *
+ * Description:
+ *    Receives the process mmap info.
  **/
-static void mig_do_receive_mm(task_t *p, struct omp_mig_mm *s)
+KCOMD_NSTATIC void mig_do_receive_mm(task_t *p, struct kcom_pkt *pkt)
 {
+
 	OMDEBUG_MIG(2, "MIG_MM\n");
-	/* copy all mm's parameter from start_code to env_end */
-	memcpy(&p->mm->start_code, s, sizeof(*s));
+	printk("FUNCTION: mig_do_receive_mm\n");
+
+	down_write(&p->mm->mmap_sem);
+	memcpy(&p->mm->start_code, pkt->data, pkt->len);
+	p->mm->exec_vm=0; /* MSD debug*/
+	up_write(&p->mm->mmap_sem);
+
+	kcom_send_ack(p, pkt);
+
+	printk("leaving FUNCTION: mig_do_receive_mm\n");
+
 }
+EXPORT_SYMBOL_GPL(mig_do_receive_mm);
 
 /**
- * mig_do_receive_mm_area - Set up an mmap
+ * mig_do_receive_vma
+ *
+ * Description:
+ *    Receives the process vma info.
  **/
-static int mig_do_receive_vma(task_t *p, struct omp_mig_vma *a)
+KCOMD_NSTATIC int mig_do_receive_vma(task_t *p, struct kcom_pkt *pkt)
 {
+	struct omp_mig_vma *a;
 	unsigned long result, prot, flags;
 	struct file *file = NULL;
 	extern asmlinkage long sys_madvise(unsigned long, size_t, int);
 
+
+	printk("FUNCTION: mig_do_receive_vma\n");
+
+	a = (struct omp_mig_vma *)pkt->data;
+
 	OMDEBUG_MIG(2, "MIG_VMA [%lx, %ld]\n", a->vm_start, a->vm_size);
 
-	/* FIXME : Temporary disabled */
-	if (0 && a->vm_file) {
-		file = (task_test_dflags(p, DREMOTE))
-			? task_rfiles_get(p, a->vm_file, -1, a->i_size)
-			: a->vm_file;
-	}
+		/* FIXME : Temporary disabled */
+		if (0 && a->vm_file) {
+			file = (task_test_dflags(p, DREMOTE))
+				? task_rfiles_get(p, a->vm_file, -1, a->i_size)
+				: a->vm_file;
+		}
 
-	/* unconvert prot+flags: */
-	flags = MAP_FIXED | MAP_PRIVATE;
-	prot = 0;
-	if (a->vm_flags & VM_GROWSDOWN)
-		flags |= MAP_GROWSDOWN;
-	if (a->vm_flags & VM_DENYWRITE)
-		flags |= MAP_DENYWRITE;
-	if (a->vm_flags & VM_EXECUTABLE)
-		flags |= MAP_EXECUTABLE;
-
-	/* copy VM_(READ|WRITE|EXEC) bits to prot */
-	prot |= (a->vm_flags & (VM_READ | VM_WRITE | VM_EXEC));
-
-	/* mmap stuff */
-	result = do_mmap_pgoff(file, a->vm_start, a->vm_size, prot,
-						flags, a->vm_pgoff);
-	if (IS_ERR((const void *) result))
-		return PTR_ERR((const void *) result);
-
-	if (a->vm_flags & VM_READHINTMASK) {
-		int behavior = (a->vm_flags & VM_SEQ_READ)
-				? MADV_RANDOM
-				: MADV_SEQUENTIAL;
-		sys_madvise(a->vm_start, a->vm_size, behavior);
-	}
+		/* unconvert prot+flags: */
+		flags = MAP_FIXED | MAP_PRIVATE;
+		prot = 0;
+		if (a->vm_flags & VM_GROWSDOWN)
+			flags |= MAP_GROWSDOWN;
+		if (a->vm_flags & VM_DENYWRITE)
+			flags |= MAP_DENYWRITE;
+		if (a->vm_flags & VM_EXECUTABLE)
+			flags |= MAP_EXECUTABLE;
+
+	/* VM_GROWSDOWN =  0x0100*/
+	/* VM_DENYWRITE =  0x0800*/
+	/* VM_EXECUTABLE = 0x1000*/
+
+	/* MAP_GROWSDOWN =  0x0100*/
+	/* MAP_DENYWRITE =  0x0800*/
+	/* MAP_EXECUTABLE = 0x1000*/
+
+
+	/* flags=a->vm_flags;*/
+
+		/* copy VM_(READ|WRITE|EXEC) bits to prot */
+		/* prot |= (a->vm_flags & (VM_READ | VM_WRITE | VM_EXEC));*/
+		prot = (VM_READ | VM_WRITE | VM_EXEC);
+		/* VM_READ =  0x1*/
+		/* VM_WRITE = 0x2*/
+		/* VM_EXEC =  0x4*/
+
+		/* mmap stuff */
+		down_write(&p->mm->mmap_sem);
+		result = do_mmap_pgoff(file, a->vm_start, a->vm_size, prot,
+							flags, a->vm_pgoff);
+		up_write(&p->mm->mmap_sem);
+
+		if (IS_ERR((const void *) result))
+			return PTR_ERR((const void *) result);
+
+		if (a->vm_flags & VM_READHINTMASK) {
+			int behavior = (a->vm_flags & VM_SEQ_READ)
+					? MADV_RANDOM
+					: MADV_SEQUENTIAL;
+			result=sys_madvise(a->vm_start, a->vm_size, behavior);
+			if (result)
+				kcom_send_nack(p, pkt);
+				return result;
+		}
+
+
+	kcom_send_ack(p, pkt);
+
+	printk("leaving FUNCTION: mig_do_receive_vma\n");
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mig_do_receive_vma);
 
 
 /**
- * mig_do_receive_page - Receive one page
+ * mig_do_receive_page
+ *
+ * Description:
+ *    Receives one process memory page.
+ *    FIXME:   remote node segfaults on system calls because
+ *    of a bug in this function.   HELP!!!
  **/
-static int mig_do_receive_page(task_t *p, unsigned long addr)
+KCOMD_NSTATIC int mig_do_receive_page(task_t *p, struct kcom_pkt *pkt)
 {
 	struct mm_struct *mm = p->mm;
 	struct vm_area_struct *vma;
 	struct page *recv_page = NULL;
-	void *kmpage; /* kmapped page */
-	int error;
+	unsigned long addr;
+	void *kmpage;
 	pgd_t * pgd;
 	pud_t * pud;
 	pmd_t * pmd;
 	pte_t * pte;
 
-	OMDEBUG_MIG(3, "MIG_PAGE [%lx]\n", addr);
+	/* FIXME:   another way?*/
+	/* Must be done from process context.*/
+
+	/* recv_page = alloc_page(GFP_HIGHUSER);*/
+	/* kfree(pkt->data);*/
 
+
+	addr=pkt->addr;
 	vma = find_vma(mm, addr);
 	if (!vma) {
 		OMBUG("vma not found (addr: %p)\n", (void *) addr);
+		kcom_send_nack(p, pkt);
 		return -1;
 	}
-
-	/* check if enough memory */
-
-	/* alloc page */
-	recv_page = alloc_page(GFP_HIGHUSER);
-	if (!recv_page) {
-		OMBUG("unable to allocate page\n");
-		return -ENOMEM;
-	}
-
-	kmpage = kmap(recv_page);
-
-	/* receive the data into the page */
-	error = comm_recv(p->om.contact, kmpage, PAGE_SIZE);
-
+	recv_page=alloc_zeroed_user_highpage(vma, addr);
+	kmpage=kmap(recv_page);
+	memcpy(kmpage, pkt->data, pkt->len);
 	kunmap(recv_page);
-	if (error < 0) {
-		OMBUG("failed to receive data\n");
-		goto out;
-	}
+	/**/
 
 	/* add the page at correct place */
+
 	pgd = pgd_offset(mm, addr);
+
 	pud = pud_alloc(mm, pgd, addr);
 	if (!pud)
 		goto out;
@@ -187,30 +396,69 @@
 	page_dup_rmap(recv_page);
 	inc_mm_counter(mm, file_rss);
 
+
+/*
+	down_write(&mm->mmap_sem);
+	install_arg_page(vma, recv_page, pkt->addr);
+	up_write(&mm->mmap_sem);
+*/
+	/* make_pages_present(pkt->addr, pkt->addr+PAGE_SIZE); // NOPE*/
+
+	kcom_send_ack(p, pkt);
+
 	return 0;
+
 out:
+	printk("receive page failed at addr %p\n", (void *) addr);
 	OMBUG("receive page failed at addr %p\n", (void *) addr);
 	__free_page(recv_page);
 	return -1;
+
 }
 
 /**
- * mig_do_receive_fp - Receive floating points registers
+ * mig_do_receive_fp
  * @p:		task
- * @fpr:	floating point registers
+ * @pkt: ->data: floating point registers.
+ *
+ * Description:
+ *    Receive floating points registers
  **/
-static void mig_do_receive_fp(task_t *p, struct omp_mig_fp *fp)
+KCOMD_NSTATIC int mig_do_receive_fp(task_t *p, struct kcom_pkt *pkt)
 {
+	struct omp_mig_fp *fp;
+
+	printk("FUNCTION: mig_do_receive_fp\n");
+	fp=(void *)pkt->data;
+
 	OMDEBUG_MIG(2, "MIG_FP\n");
 	set_used_math();
 	arch_mig_receive_fp(p, fp);
+
+	kcom_send_ack(p, pkt);
+
+	printk("leaving FUNCTION: mig_do_receive_fp\n");
+
+	return 0;
 }
 
 /**
- * mig_do_receive_misc - Receive normal registers, limits
+ * mig_do_receive_misc
+ **/
+/**
+ * mig_do_receive_proc_context
+ * @p:		task
+ * @pkt:		->data: normal registers, limits.
+ *
+ * Description:
+ *    Receive normal registers, limits
  **/
-static void mig_do_receive_proc_context(task_t *p, struct omp_mig_task *m)
+KCOMD_NSTATIC int mig_do_receive_proc_context(task_t *p, struct kcom_pkt *pkt)
 {
+	struct omp_mig_task *m;
+
+	m=(struct omp_mig_task *)pkt->data;
+
 	OMDEBUG_MIG(1, "MIG_TASK\n");
 	/* arch specific proc receive context */
 	arch_mig_receive_proc_context(p, m);
@@ -220,15 +468,12 @@
 	p->om.tgid = m->tgid;
 
 	/* copy credentials */
-	p->uid = m->uid;
-	p->euid = m->euid;
-	p->suid = m->suid;
-	p->fsuid = m->fsuid;
-
-	p->gid = m->gid;
-	p->egid = m->egid;
-	p->sgid = m->sgid;
-	p->fsgid = m->fsgid;
+	sys_setuid(m->uid);
+	sys_setresuid(m->uid,m->euid,m->suid);
+	sys_setfsuid(m->fsuid);
+	sys_setgid(m->gid);
+	sys_setresgid(m->gid, m->egid, m->sgid);
+	sys_setfsgid(m->fsgid);
 
 	/* signals stuffs */
 	p->blocked = m->blocked;
@@ -238,177 +483,196 @@
 	memcpy(p->sighand->action, m->sighand, sizeof(struct k_sigaction)
 								* _NSIG);
 
-	/* FIXME we don't trust the other node anyway so copy rlimit from node[nr] */
-
-	memcpy(p->comm, m->comm, sizeof(m->comm));
+  	/* FIXME we don't trust the other node anyway so copy rlimit from node[nr] */
 
-	p->personality = m->personality;
+  	memcpy(p->comm, m->comm, sizeof(m->comm));
+	/* p->personality = m->personality;*/
+	set_personality(m->personality);
 	arch_pick_mmap_layout(p->mm);
+
+	kcom_send_ack(p, pkt);
+	printk("leaving FUNCTION: mig_do_receive_proc_context\n");
+
+	return 0;
 }
 
 /**
- * mig_do_receive - Receive all process stuff (mm, pages, fpr, ..)
+ * mig_do_receive
+ * @p:		task
+ *
+ * Description:
+ *    Main loop to receive all process stuff (mm, pages, fpr, ..)
  **/
 int mig_do_receive(task_t *p)
 {
-	int error;
-	unsigned int got_not_coming = 0;
-	unsigned long data;
-	struct omp_req req;
+	struct kcom_task *mytsk=NULL;
+	struct kcom_pkt *pkt, *pkt_next;
+	int ret;
 
-	data = __get_free_page(GFP_KERNEL);
-	if (!data)
-		goto fail;
 
 	task_set_dflags(p, DINCOMING);
-	clear_used_math();
+	/* clear_used_math();*/
 
-	while (1) {
-		error = comm_recv(p->om.contact, &req, sizeof(req));
-		if (error < 0)
-			goto fail;
-
-		BUG_ON(req.dlen > PAGE_SIZE);
-		error = comm_recv(p->om.contact, (void *) data, req.dlen);
-		if (error < 0)
-			goto fail;
-
-		switch (req.type) {
-		case MIG_MM:
-			mig_do_receive_mm(p, (struct omp_mig_mm *) data);
-			break;
-		case MIG_VMA:
-			if (mig_do_receive_vma(p, (struct omp_mig_vma *) data))
-				goto fail;
-			break;
-		case MIG_PAGE:
-			if (mig_do_receive_page(p, *((unsigned long *) data)))
-				goto fail;
-			break;
-		case MIG_FP:
-			mig_do_receive_fp(p, (struct omp_mig_fp *) data);
-			break;
-		case MIG_ARCH:
-			if (arch_mig_receive_specific(p, (struct omp_mig_arch *) data))
-				goto fail;
-			break;
-		/* this is the last thing we do in the chain of receiving,
-		 * so return 0 after we're done */
-		case MIG_TASK:
-			mig_do_receive_proc_context(p, (struct omp_mig_task *) data);
-			comm_send_req(p->om.contact, MIG_TASK | REPLY);
-			task_clear_dflags(p, DINCOMING);
-
-			flush_tlb_mm(p->mm); /* for all the new pages */
-			return 0;
-		case MIG_ABORT:
-			printk("mig_do_recv(): got MIG_ABORT\n");
-			got_not_coming = 1;
-			goto fail;
-		default:
-			printk("mig_do_recv(): got default\n");
-			goto fail;
-		}
+	/* Wait for kcomd to set up the kcom_task struct*/
+	while (mytsk==NULL) {
+		schedule_timeout_interruptible(HZ/1000);
+	/*spin_lock();*/
+		mytsk=kcom_task_find(p->pid);
+	/*spin_unlock();*/
 	}
-fail:
-	task_clear_dflags(p, DINCOMING);
-	free_page(data);
 
-	OMBUG("failed\n");
-	return -1;
-}
-
-static NORET_TYPE int mig_handle_migration(void *ptr)
-{
-	task_t *p = current;
-	/* link against the other end */
-	struct socket *link = (struct socket *) ptr;
-	int error;
+	/* Initialize remote proc's whereto*/
+	if (task_test_dflags(p, DREMOTE)) {
+		memcpy(p->om.whereto, &mytsk->node->addr, sizeof(mytsk->node->addr));
+		printk("Setting mytsk->rpid (currently: %u) = p->pid (%u)\n", mytsk->rpid, p->pid);
+	}
 
-	OM_VERBOSE_MIG("[OM] receiving new process\n");
+	set_current_state(TASK_INTERRUPTIBLE);
+	/* spin_lock(&mytsk->spinlock);*/
+	while (1) {
 
-	task_set_comm(p, link);
+		if (!list_empty(&mytsk->in_packs))
+			list_for_each_entry_safe(pkt, pkt_next, &mytsk->in_packs, list) {
+				printk("mig_do_receive:  msgid=%u\n", pkt->msgid);
+
+				switch (pkt->type & MIG_MASK) {
+
+					case MIG_MM:
+						mig_do_receive_mm(p, pkt);
+						list_del(&pkt->list);
+						kmem_cache_free(kcom_pkt_cachep, pkt);
+						break;
+
+					case MIG_VMA:
+						ret=mig_do_receive_vma(p, pkt);
+						list_del(&pkt->list);
+						kmem_cache_free(kcom_pkt_cachep, pkt);
+						if (ret)
+							return ret;
+						break;
+
+					case MIG_PAGE:
+						ret=mig_do_receive_page(p, pkt);
+						list_del(&pkt->list);
+						kmem_cache_free(kcom_pkt_cachep, pkt);
+						if (ret)
+							return ret;
+						break;
+
+					case MIG_FP:
+						ret=mig_do_receive_fp(p, pkt);
+						list_del(&pkt->list);
+						kmem_cache_free(kcom_pkt_cachep, pkt);
+						if (ret)
+							return ret;
+						break;
+
+					/* this is the last thing we do in the chain of receiving,
+					 * so return 0 after we're done */
+					case MIG_TASK:
+						ret=mig_do_receive_proc_context(p, pkt);
+
+						task_clear_dflags(p, DINCOMING);
+						flush_tlb_mm(p->mm); /* for all the new pages */
+
+						list_del(&pkt->list);
+						kmem_cache_free(kcom_pkt_cachep, pkt);
+						/* spin_unlock(&mytsk->spinlock);*/
+
+						if (ret)
+							return ret;
+						set_current_state(TASK_RUNNING);
+						return 0;
+
+					default:
+						printk("[pid: %u] Unknown packet type 0x%x received.\n", p->pid, pkt->type);
+						break;
 
-	error = obtain_mm(p);
-	if (error)
-		goto fail;
-	if (mig_recv_hshake(link))
-		goto fail;
+				}
 
-	error = mig_do_receive(p);
+			}
 
-	if (error)
-		goto fail;
+	/* spin_unlock(&mytsk->spinlock);*/
+	printk("process %u going to sleep.\n", p->pid);
+	schedule();
+	printk("process %u waking up.\n", p->pid);
+	set_current_state(TASK_INTERRUPTIBLE);
+	/* spin_lock(&mytsk->spinlock);*/
+	}
+	/* spin_unlock(&mytsk->spinlock);*/
 
-	OM_VERBOSE_MIG("[OM] starting process(%d)\n", p->pid);
-	reparent_to_init();
-	arch_kickstart(p);
-	/*NOTREACHED*/
 
-fail:
-	OMBUG("failed\n");
-	do_exit(SIGKILL);
-	/*NOTREACHED*/
 }
 
 /**
- * openmosix_mig_daemon - openMosix migration daemon
- * @nothing:	unused
+ * mig_handle_migration
+ * @*pid:		address to pid used in mig_do_receive_init.
+ *             mig_do_receive_init waits until pid!=0, before setting
+ *             up task and sending ack back to home node.
  *
  * Description:
- * 	start the migration daemon.
- * 	wait for communication, and if it is a remote request
- * 	then start a user-thread with the new program to run
+ *    This is the newly created process.
  **/
-int openmosix_mig_daemon(void *nothing)
+KCOMD_NSTATIC NORET_TYPE int mig_handle_migration(pid_t *pid)
 {
 	task_t *p = current;
 	int error;
-	struct socket *mlink;
-	struct sockaddr saddr;
 
-	om_daemonize("omkmigd", 0);
+	/* reparent before anything real happens to the process so nothing gets*/
+	/* re-initialized.*/
+	reparent_to_init();
+	error = obtain_mm(p);
+	if (error)
+		goto fail;
+	/* clear_used_math(); // not really sure what this does?*/
 
-	task_set_dflags(p, DREMOTEDAEMON);
+	task_set_dflags(p, DREMOTE);
+	*pid=p->pid;
 
-	set_our_addr(AF_INET, &saddr, REMOTE_DAEMON_PORT);
+	OM_VERBOSE_MIG("[OM] receiving new process\n");
 
-restart:
-	if (!p->om.contact) {
-		p->om.contact = comm_setup_listen(&saddr);
-		if (!p->om.contact) {
-			printk(KERN_WARNING
-				"omkmigd: failed to open mig service\n");
-			flush_signals(p);
-			set_current_state(TASK_INTERRUPTIBLE);
-			schedule_timeout(HZ);
-			goto restart;
-		}
-	}
+	error = mig_do_receive(p);
+	if (error)
+		goto fail;
 
-	/* migration daemon loop */
-	while (1)
-	{
-		error = comm_accept(p->om.contact, &mlink, &saddr, 0UL);
-		if (error == -EINTR || error == -ERESTART || error == -EAGAIN
-		   || error == -ERESTARTSYS)
-		{
-			if (sigismember(&(p->pending.signal), SIGCHLD)) {
-				printk("omigd: SIGCHLD caught\n");
-			}
-			flush_signals(p);
-			continue;
-		} else if (error) {
-			OMBUG("failed to accept\n");
-			comm_close(mlink);
-			goto restart;
-		}
+	/* clear_tsk_thread_flag(p, TIF_SIGPENDING); // Added by MSD ???*/
+	/* init_sigpending(&p->pending); // Added by MSD ????*/
+
+	/* OM_VERBOSE_MIG("[OM] starting process(%d)\n", p->pid);*/
+	set_current_state(TASK_RUNNING);
+	/* set_current_state(TASK_INTERRUPTIBLE);*/
+	schedule();
+	if (task_test_dflags(p, DREMOTE))
+	printk("[OM] starting remote process(%d)\n", p->pid);
+	else
+	printk("[OM] starting local process(%d)\n", p->pid);
+
+	#if 0
+	flush_signals(p); //MSDMSD
+	reparent_to_init(); //MSD
+	cap_clear(p->cap_permitted);
+	cap_clear(p->cap_effective);
+	cap_task_reparent_to_init(p);
+	p->cap_permitted=0;
+	p->cap_effective=0;
+	#endif
 
-		error = user_thread(mig_handle_migration, (void *) mlink, 0);
-		if (error < 0)
-			comm_close(mlink);
+	clear_thread_flag(TIF_SIGPENDING);
+	/* flush_thread();*/
+	arch_kickstart(p);
+	/*NOTREACHED*/
+
+	printk("process %u, waking up. YOU SHOULD NOT SEE THIS!!!!!\n", p->pid);
+	while (1) {
+		set_current_state(TASK_INTERRUPTIBLE);
+		schedule();
 	}
 
-	/* Not reached, just to prevent warning on recent gcc: */
-	return 0;
+
+fail:
+	printk("mig_handle_migration failed with %d\n", error);
+	OMBUG("failed\n");
+	do_exit(SIGKILL);
+	/*NOTREACHED*/
 }
+

Index: linux/hpc/migsend.c
===================================================================
--- linux.orig/hpc/migsend.c	2006-09-20 17:07:00.000000000 +0200
+++ linux/hpc/migsend.c	2006-09-20 18:00:28.000000000 +0200
@@ -31,6 +31,8 @@
 #include <hpc/hpc.h>
 #include <hpc/debug.h>
 #include <hpc/service.h>
+#include <linux/in.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/mig.h>
 #include <hpc/protocol.h>
@@ -63,7 +65,7 @@
 		/* it's a no go ... */
 		OMBUG("no go\n");
 		return -1;
-	}			
+	}
 
 	return 0;
 }
@@ -77,12 +79,20 @@
 static int mig_send_fp(task_t *p)
 {
 	struct omp_mig_fp m;
+	struct sockaddr_in *dest_ptr;
 
-	if (!used_math())
+	dest_ptr=(void *)p->om.whereto;
+	if (!used_math()) {
+		printk("leaving FUNCTION: mig_send_fp, !used_math\n");
 		return 0;
+	}
 
 	arch_mig_send_fp(p, &m);
-	return comm_send_hd(p->om.contact, MIG_FP, &m, sizeof(m));
+	if (task_test_dflags(p, DREMOTE))
+		kcom_send_with_ack(MIG_FP | REM_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
+	else
+		kcom_send_with_ack(MIG_FP | DEP_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
+	return 0;
 }
 
 
@@ -95,11 +105,18 @@
  **/
 static int mig_send_mm(task_t *p)
 {
-	struct omp_mig_mm s;
+	struct sockaddr_in *dest_ptr;
+	int ret;
 
-	memcpy(&s, &p->mm->start_code, sizeof(s));
+	dest_ptr=(void *)p->om.whereto;
+	printk("FUNCTION: mig_send_mm.\n");
+	if (task_test_dflags(p, DREMOTE))
+		ret=kcom_send_with_ack(MIG_MM | REM_FLG, sizeof(struct omp_mig_mm), (char *)&p->mm->start_code, 0, dest_ptr);
+	else
+		ret=kcom_send_with_ack(MIG_MM | DEP_FLG, sizeof(struct omp_mig_mm), (char *)&p->mm->start_code, 0, dest_ptr);
+	printk("leaving FUNCTION: mig_send_mm\n");
 
-	return comm_send_hd(p->om.contact, MIG_MM, &s, sizeof(s));
+	return ret;
 }
 
 /**
@@ -141,23 +158,29 @@
 	struct vm_area_struct *vma;
 	struct omp_mig_vma m;
 	int ret = 0;
+	struct sockaddr_in *dest_ptr;
 
+	dest_ptr=(void *)p->om.whereto;
 	for (vma = p->mm->mmap; vma; vma = vma->vm_next)
 	{
 		m.vm_start= vma->vm_start;
 		m.vm_size = vma->vm_end - vma->vm_start;
 		m.vm_flags = vma->vm_flags;
 		m.vm_file = vma->vm_file;
-		m.vm_pgoff = 0;
+		// m.vm_pgoff = 0;
+		m.vm_pgoff = vma->vm_pgoff;
 		if (vma->vm_file)
 			mig_send_vma_file(p, vma, &m);
 
-		ret = comm_send_hd(p->om.contact, MIG_VMA, &m, sizeof(m));
-		if (ret < 0) {
-			OMBUG("send vma failed\n");
-			break;
+		if (task_test_dflags(p, DREMOTE))
+			ret=kcom_send_with_ack(MIG_VMA | REM_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
+		else
+			ret=kcom_send_with_ack(MIG_VMA | DEP_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
+
+		if (ret != 0) {
+			printk("ERROR sending vmas\n");
+			return -1;
 		}
-
 	}
 	return ret;
 }
@@ -174,29 +197,30 @@
 {
 	struct vm_area_struct * vma;
 	unsigned long addr;
-	int error;
+	struct sockaddr_in *dest_ptr;
+	char *data;
+
+	printk("FUNCTION: mig_send_pages\n");
+	data=kzalloc(PAGE_SIZE, GFP_KERNEL);
 
+	dest_ptr=(void *)p->om.whereto;
 	for (vma = p->mm->mmap; vma; vma = vma->vm_next)
 	{
 		if (!(vma->vm_flags & VM_READ))
 			continue;
 		for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE)
 		{
-			error = comm_send_hd(p->om.contact, MIG_PAGE, &addr,
-								sizeof(addr));
-			if (error < 0)
-				goto fail;
-
-			error = comm_send(p->om.contact, (void *) addr,
-								PAGE_SIZE);
-			if (error < 0)
-				goto fail;
+			// FIXME:   mem pages need to be copied into a buffer, then send the buffer.   ???
+			memcpy(data, (void *)addr, PAGE_SIZE);
+			if (task_test_dflags(p, DREMOTE))
+				kcom_send_with_ack(MIG_PAGE | REM_FLG, PAGE_SIZE, (char *)data, addr, dest_ptr);
+			else
+				kcom_send_with_ack(MIG_PAGE | DEP_FLG, PAGE_SIZE, (char *)data, addr, dest_ptr);
+
 		}
 	}
+	kfree(data);
 	return 0;
-fail:
-	OMBUG("failed (addr: %p)\n", (void *) addr);
-	return -1;
 }
 
 
@@ -210,8 +234,9 @@
 static int mig_send_proc_context(task_t *p)
 {
 	struct omp_mig_task m;
-	struct omp_req req;
-	int error;
+	struct sockaddr_in* dest_ptr;
+
+	dest_ptr=(void *)p->om.whereto;
 
 	m.ptrace = p->ptrace;
 
@@ -248,18 +273,13 @@
 
 	arch_mig_send_proc_context(p, &m);
 
-	error = comm_send_hd(p->om.contact, MIG_TASK, &m, sizeof(m));
-	if (error < 0)
-		goto fail;
-
-	error = comm_recv(p->om.contact, &req, sizeof(req));
+	if (task_test_dflags(p, DREMOTE))
+		kcom_send_with_ack(MIG_TASK | REM_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
+	else
+		kcom_send_with_ack(MIG_TASK | DEP_FLG, sizeof(m), (char *)&m, 0, dest_ptr);
 
-	if (req.type == (MIG_TASK | REPLY))
-		return 0; /* commit point */
+	return 0;
 
-fail:
-	OMBUG("failed\n");
-	return -1;
 }
 
 /**
@@ -271,8 +291,20 @@
  **/
 int mig_do_send(task_t *p)
 {
+	struct sockaddr_in* dest_ptr=(void *)p->om.whereto;
+	unsigned int addr=dest_ptr->sin_addr.s_addr;
 	arch_mig_send_pre(p);
 
+	if (task_test_dflags(p, DREMOTE))  {
+		printk("Sending MIG_GO_HOME\n");
+		if (kcom_send_with_ack(MIG_GO_HOME | REM_FLG, 0, NULL, 0, dest_ptr))
+			goto fail_mig;
+	} else {
+		printk("Sending MIG_INIT\n");
+		if (kcom_send_with_ack(MIG_INIT | DEP_FLG, 0, NULL, 0, dest_ptr))
+			goto fail_mig;
+	}
+
 	if (mig_send_mm(p)) goto fail_mig;
 	if (mig_send_vmas(p)) goto fail_mig;
 	if (mig_send_pages(p)) goto fail_mig;
@@ -282,9 +314,13 @@
 
 	arch_mig_send_post(p);
 
+	printk("Process %u now migrated to %u.%u.%u.%u\n", p->pid,
+					(0x000000FF & addr), (0x0000FF00 & addr)>>8,
+					(0x00FF0000 & addr)>>16, (0xFF000000 & addr) >> 24);
+
 	return 0;
 fail_mig:
+	printk("error FUNCTION: mig_do_send\n");
 	OMBUG("failed\n");
-	comm_send_req(p->om.contact, MIG_ABORT);
 	return -1;
 }

Index: linux/hpc/copyuser.c
===================================================================
--- linux.orig/hpc/copyuser.c	2006-09-18 23:29:14.000000000 +0200
+++ linux/hpc/copyuser.c	2006-09-18 23:35:45.000000000 +0200
@@ -15,6 +15,8 @@
 #include <linux/sched.h>
 #include <hpc/protocol.h>
 #include <hpc/debug.h>
+#include <linux/in.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/hpc.h>
 
@@ -32,7 +34,9 @@
 unsigned long deputy_copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	struct omp_usercopy_req u;
-	int error;
+	task_t *p=current;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
+	int i;
 
 	if (in_atomic())
 		return n;
@@ -42,18 +46,10 @@
 
 	OMDEBUG_CPYUSER(1, "copy user[0x%p]->[0x%p][%ld]\n", from, to, n);
 
-	error = comm_send_hd(current->om.contact, DEP_COPY_FROM_USER, &u, sizeof(u));
-	if (error < 0)
-		goto error;
-
-	error = comm_recv(current->om.contact, to, n);
-	if (error < 0)
-		goto error;
+	i=kcom_send_with_response(DEP_COPY_FROM_USER | PKT_NEW_MSG | DEP_FLG, sizeof(u), (char *)&u, 0, (char *)to, dest_ptr);
+
 
 	return 0;
-error:
-	OMBUG("error %d\n", error);
-	return -1;
 }
 EXPORT_SYMBOL(deputy_copy_from_user);
 
@@ -69,25 +65,21 @@
 unsigned long deputy_strncpy_from_user(char *dst, const char __user *src,
 							long count)
 {
-	struct omp_usercopy_req u;
-	int error;
+	task_t *p=current;
+	struct omp_usercopy_req *u;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
+	int i;
 
 	OMDEBUG_CPYUSER(1, "strcpy user[0x%p]->[0x%p][%ld]\n", src, dst, count);
-	u.addr = (unsigned long) src;
-	u.len = count;
-	error = comm_send_hd(current->om.contact, DEP_STRNCPY_FROM_USER, &u,
-								sizeof(u));
-	if (error < 0)
-		goto error;
-
-	error = comm_recv(current->om.contact, dst, count);
-	if (error < 0)
-		goto error;
+	u=kzalloc(sizeof(*u), GFP_KERNEL);
+
+	u->addr = (unsigned long) src;
+	u->len = count;
+
+	i=kcom_send_with_response(DEP_STRNCPY_FROM_USER | PKT_NEW_MSG | DEP_FLG, sizeof(*u), (char *)u, 0, dst, dest_ptr);
 
 	return 0;
-error:
-	OMBUG("error %d\n", error);
-	return -1;
+
 }
 
 /**
@@ -101,29 +93,22 @@
  **/
 unsigned long deputy_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
-	struct omp_usercopy_req u;
-	int error;
+	int i;
+	char *buf;
+	task_t *p=current;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
 
 	if (in_atomic())
 		return n;
 
-	OMDEBUG_CPYUSER(1, "copy [0x%p]->user[0x%p][%ld]\n", from, to, n);
-	u.addr = (unsigned long) to;
-	u.len = n;
+	// Instead of sending two packets, we create one larger packet and send it.
+	buf=kzalloc(n, GFP_KERNEL);
+	memcpy(buf, from, n);
 
-	error = comm_send_hd(current->om.contact, DEP_COPY_TO_USER, &u,
-							sizeof(u));
-	if (error < 0)
-		goto error;
-
-	error = comm_send(current->om.contact, (void *) from, n);
-	if (error < 0)
-		goto error;
+	i=kcom_send_with_ack(DEP_COPY_TO_USER | PKT_NEW_MSG | DEP_FLG, n, buf, (unsigned long) to, dest_ptr);
 
 	return 0;
-error:
-	OMBUG("error %d\n", error);
-	return -1;
+
 }
 EXPORT_SYMBOL(deputy_copy_to_user);
 
@@ -138,25 +123,18 @@
 unsigned long deputy_strnlen_user(const char *s, long n)
 {
 	struct omp_usercopy_req u;
-	int error;
 	long ret;
+	int i;
+	task_t *p=current;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
 
 	OMDEBUG_CPYUSER(1, "strlen user[0x%p][%ld]\n", s, n);
 	u.addr = (unsigned long) s;
 	u.len = n;
 
-	error = comm_send_hd(current->om.contact, DEP_STRNLEN_USER, &u, sizeof(u));
-	if (error < 0)
-		goto out;
-
-	error = comm_recv(current->om.contact, &ret, sizeof(ret));
-	if (error < 0)
-		goto out;
+	i=kcom_send_with_response(DEP_STRNLEN_USER | PKT_NEW_MSG | DEP_FLG, sizeof(u), (char *)&u, 0, (char *)&ret, dest_ptr);
 
 	return ret;
-out:
-	OMBUG("error %d\n", error);
-	return 0;
 }
 EXPORT_SYMBOL(deputy_strnlen_user);
 
@@ -172,7 +150,9 @@
 static inline long deputy_put_userX(s64 value, const void *addr, size_t size)
 {
 	struct omp_usercopy_emb u;
-	int error;
+	int i;
+	task_t *p=current;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
 
 	OMDEBUG_CPYUSER(1, "put (%lld)->user[0x%p][%zd]\n", value, addr, size);
 
@@ -180,13 +160,9 @@
 	u.len = size;
 	u.val = value;
 
-	error = comm_send_hd(current->om.contact, DEP_PUT_USER, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	i=kcom_send_with_ack(DEP_PUT_USER | PKT_NEW_MSG | DEP_FLG, sizeof(u), (char *)&u, 0, dest_ptr);
+
 	return 0;
-out:
-	OMBUG("error %d\n", error);
-	return -EFAULT;
 }
 
 /**
@@ -227,32 +203,21 @@
  **/
 static inline long deputy_get_userX(s64 *value, const void *addr, size_t size)
 {
+	task_t *p=current;
 	struct omp_usercopy_req u;
-	int error;
 	s64 ret;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
+	int i;
 
 	OMDEBUG_CPYUSER(1, "get user[0x%p][%zd]\n", addr, size);
 	u.addr = (unsigned long) addr;
 	u.len = size;
 
-	error = comm_send_hd(current->om.contact, DEP_GET_USER, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	i=kcom_send_with_response(DEP_GET_USER | PKT_NEW_MSG | DEP_FLG, sizeof(u), (char *)&u, 0, (void *)&ret, dest_ptr);
 
-	error = comm_recv(current->om.contact, &ret, sizeof(ret));
-	if (error < 0)
-		goto out;
+	*value=ret;
 
-	switch (size) {
-	case 1: { s8 *v = (s8 *) value; *v = (s8) ret; break; }
-	case 2: { s16 *v = (s16 *) value; *v = (s16) ret; break; }
-	case 4: { s32 *v = (s32 *) value; *v = (s32) ret; break; }
-	case 8: *value = ret; break;
-	}
 	return 0;
-out:
-	OMBUG("error %d\n", error);
-	return -EFAULT;
 }
 
 /**
@@ -281,105 +246,95 @@
 /**
  * remote_copy_user - Copy to or from user for deputy
  **/
-static int remote_copy_user(task_t *p, int request)
+static int remote_copy_from_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_req u;
 	void *buf = NULL;
-	int error;
+	int ret;
 
-	error = comm_recv(p->om.contact, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	memcpy(&u, pkt->data, pkt->len);
 
 	buf = kmalloc(u.len, GFP_KERNEL);
 	if (!buf)
 		goto out;
 
-	switch (request) {
-	case DEP_COPY_FROM_USER:
-		copy_from_user(buf, (const void __user *) u.addr, u.len);
-
-		error = comm_send(p->om.contact, buf, u.len);
-		if (error < 0)
-			goto out;
-		break;
-	case DEP_COPY_TO_USER:
-		error = comm_recv(p->om.contact, buf, u.len);
-		if (error < 0)
-			goto out;
+	ret=copy_from_user(buf, (const void __user *) u.addr, u.len);
+
+	kcom_send_resp(p, u.len, buf, pkt);
+
+	return ret;
 
-		copy_to_user((void __user *) u.addr, buf, u.len);
-		break;
-	}
 out:
 	kfree(buf);
-	return error;
+	return -1;
 }
 
+static int remote_copy_to_user(task_t *p, struct kcom_pkt *pkt)
+{
+	int ret;
+	ret=copy_to_user((void __user *) pkt->addr, pkt->data, pkt->len);
+
+	kcom_send_ack(p, pkt);
+
+	return ret;
+
+}
 /**
  * remote_strncpy_from_user - strncpy from user for deputy
  **/
-static int remote_strncpy_from_user(task_t *p)
+static int remote_strncpy_from_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_req u;
 	void *buf = NULL;
-	int error;
+	int ret;
 
-	error = comm_recv(p->om.contact, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	memcpy(&u, pkt->data, sizeof(u));
 
 	buf = kmalloc(u.len, GFP_KERNEL);
 	if (!buf)
 		goto out;
 
-	strncpy_from_user(buf, (const char __user *) u.addr, u.len);
+	ret=strncpy_from_user(buf, (const char __user *) u.addr, u.len);
 
-	error = comm_send(p->om.contact, buf, u.len);
-	if (error < 0)
-		goto out;
+	kcom_send_resp(p, u.len, buf, pkt);
+
+	return ret;
 
 out:
-	kfree(buf);
-	return error;
+	return -1;
+
 }
 
 /**
  * remote_strnlen_from_user - strnlen from user for deputy
  **/
-static int remote_strnlen_user(task_t *p)
+static int remote_strnlen_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_req u;
-	long ret;
-	int error;
+	long *ret_ptr;
 
-	error = comm_recv(p->om.contact, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	memcpy(&u, pkt->data, pkt->len);
+
+	ret_ptr=kzalloc(sizeof(long), GFP_KERNEL);
 
-	ret = (u.len)
+	*ret_ptr = (u.len)
 		? strnlen_user((const char __user *) u.addr, u.len)
 		: strlen_user((const char __user *) u.addr);
 
-	error = comm_send(p->om.contact, &ret, sizeof(ret));
-	if (error < 0)
-		goto out;
-out:
-	return error;
+	kcom_send_resp(p, sizeof(*ret_ptr), (char *)ret_ptr, pkt);
+
+	return 0;
 }
 
 /**
  * remote_put_user - put user for deputy
  **/
-static int remote_put_user(task_t *p)
+static int remote_put_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_emb u;
 	long ret;
-	int error;
 
-	error = comm_recv(p->om.contact, &u, sizeof(u));
-	if (error < 0)
-		goto out;
+	memcpy(&u, pkt->data, pkt->len);
 
 	switch (u.len) {
 	case 1: ret = put_user(u.val, (u8 *) u.addr); break;
@@ -388,81 +343,112 @@
 	case 8: ret = put_user(u.val, (u64 *) u.addr); break;
 	default: ret = -EFAULT;
 	}
+
+	kcom_send_ack(p, pkt);
+
 	return ret;
-out:
-	return error;
 }
 
 /**
  * remote_get_user - get user for deputy
  **/
-static int remote_get_user(task_t *p)
+static int remote_get_user(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_usercopy_req u;
-	s64 ret;
-	int error;
+	s64 *ret;
+
+	memcpy(&u, pkt->data, sizeof(struct omp_usercopy_req));
+	ret=kzalloc(sizeof(*ret), GFP_KERNEL);
 
-	error = comm_recv(p->om.contact, &u, sizeof(u));
-	if (error < 0)
-		goto out;
 
 	switch (u.len) {
-	case 1: get_user(ret, (u8 *) u.addr); break;
-	case 2: get_user(ret, (u16 *) u.addr); break;
-	case 4: get_user(ret, (u32 *) u.addr); break;
+	case 1: get_user(*ret, (u8 *) u.addr); break;
+	case 2: get_user(*ret, (u16 *) u.addr); break;
+	case 4: get_user(*ret, (u32 *) u.addr); break;
 #if BITS_PER_LONG == 64
-	case 8: get_user(ret, (u64 *) u.addr); break;
+	case 8: get_user(*ret, (u64 *) u.addr); break;
 #endif
 	}
 
-	error = comm_send(p->om.contact, &ret, sizeof(ret));
-	if (error < 0)
-		goto out;
-out:
-	return error;
+	kcom_send_resp(p, sizeof(*ret), (char *)ret, pkt);
+return 0;
 }
 
-
 /**
  * remote_handle_user - Handle user copy until receiving @endtype
  **/
 int remote_handle_user(task_t *p, int endtype)
 {
-	struct omp_req req;
-	int ret = 1;
-	int error;
-
-	while (ret != 0) {
-		error = comm_recv(p->om.contact, &req, sizeof(req));
-		if (error < 0)
-			return error;
-		if (req.type == endtype) {
-			ret = 0;
-			break;
+	int ret = 0;
+	struct kcom_pkt *pkt, *pkt_next, *send_pkt;
+	struct kcom_task *task;
+
+	task=kcom_task_find(p->pid);
+
+	while (1) {
+
+		set_current_state(TASK_INTERRUPTIBLE);
+		list_for_each_entry_safe(pkt, pkt_next, &task->in_packs, list) {
+
+			switch ( pkt->type & SYSCALL_MASK ) {
+
+				case DEP_STRNCPY_FROM_USER :
+					remote_strncpy_from_user(p, pkt);
+					break;
+
+			 	case DEP_GET_USER :
+					remote_get_user(p, pkt);
+					break;
+
+				case DEP_COPY_FROM_USER :
+					remote_copy_from_user(p, pkt);
+					break;
+
+				case DEP_COPY_TO_USER :
+					remote_copy_to_user(p, pkt);
+					break;
+
+				case DEP_PUT_USER :
+					remote_put_user(p, pkt);
+					break;
+
+			 	case SYSCALL_DONE :
+					send_pkt=kcom_pkt_create(0, MIG_SYSCALL | PKT_ACK | SYSCALL_DONE | REM_FLG, PKT_ACK, NULL);
+					send_pkt->msgid=pkt->msgid;
+					send_pkt->hpid=task->hpid;
+					send_pkt->rpid=task->rpid;
+					send_pkt->resp=pkt->resp;
+
+					list_add_tail(&send_pkt->list, &task->out_packs);
+
+					if (kcomd_task) {
+						send_sig(SIGHUP, kcomd_task, 0);
+					} else
+						printk("Unable to signal kcomd\n");
+
+					ret=pkt->addr;  /* the return value is stored in the addr field */
+					list_del(&pkt->list);
+					kmem_cache_free(kcom_pkt_cachep, pkt);
+					set_current_state(TASK_RUNNING);
+					schedule();
+					return ret;
+					break;
+
+			}
+
+			list_del(&pkt->list);
+			kmem_cache_free(kcom_pkt_cachep, pkt);
 		}
 
-		switch (req.type) {
-		case DEP_COPY_FROM_USER:
-		case DEP_COPY_TO_USER:
-			remote_copy_user(p, req.type);
-			break;
-		case DEP_STRNCPY_FROM_USER:
-			remote_strncpy_from_user(p);
-			break;
-		case DEP_STRNLEN_USER:
-			remote_strnlen_user(p);
-			break;
-		case DEP_PUT_USER:
-			remote_put_user(p);
-			break;
-		case DEP_GET_USER:
-			remote_get_user(p);
-			break;
-		default:
-			OMBUG("unexpected type [%x]\n", req.type);
-			remote_disappear();
-			break;
+		if (list_empty(&task->in_packs)) {
+			schedule();
 		}
+
 	}
-	return ret;
+
+	printk( KERN_ERR "openMosix: %s is not supposed to end that way ! (%s:%n)\n"
+	      , __FUNCTION__, __FILE__, __LINE__);
+
+	return -1;
+
 }

Index: linux/hpc/deputy.c
===================================================================
--- linux.orig/hpc/deputy.c	2006-09-14 11:40:48.000000000 +0200
+++ linux/hpc/deputy.c	2006-09-14 12:01:31.000000000 +0200
@@ -29,6 +29,7 @@
 #include <hpc/arch.h>
 #include <hpc/syscalls.h>
 #include <hpc/debug.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/hpc.h>
 
@@ -41,26 +42,43 @@
 /**
  * deputy_do_syscall - process a syscall coming from remote
  **/
-static int deputy_do_syscall(task_t *p)
+static int deputy_do_syscall(task_t *p, struct kcom_pkt *pkt)
 {
 	struct omp_syscall_req s;
 	struct omp_syscall_ret r;
-	int error;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
+	struct kcom_task *send_tsk;
+	struct kcom_pkt *send_pkt;
+	// task_t *kcomd_task;
+
+	// Send MIG_SYSCALL ack
+	send_tsk=kcom_task_find(p->pid);
+
+	send_pkt=kcom_pkt_create(0, MIG_SYSCALL | PKT_ACK | DEP_FLG, PKT_ACK, NULL);
+
+	send_pkt->msgid=pkt->msgid; // responses have same msg id as pkt they are responding to.
+	send_pkt->hpid=pkt->hpid;
+	send_pkt->rpid=pkt->rpid;
+	send_pkt->resp=pkt->resp;
+
+	// spin_lock(&send_tsk->spinlock);
+	list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+	// spin_unlock(&send_tsk->spinlock);
+
+	if (kcomd_task != NULL)
+		send_sig(SIGHUP,kcomd_task,0);
+
+	if (sizeof(struct omp_syscall_req) != pkt->len)
+		printk("ERROR in deputy_do_syscall.  data size of %d does not match expected %d\n", pkt->len, sizeof(struct omp_syscall_req));
+	memcpy(&s, pkt->data, pkt->len);
 	
-	error = comm_recv(p->om.contact, &s, sizeof(s));
-	if (error < 0)
-		return -1;
-
 	OMDEBUG_SYS(1, "[deputy] receive syscall %d\n", s.n);
 
 	/* do the syscall and put reply in r */
 	r.ret = arch_exec_syscall(s.n, (syscall_parameter_t *) &s.arg);
 	
-	error = comm_send_hd(p->om.contact, REM_SYSCALL | REPLY, &r, sizeof(r));
-	if (error < 0)
-		return -1;
+	kcom_send_with_ack(MIG_SYSCALL | PKT_NEW_MSG | SYSCALL_DONE | DEP_FLG, 0, NULL, r.ret, dest_ptr);
 
-	OMDEBUG_SYS(2, "[deputy] replied to syscall %d\n", s.n);
 
 	return 0;
 }
@@ -379,25 +397,38 @@
 static inline void deputy_do_sigpending(task_t *p)
 {
 	siginfo_t info;
+
+	#if 0
 	struct omp_signal s;
-	int signr, error;
+	struct sockaddr_in *dest_ptr=p->om.whereto;
+	#endif
+
+	int signr;
 
 	do_signal(ARCH_TASK_GET_USER_REGS(p), NULL);
 	
-	return;
-	
-	for (;;) {
+
+	printk(KERN_WARNING "%s does not yet know how to process signals ... \n", __FUNCTION__ );
+
+	while (signal_pending (p)) {
 		signr = dequeue_signal(p, &p->blocked, &info);
-		if (!signr)
-			break;
 
-		s.signr = signr;
-		memcpy(&s.siginfo, &info, sizeof(siginfo_t));
+	/* FIXME:   was kinda working, not now. */
+	#if 0
+		if (signr != SIGKILL) {
+			s.signr = signr;
+			memcpy(&s.siginfo, &info, sizeof(siginfo_t));
+
+			kcom_send_with_ack(MIG_SYSCALL | PKT_NEW_MSG | DEP_SIGNAL | DEP_FLG, sizeof(struct omp_signal), &s, 0, dest_ptr);
+			printk("Signal %d sent to remote.\n", signr);
+		} else
+			printk("Signal %d discarded.\n", signr);
+	#endif
 
-		error = comm_send_hd(p->om.contact, DEP_SIGNAL, &s, sizeof(s));
-		if (error < 0)
-			OMBUG("error %d\n", error);
 	}
+
+	return;
+
 }
 
 /**
@@ -413,6 +444,7 @@
 	}
 }
 
+#if 0
 /**
  * deputy_process_communication - process has receive communication in deputy
  **/
@@ -455,20 +487,39 @@
 	if (error < 0)
 		deputy_die_on_communication();
 }
+#endif
 
 /**
  * deputy_main_loop - process loop when process is deputy
  **/
 void deputy_main_loop(void)
 {
-	int has_communication;
+	task_t *p=current;
+	struct kcom_task *mytsk;
+	int error=0;
+	struct kcom_pkt *pkt, *pkt_next;
 	
+	mytsk=kcom_task_find(p->pid);
 	while (task_test_dflags(current, DDEPUTY))
 	{
-		has_communication = comm_wait(current->om.contact);
-		if (has_communication)
-			deputy_process_communication(current);
+		set_current_state(TASK_INTERRUPTIBLE);
+
+		if (!list_empty(&mytsk->in_packs)) {
+
+			list_for_each_entry_safe(pkt, pkt_next, &mytsk->in_packs, list) {
+
+				if ((pkt->type & MIG_MASK) == MIG_SYSCALL) {
+
+
+					error = deputy_do_syscall(p, pkt);
+					list_del(&pkt->list);
+				}
+			}
+
+		}
+
 		deputy_process_misc(current);
+		schedule();
 	}
 }

Index: linux/hpc/proc.c
===================================================================
--- linux.orig/hpc/proc.c	2006-09-15 00:45:00.000000000 +0200
+++ linux/hpc/proc.c	2006-09-15 00:48:12.000000000 +0200
@@ -27,6 +27,9 @@
 #include <hpc/proc.h>
 #include <hpc/version.h>
 #include <hpc/service.h>
+#include <linux/inet.h>
+#include <linux/in.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/task.h>
 #include <hpc/debug.h>
@@ -37,29 +40,77 @@
  */
 static int proc_pid_set_where(struct task_struct *p, char *buf, size_t size)
 {
-	int ret;
-	struct sockaddr destination;
+	struct sockaddr_in* dest_ptr=(struct sockaddr_in *)p->om.whereto;
+	struct sockaddr_in *cur_addr;
+	struct kcom_task *tsk;
+	struct kcom_node *node;
+	/* task_t *kcomd_task;*/
+
+	buf[size-1]='\0';	 /* ensures no trailing crap in 'where' file*/
 
 	if (size >= 4 && strnicmp(buf, "home", 4) == 0) {
-		printk("HOME detected\n");
-		ret = task_register_migration(p, NULL);
+		printk("HOME detected - ");
+		/* p->om.whereto=NULL;*/
+		/* ret = task_register_migration(p, NULL);*/
+		if (task_test_dflags(p, DDEPUTY)) { /* if already migrated and this is home node*/
+			printk("on deputy node\n");
+			node=kcom_node_find((struct sockaddr *)dest_ptr);
+			if (node==NULL)
+				return 1;
+			tsk=kcom_task_find(p->pid);
+			if (tsk==NULL)
+				return 1;
+			kcom_task_send(tsk, MIG_COME_HOME | PKT_NEW_MSG | DEP_FLG, 0, NULL, NULL, 0);
+
+			#if 0
+			read_lock(&tasklist_lock);
+			kcomd_task = find_task_by_pid(kcom_pid);
+			read_unlock(&tasklist_lock);
+			#endif
+
+			if (kcomd_task != NULL)
+				send_sig(SIGHUP,kcomd_task,0);
+			/* kcom_send(MIG_COME_HOME | PKT_NEW_MSG | DEP_FLG, 0, NULL, 0, dest_ptr);*/
+		} else {
+			printk("on remote node\n");
+			task_register_migration(p);
+		}
 	} else {
-		ret = string_to_sockaddr(buf, &destination);
-		if (ret >= 0)
-			task_register_migration(p, &destination);
+		if (task_test_dflags(p, DDEPUTY)) {
+			tsk=kcom_task_find(p->pid);
+			if (tsk) {
+				cur_addr=(struct sockaddr_in *)&tsk->node->addr;
+
+				if ( in_aton(buf) == cur_addr->sin_addr.s_addr) { /* home -> remote - redundant migration;*/
+					printk("Process already migrated to %s\n", buf);
+					return size;
+				} /* FIXME:  remote to remote migration*/
+			}
+		} else {
+
+			dest_ptr->sin_family=AF_INET;
+			dest_ptr->sin_port=htons(DAEMON_IP4_PORT);
+			dest_ptr->sin_addr.s_addr=in_aton(buf);
+
+			task_register_migration(p);
+		}
 	}
+
 	return size;
 }
 
 static int proc_pid_get_where(struct task_struct *p, char *buf, size_t size)
 {
 	int length;
-	struct sockaddr address;
+	struct sockaddr_in *saddr;
+	unsigned int addr;
 
-	if (p->om.contact && task_test_dflags(p, DMIGRATED)) {
-		comm_getname(p->om.contact, &address, 1);
-		length = sockaddr_to_string(&address, buf);
-		length += sprintf(buf + length, "\n");
+	if (task_test_dflags(p, DMIGRATED)) {
+	/* FIXME:  what if process is currently migrating? */
+		saddr=(struct sockaddr_in *)p->om.whereto;
+		addr=saddr->sin_addr.s_addr;
+		length=sprintf(buf, "%u.%u.%u.%u\n", (0x000000FF & addr), (0x0000FF00 & addr)>>8,
+							(0x00FF0000 & addr)>>16, (0xFF000000 & addr) >> 24);
 	} else
 		length = sprintf(buf, "home\n");

Index: linux/hpc/kernel.c
===================================================================
--- linux.orig/hpc/kernel.c	2006-09-14 22:54:29.000000000 +0200
+++ linux/hpc/kernel.c	2006-09-14 22:55:47.000000000 +0200
@@ -26,6 +26,9 @@
 #include <hpc/comm.h>
 #include <hpc/mig.h>
 
+#include <hpc/kcom.h>
+#include <hpc/prototype.h>
+
 struct openmosix_options om_opts;
 EXPORT_SYMBOL_GPL(om_opts);
 
@@ -157,11 +160,26 @@
 
 static inline int remote_pre_usermode(void)
 {
-	task_t *p = current;
-
-	if (p->om.contact && comm_peek(p->om.contact))
-		remote_do_comm(p);
+	task_t *p=current;
+	struct kcom_task *mytsk;
+	struct kcom_pkt *pkt, *pkt_next;
+
+	
+	/* FIXME:  should we just add a *kcom_task to the proc struct, and speed this up? */
+	mytsk=kcom_task_find(p->pid);
+		if (!list_empty(&mytsk->in_packs))
+			list_for_each_entry_safe(pkt, pkt_next, &mytsk->in_packs, list) {
+				printk("packet found.\n");
+				if ((pkt->type & MIG_MASK) == MIG_SYSCALL) {
+					if ((pkt->type & SYSCALL_MASK) == DEP_SIGNAL) {
+						printk("Signal pkt found.\n");
+						remote_do_signal(p, pkt);
+						list_del(&pkt->list);
+					}
+				}
+			}
 	return 0;
+	
 }
 
 static inline int deputy_pre_usermode(void)
@@ -206,7 +224,6 @@
 static int __init openmosix_init(void)
 {
 	/* kick off the kernel threads: */
-	kernel_thread(openmosix_mig_daemon, NULL, 0);
 
 	return 0;
 }

Index: linux/hpc/remote.c
===================================================================
--- linux.orig/hpc/remote.c	2006-09-15 00:49:59.000000000 +0200
+++ linux/hpc/remote.c	2006-09-15 00:52:26.000000000 +0200
@@ -28,10 +28,12 @@
 #include <hpc/comm.h>
 #include <hpc/task.h>
 #include <hpc/arch.h>
+#include <hpc/kcom.h>
 #include <hpc/prototype.h>
 #include <hpc/service.h>
 #include <hpc/hpc.h>
 #include <hpc/debug.h>
+#include <asm/unistd.h>
 
 
 NORET_TYPE void remote_disappear(void)
@@ -149,19 +151,47 @@
 	return -1;
 }
 
-static int remote_do_signal(task_t *p)
+int remote_do_signal(task_t *p,  struct kcom_pkt *pkt)
 {
 	struct omp_signal s;
 	unsigned long flags;
 	int error;
-
-	error = comm_recv(p->om.contact, &s, sizeof(s));
+	struct kcom_pkt *send_pkt;
+	struct kcom_task *send_tsk;
+	// task_t *kcomd_task;
+
+	printk("FUNCTION: remote_do_signal\n");
+	memcpy(&s, pkt->data, pkt->len);
+	// error = comm_recv(p->om.contact, &s, sizeof(s));
 	printk("received signal %d\n", s.signr);
 
+	
 	spin_lock_irqsave(&p->sighand->siglock, flags);
 	error = __group_send_sig_info(s.signr, &s.siginfo, p);
 	spin_unlock_irqrestore(&p->sighand->siglock, flags);
+	
 
+	send_tsk=kcom_task_find(p->pid);
+	send_pkt=kcom_pkt_create(0, MIG_SYSCALL | PKT_ACK | DEP_SIGNAL | REM_FLG, PKT_ACK, NULL);
+	send_pkt->msgid=pkt->msgid;
+	send_pkt->hpid=send_tsk->hpid;
+	send_pkt->rpid=send_tsk->rpid;
+	send_pkt->resp=pkt->resp;
+
+	list_add_tail(&send_pkt->list, &send_tsk->out_packs);
+
+	#if 0
+	read_lock(&tasklist_lock);
+	kcomd_task=find_task_by_pid(kcom_pid);
+	read_unlock(&tasklist_lock);
+	#endif
+
+	if (kcomd_task) {
+		printk("Signaling kcomd\n");
+		send_sig(SIGHUP, kcomd_task, 0);
+	} else
+		printk("Unable to signal kcomd\n");
+	printk("leaving FUNCTION: remote_do_signal\n");
 	return 0;
 }
 
@@ -179,7 +209,9 @@
 
 	switch (req.type) {
 	case DEP_SIGNAL:
+		#if 0
 		error = remote_do_signal(p);
+		#endif
 		break;
 	case DEP_COMING_HOME:
 		printk("remote_do_comm(): got DEP_COMING_HOME\n");
@@ -207,7 +239,10 @@
 	task_t *p = current;
 	struct omp_syscall_req s;
 	struct omp_syscall_ret r;
-	int error, i;
+	int i;
+	struct sockaddr_in *dest_ptr=(struct sockaddr_in *)p->om.whereto;
+
+	printk("FUNCTION: remote_do_syscall\n");
 
 	OMDEBUG_SYS(1, "[remote] remote syscall %d\n", n);
 
@@ -215,23 +250,29 @@
 	for (i = 0; i < NR_MAX_SYSCALL_ARG; i++)
 		s.arg[i] = arch_get_sys_arg(i, regs);
 
-	error = comm_send_hd(p->om.contact, REM_SYSCALL, &s, sizeof(s));
-	if (error < 0)
-		goto error;
+	printk("syscall: [%d]\n", s.n);
+	kcom_send_with_ack(MIG_SYSCALL | REM_FLG, sizeof(s), (char *)&s, 0, dest_ptr);
+
+	// Set interruptible so we can sleep, but if the remote sends syscall info requests (COPY_TO/FROM_USER, etc), then we have to answer those before sleeping.
+	set_current_state(TASK_INTERRUPTIBLE);
 
 	OMDEBUG_SYS(3, "[remote] waiting deputy answer\n");
 
-	error = remote_handle_user(p, REM_SYSCALL|REPLY);
-	if (error < 0)
+	// if exitting, no need for remote_handle_user
+	if ((n != __NR_exit_group) && (n != __NR_exit))
+		r.ret = remote_handle_user(p, REM_SYSCALL|REPLY);
+	else { // exit 
+		printk("leaving FUNCTION: remote_do_syscall; exit\n");
 		goto error;
+	}
 
-	error = comm_recv(p->om.contact, &r, sizeof(r));
-	if (error < 0)
-		goto error;
 
-	OMDEBUG_SYS(2, "[remote] sys[%d] = %ld\n", n, r.ret);
+	// OMDEBUG_SYS(2, "[remote] sys[%d] = %ld\n", n, r.ret);
+	printk("[remote] sys[%d] = %ld\n", n, r.ret);
 
+	printk("leaving FUNCTION: remote_do_syscall\n");
 	return r.ret;
+
 error:
 	remote_disappear();
 	return -1;

Index: linux/hpc/task.c
===================================================================
--- linux.orig/hpc/task.c	2006-09-18 23:40:22.000000000 +0200
+++ linux/hpc/task.c	2006-09-18 23:52:48.000000000 +0200
@@ -23,8 +23,9 @@
 #include <hpc/task.h>
 #include <hpc/comm.h>
 #include <hpc/mig.h>
-#include <hpc/prototype.h>
 #include <hpc/hpc.h>
+#include <hpc/kcom.h>
+#include <hpc/prototype.h>
 
 /**
  * task_set_comm - swap openMosix link for a process (return old one)
@@ -124,14 +125,26 @@
  **/
 void task_request_move(task_t *p)
 {
-	struct sockaddr *addr;
+	struct sockaddr *dest_ptr=p->om.whereto;
+
+/*
+ * FIXME:
+ *  1) home -> remote - fresh migration; *
+ *  2) home -> remote - redundant migration; *
+ *  3) home -> 'home';
+ *  4) home -> new remote;
+ *  5) remote -> 'home' - fresh migration; *
+ *  6) remote -> home ip address;
+ *  7) remote -> remote ip address;
+ *  8) remote -> new remote - initiated from remote;
+ *  9) remote -> new remote - initiated from home;
+ *  10) remote -> home - bring home initiated from home;
+ */
 
 	task_clear_dreqs(p, DREQ_MOVE);
-	addr = p->om.whereto;
-	p->om.whereto = NULL;
 
-	task_move_to_node(p, addr, 0);
-	kfree(addr);
+	task_move_to_node(p, dest_ptr, 0);
+
 }
 
 /**
@@ -142,7 +155,8 @@
 	task_t *parent = current;
 
 	memset(&p->om, 0, sizeof(om_task_t));
-	
+	printk("leaving FUNCTION: task_local_bring\n");
+
 	if (p->pid == 1)
 		task_set_stay(p, DSTAY_SYSTEM);
 
@@ -154,7 +168,7 @@
 		task_set_dflags(p, DDEPUTY);
 
 	INIT_LIST_HEAD(&p->om.rfiles);
-	
+
 	return 0;
 }
 
@@ -164,14 +178,19 @@
 int openmosix_task_exit(void)
 {
 	task_t *p = current;
-	
+
 	if (!task_test_dflags(p, DDEPUTY | DREMOTE))
 		return 0;
 
+	dump_stack();
+	kcom_task_delete(p->pid);
 	task_heldfiles_clear(p);
-	
+	kfree(p->om.whereto);
+
+	#if 0
 	if (p->om.contact)
-		comm_close(p->om.contact);
+		 comm_close(p->om.contact);
+	#endif
 	return 0;
 }
 
@@ -195,21 +214,15 @@
 /**
  * task_register_migration - register a migration for this process
  * @p:		task to migrate
- * @dest:	destination of the migration (NULL == home)
  **/
-int task_register_migration(task_t *p, struct sockaddr *dest)
+int task_register_migration(task_t *p)
 {
-	if (dest) {
-		p->om.whereto = kmalloc(sizeof(struct sockaddr), GFP_KERNEL);
-		if (!p->om.whereto)
-			return -1;
-		memcpy(p->om.whereto, dest, sizeof(struct sockaddr));
-	}
 	task_set_dreqs(p, DREQ_MOVE);
 	wake_up_process(p);
 	set_ti_thread_flag(p->thread_info, TIF_NEED_RESCHED);
 	return 0;
 }
+EXPORT_SYMBOL_GPL(task_register_migration);
 
 /**
  * task_do_request - current task processes requests coming from other tasks