[Myrinet] hybrid mpich support
"Van Maren, Kevin" <[email protected]>
| Newsgroups | gmane.network.myrinet.general |
|---|---|
| Message-ID | <3FAD1088D4556046AEC48D80B47B478C0101F472@usslc-exch-4.slc.unisys.com> |
[Earlier submission rejected since I wasn't on the list.] Here is a patch, relative to the 1.2.4..8a release, that adds the ability to restrict shared memory on a SMP to only those processes that use the same Myrinet interface. I developed this code to do some experimentation on a NUMA-ish machine, but I thought it may be more generally useful. It is not complete (MPD support isn't runtime- tunable nad is untested), but I would appreciate feedback/ comments on the code. It works for me, but YMMV. Also included is a fix for shared memory (Myricom #13195) to correct the mmap() size calculation, and some changes to eliminate warnings (unsigned i < 0 comparisons). Kevin Van Maren <<hybrid_patch.txt>>
hybrid_patch.txt
(text/plain, 8.4 KB)
diff -u -r virgin-8a/mpid/ch_gm/gmpi.h hybrid-8a/mpid/ch_gm/gmpi.h
--- virgin-8a/mpid/ch_gm/gmpi.h Fri Aug 9 09:08:03 2002
+++ hybrid-8a/mpid/ch_gm/gmpi.h Fri Sep 27 14:08:17 2002
@@ -106,6 +106,7 @@
unsigned int unexpected_short;
unsigned int eager_size;
unsigned int shmem;
+ unsigned int hybrid_shmem;
unsigned int magic;
unsigned int mpd;
struct sockaddr_in master_addr;
diff -u -r virgin-8a/mpid/ch_gm/gmpi_conf.c hybrid-8a/mpid/ch_gm/gmpi_conf.c
--- virgin-8a/mpid/ch_gm/gmpi_conf.c Fri Aug 9 09:08:03 2002
+++ hybrid-8a/mpid/ch_gm/gmpi_conf.c Sun Sep 29 18:43:29 2002
@@ -119,7 +119,7 @@
static void
gmpi_getconf (void)
{
- char *gmpi_eager, *gmpi_shmem, *gmpi_recvmode;
+ char *gmpi_eager, *gmpi_shmem, *gmpi_hybrid_shmem, *gmpi_recvmode;
unsigned int i, j, port_id;
int board_id;
@@ -200,8 +200,19 @@
smpi.num_local_nodes = 0;
for (j = 0; j < MPID_MyWorldSize; j++)
{
- if (strcmp (my_hostname, &(hostnames[j*256])) == 0)
- {
+
+ /* XXX: Note hybrid_shmem is 0 at this point. */
+ printf("MPD doesnot support hybrid intra-node communications\n");
+ /* XXX: Add support to MPD */
+
+ /* Local if same hostname AND (if hybrid) same board number */
+ if ((strncmp (my_hostname, &(hostnames[j*256]), 256) == 0) &&
+ ((gmpi.hybrid_shmem == 0) ||
+ (gmpi.board_ids[j] == gmpi.board_ids[MPID_MyWorldRank])))
+ {
+ printf("node %d, board %d matches node %d\n",
+ MPID_MyWorldRank, gmpi.board_ids[MPID_MyWorldRank], j);
+ fflush(stdout);
if (j == MPID_MyWorldRank)
{
smpi.my_local_id = smpi.num_local_nodes;
@@ -225,6 +236,7 @@
unsigned int count, magic_number, master_port1, master_port2;
int gmpi_sockfd;
struct hostent *master;
+ int rc;
/* mpirun with sockets */
gmpi.mpd = 0;
@@ -235,7 +247,10 @@
gmpi_getenv ("GMPI_ID", &gmpi_id, "the MPI ID of the process", 1);
gmpi_getenv ("GMPI_NP", &gmpi_np, "the number of MPI processes", 1);
gmpi_getenv ("GMPI_BOARD", &gmpi_board, "the specified board", 1);
-
+
+ /* Need this flag to determine how to parse the local info */
+ gmpi_getenv ("GMPI_HYBRID_SHMEM", &gmpi_hybrid_shmem, NULL, 0);
+
if (sscanf (gmpi_magic, "%d", &magic_number) != 1)
{
fprintf (stderr, "<MPICH-GM> Error: Bad magic number "
@@ -282,6 +297,18 @@
gmpi_abort (0);
}
+ /* Check for hybrid flag: only shared memory if same node AND board */
+ if (/* (gmpi.shmem) && */ gmpi_hybrid_shmem &&
+ (strcmp (gmpi_hybrid_shmem, "1") == 0))
+ {
+ gmpi.hybrid_shmem = 1;
+ }
+ else
+ {
+ gmpi.hybrid_shmem = 0;
+ }
+
+
/* data allocation */
gmpi_allocate_world (MPID_MyWorldSize);
@@ -350,14 +377,14 @@
(int) getpid ());
while (count < strlen (buffer))
{
- i = write (gmpi_sockfd, &(buffer[count]), strlen (buffer) - count);
- if (i < 0)
+ rc = write (gmpi_sockfd, &(buffer[count]), strlen (buffer) - count);
+ if (rc < 0)
{
fprintf (stderr, "[%d] Error: write to socket failed !\n",
MPID_MyWorldRank);
gmpi_abort (0);
}
- count += i;
+ count += rc;
}
close (gmpi_sockfd);
@@ -391,14 +418,14 @@
sprintf (buffer, "<->%d:%d<->\n", magic_number, MPID_MyWorldRank);
while (count < strlen (buffer))
{
- i = write (gmpi_sockfd, &(buffer[count]), strlen (buffer) - count);
- if (i < 0)
+ rc = write (gmpi_sockfd, &(buffer[count]), strlen (buffer) - count);
+ if (rc < 0)
{
fprintf (stderr, "[%d] Error: write to socket (2) failed !\n",
MPID_MyWorldRank);
gmpi_abort (0);
}
- count += i;
+ count += rc;
}
/* Get the whole GM mapping from the master */
@@ -406,15 +433,15 @@
gm_bzero (buffer, GMPI_SOCKET_BUFFER_SIZE * sizeof(char));
while (strstr (buffer, "]]]") == NULL)
{
- i = read (gmpi_sockfd, &(buffer[count]),
+ rc = read (gmpi_sockfd, &(buffer[count]),
GMPI_SOCKET_BUFFER_SIZE - count);
- if (i < 0)
+ if (rc < 0)
{
fprintf (stderr, "[%d] Error: read from socket failed !\n",
MPID_MyWorldRank);
gmpi_abort (0);
}
- count += i;
+ count += rc;
}
close (gmpi_sockfd);
@@ -465,14 +492,19 @@
MPID_MyWorldRank);
gmpi_abort (0);
}
-
+
if (i == MPID_MyWorldRank)
{
smpi.my_local_id = smpi.num_local_nodes;
}
- smpi.local_nodes[i] = smpi.num_local_nodes;
- smpi.num_local_nodes++;
-
+
+ /* In hybrid mode they are only really local with same board number */
+ if ((gmpi.hybrid_shmem == 0) ||
+ ((gmpi.board_ids[i] == gmpi.board_ids[MPID_MyWorldRank]))) {
+ smpi.local_nodes[i] = smpi.num_local_nodes;
+ smpi.num_local_nodes++;
+ }
+
sprintf (temp, "<%d>", i);
j += strlen (temp);
}
diff -u -r virgin-8a/mpid/ch_gm/gmpi_smppriv.c hybrid-8a/mpid/ch_gm/gmpi_smppriv.c
--- virgin-8a/mpid/ch_gm/gmpi_smppriv.c Fri Aug 9 09:08:04 2002
+++ hybrid-8a/mpid/ch_gm/gmpi_smppriv.c Sun Sep 29 18:42:48 2002
@@ -1062,8 +1062,15 @@
gmpi_regcache_init();
}
#endif
-
- sprintf (shmem_file , "/tmp/gmpi_shmem-%d.tmp", gmpi.magic);
+
+ /* XXX: This breaks the clean-up code. */
+ /* In Hybrid mode use a separate temporary file for each board */
+ if (gmpi.hybrid_shmem)
+ sprintf (shmem_file , "/tmp/gmpi_shmem-%d-%d.tmp", gmpi.magic,
+ gmpi.board_ids[MPID_MyWorldRank]);
+ else
+ sprintf (shmem_file , "/tmp/gmpi_shmem-%d.tmp", gmpi.magic);
+
#if PRINT_CONFINFO
printf("shmem_file = '%s'\n",shmem_file);
#endif
@@ -1081,7 +1088,8 @@
/* compute the size of this file */
size = (SMPI_CACHE_LINE_SIZE + sizeof(struct shared_mem)
+ (smpi.num_local_nodes * (smpi.num_local_nodes-1)
- * (SMPI_ALIGN(SMPI_LENGTH_QUEUE))));
+ * (SMPI_ALIGN(SMPI_LENGTH_QUEUE+SMPI_CACHE_LINE_SIZE))));
+ // * (SMPI_ALIGN(SMPI_LENGTH_QUEUE))));
/* initialization of the shared memory file */
if (smpi.my_local_id == 0)
@@ -1231,6 +1239,7 @@
+ (smpi.num_local_nodes
* (smpi.num_local_nodes-1)
* (SMPI_LENGTH_QUEUE+SMPI_CACHE_LINE_SIZE))));
+ // * (SMPI_ALIGN(SMPI_LENGTH_QUEUE+SMPI_CACHE_LINE_SIZE))));
close(smpi.fd);
gm_destroy_lookaside(smpi.send_fifo_lookaside);
diff -u -r virgin-8a/mpid/ch_gm/mpirun.ch_gm.pl.in hybrid-8a/mpid/ch_gm/mpirun.ch_gm.pl.in
--- virgin-8a/mpid/ch_gm/mpirun.ch_gm.pl.in Fri Aug 9 09:08:04 2002
+++ hybrid-8a/mpid/ch_gm/mpirun.ch_gm.pl.in Fri Sep 27 14:45:03 2002
@@ -22,6 +22,7 @@
$delay_rexec = 0;
$np = 1;
$use_shmem = 1;
+$use_hybrid_shmem = 0;
$rexec = "@RSHCOMMAND@";
$arch = "@ARCH@";
$varenv = '';
@@ -213,6 +214,7 @@
print (STDERR " -machinefile <file> Specifies a machine file, default is\n");
print (STDERR " $default_machinefile.\n");
print (STDERR " --gm-no-shmem Disable the shared memory support (enabled by default).\n");
+ print (STDERR " --gm-hybrid-shmem Enable hybrid shared memory support (disabled by default).\n");
print (STDERR " --gm-wait <n> Wait <n> seconds between each spawning step.\n");
print (STDERR " --gm-kill <n> Kill all processes <n> seconds after the first exits.\n");
print (STDERR " --gm-eager <n> Specifies the Eager/Rendez-vous protocol threshold size.\n");
@@ -259,6 +261,8 @@
$machine_file = $ARGV[0];
} elsif ($_ eq '--gm-no-shmem') {
$use_shmem = 0;
+ } elsif ($_ eq '--gm-hybrid-shmem') {
+ $use_hybrid_shmem = 1;
} elsif ($_ eq '--gm-wait') {
shift;
usage ("No waiting time specified (--gm-wait) !") unless @ARGV >= 1;
@@ -329,7 +333,9 @@
print ("Dry-run mode enabled (Testing).\n") if $dry_run;
print ("Machines file is $machine_file\n");
print ("Shared memory for intra-nodes coms is enabled.\n") if $use_shmem;
+ print ("Hybrid intra-nodes coms is enabled (if shmem).\n") if $use_hybrid_shmem;
print ("Shared memory for intra-nodes coms is disabled.\n") if !$use_shmem;
+ print ("Hybrid intra-nodes coms is disabled.\n") if !$use_hybrid_shmem;
print ("Delay of $delay_rexec between spanwing steps.\n") if $delay_rexec;
print ("Processes will be killed $kill_time after first exits.\n") if $kill_time;
print ("GM receive mode used: $recv_mode.\n");
@@ -524,6 +530,9 @@
print "Shared memory file: $shmem_file\n\n" if $verbose;
$varenv .= " GMPI_SHMEM=1";
+ if ($use_hybrid_shmem) {
+ $varenv .= " GMPI_HYBRID_SHMEM=1";
+ }
} else {
$varenv .= " GMPI_SHMEM=0";
}