[Myrinet] bench

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <unistd.h>

#include <sys/types.h>
#include <sched.h>
#include <sys/mman.h>

#include <fcntl.h>
#include <errno.h>


// API Myrinet
# include "mpi.h"

#if defined(__GM_REG__) || defined(__GM_DMA__)
# include "gm.h"
#define GM_EXAMPLE_BOARD_SENDER 0
#define GM_EXAMPLE_BOARD_RECEIVER 1
#define GM_EXAMPLE_PEER_PORT 2
#define GM_EXAMPLE_PRIORITY GM_LOW_PRIORITY
#define GM_EXAMPLE_DEBUG 1
#endif

#define TAG 10
#define ECHANTILLON 	3
#define NSEC_PER_SEC 	1000000000


/*
#define PINFO() printf("lig. %d, file %s\n",__LINE__,__FILE__);
#define PINFO()
*/
#define PINFO()

// TSC
#define CPU_CLOCK 2180,832
#include <asm/msr.h>
typedef unsigned long long int i64;
union
{
  unsigned long long total;
  struct
  {
    unsigned long MSL;
    unsigned long LSL;
  };
}
temps;


typedef struct
{
  double min;
  double max;
  double moy;
}
extrema;

extrema *result_m;
extrema *ptr_extrema;
i64 tab_result_tps[ECHANTILLON + 1];

/************************ functions **************************************/

inline unsigned long long int GetTick ()
{
  rdtsc (temps.MSL, temps.LSL);

  return temps.total;
}

extrema *min_max_moy (i64 * tab, int taille, int diviseur)
{
  double Min, Max, tmp;
  double Moy;
  int i;

  Min = 99999999999;
  Max = 0;
  Moy = 0;
  
  PINFO()
  
  for (i = 0; i < taille; i++)
  {
    tmp = tab[i];
    if (tmp < Min)
      Min = tmp;
    if (tmp > Max)
      Max = tmp;
    Moy += (double) tmp;
  }
	printf("min %f, moy %d, max %d\n", Min, Moy, Max);
#ifdef __DEBUG__
  printf ("diviseur %d", diviseur);
#endif

  if (diviseur == 0)
  {
  	printf("Erreur: division par 0 dans min_max_moy.\n");
	exit(-1);
  }
  tmp = 1.;  
  tmp /= (double)(diviseur);

#ifdef __DEBUG__
  printf (", tmp %f\n", tmp);
#endif
  PINFO()
  result_m->min = (Min) * tmp;
  PINFO()
  result_m->max = (Max) * tmp;
  PINFO()
  result_m->moy = ((Moy) * tmp)/taille;
  PINFO()
  return result_m;
}

void usage (int argc, char *argv[])
{
PINFO()
  printf ("usage: %s type taille_mem nb_envoi [ envoi|reception ]\n", argv[0]);
  printf("\tce programme teste la latence réseau d'un réseau Myrinet, soit avec GM "\
  		"soit avec MPI.\n"\
		"\t type \t1 pour MPI, 2 pour GM.\n"\
		"\t taille_mem taille mémoire du message.\n"\
		"\t nb_envoi \tNombre de test du message.\n"\
		"\t 0|1 = envoi|reception Dans le cas de GM, precise si le programme se met en"\
		"reception ou en envoi de données.\n");
  exit (-1);
}

int test_mpi(int argc,char **argv)
{
  MPI_Status stat;
  int myid,numnodes,next;
  i64 tps_deb, tps_fin, tps_exec;
  int i, j, nb_envoi, size_mem;
  void* message;
  double tmp;
PINFO()
//  printf("arg1 %s, arg2 %s\n",argv[0],argv[1]);

  size_mem = atoi(argv[0]);
  nb_envoi = atoi(argv[1]);
PINFO()
  if (size_mem>0)
  	message = (char *) calloc(size_mem,sizeof(char));
  else
  	message = NULL; size_mem = 0;
PINFO()

  MPI_Init( &argc, &argv );
  MPI_Comm_rank( MPI_COMM_WORLD, &myid);
  MPI_Comm_size( MPI_COMM_WORLD, &numnodes);

PINFO()
  next = myid + 1;
  if (next == numnodes) next = 0;
  PINFO()

  // serveur
  if (myid == 0) 
    {

      for (i=0;i<ECHANTILLON;i++)
        {
  PINFO()

	  tps_deb = GetTick();
	  
	  	for (j=0; j<nb_envoi; j++)
		  {
  PINFO()
		  MPI_Send(message, size_mem , MPI_INT, next, TAG, MPI_COMM_WORLD );
		  MPI_Recv(message, size_mem, MPI_INT, MPI_ANY_SOURCE, TAG, MPI_COMM_WORLD,&stat);
		  }

	  tps_fin = GetTick();
//	  printf("nb_envoi %d\n", nb_envoi);


	  tps_exec = tps_fin - tps_deb;
	  tab_result_tps[i] = tps_exec;
/*  PINFO()
    printf(" 1=> nb_envoi %d\n", nb_envoi);
    tmp = 1./(double)(nb_envoi*2);
    printf(" 2=> nb_envoi %d, tmp %f\n", nb_envoi, tmp);
	printf(" 3=> tps_exec %ll, nb_envoi %d, tmp %f\n",tps_exec, nb_envoi, tmp);
    //printf(" 3=> i=%d, tps_exec %ld, nb_envoi %d, tmp %f, res %ld \n",i,tps_exec, nb_envoi, tmp, tps_exec*tmp);
    printf(" 4=> nb_envoi %d, tmp %f, res %f\n", nb_envoi, tmp,tps_exec*tmp*1000/CPU_CLOCK);
*/
       } 
    
  PINFO()
/*
      ////  Moyennage, tri et formatage des donnees 
    ptr_extrema = min_max_moy (&tab_result_tps[1], (ECHANTILLON - 1),nb_envoi*2);
  PINFO()
    printf("Latence  min : %.2f  moy : %.2f  max : %.2f ns",
       ptr_extrema->min, ptr_extrema->moy, ptr_extrema->max);
*/
  PINFO()
    } 
  else 
    {

	  // client
      for (i=0;i<ECHANTILLON;i++)
        {
  PINFO()

	  MPI_Recv(message, size_mem, MPI_INT, MPI_ANY_SOURCE, TAG, MPI_COMM_WORLD,&stat);
	  MPI_Send(message, size_mem, MPI_INT, next, TAG, MPI_COMM_WORLD );
       }
    }
PINFO()
   free(message);
PINFO()

  MPI_Barrier( MPI_COMM_WORLD );
PINFO()
  MPI_Finalize();
PINFO()
  return 0;

}

#if defined(__GM_REG__) || defined(__GM_DMA__)
int test_gm(int argc,char **argv)
{
  i64 tps_deb, tps_fin, tps_exec;
  int i, j, nb_envoi, size_mem, ret;
  void* message;

  //// Rappel: seul les ports 2,4,5,6,7 sont disponibles pr utilisation courante 
  struct gm_port* port = NULL;
  unsigned int unit = GM_EXAMPLE_BOARD_RECEIVER, 
  		port_id = GM_EXAMPLE_PEER_PORT;
  char* port_name = "gm_myport";
  enum gm_api_version version = GM_API_VERSION;
  enum gm_priority prio = GM_HIGH_PRIORITY;
  unsigned int target_mode_id, target_port_id, local_node_id;
  gm_remote_ptr_t target_buffer;					// A definir pr DMA
  gm_size_t mask;
  char dest_node_name[64];
  int  dest_node_id;
  int action = atoi(argv[2]);
  size_mem = atoi(argv[0]);
  nb_envoi = atoi(argv[1]);
PINFO()    
  if (size_mem>0)
  	message = (char *) calloc(size_mem,sizeof(char));
  else
  	message = NULL; size_mem = 0;
PINFO()
  ////  ---------- Initialisation ---------- 
	gm_init();
	ret = gm_open(&port, unit, port_id, port_name, version);
	if (ret != GM_SUCCESS)
	{
		gm_perror("gm_open",errno);
		exit(-1);
	}

	strcpy(dest_node_name,argv[1]);	
	dest_node_id = gm_host_name_to_node_id(port,dest_node_name);
	printf("[send] Destination node name %s, node id %d\n",
		   dest_node_name, dest_node_id);
	gm_get_node_id(port,&local_node_id);
	
	gm_provide_receive_buffer(port, message, size_mem, prio);
	
	// recommended (but optionnal)
	gm_set_acceptable_sizes( port,GM_HIGH_PRIORITY, mask );
	gm_set_acceptable_sizes( port,GM_LOW_PRIORITY, mask );


	//// init memory
	//message = gm_dma_*alloc();
	// ou
#ifdef __GM_REGISTER__
	message = gm_register_memory(4port, 
				&message,
				(unsigned int)(size_mem*sizeof(char))
				);		// DMA transfer register
#elif defined(__GM_DMA__)
	message = gm_dma_malloc(port, size_mem*sizeof(char));
#else // base alloc 
	message = gm_malloc(size_mem*sizeof(char));
#endif
PINFO()
	// allow DMA transfert
	gm_allow_remote_memory_access(port);
	
	if (action==0)
	{

      for (i=0;i<ECHANTILLON;i++)
        {
	  PINFO()
	  //// Debut du chronometrage 
	  tps_deb = GetTick();

	  	for (j=0; j<nb_envoi; j++)
		  {
			/// send 
#if defined(__GM_REGISTER__) || defined(__GM_DMA__)
								// send in memory
								/// only if gm_allow_remote_memory_access(port);
			gm_directed_send_with_callback( port, message,
					target_buffer,						// remote target buffer
					size_mem,							// number of bytes to send
					prio,
					target_mode_id, target_port_id,
					NULL, NULL							// callback, context
					);			

			// OU
#else
			gm_send( port, message,
					size_mem,						// size of remote buffer
					size_mem,						// number of bytes to send
					prio,
					target_mode_id, target_port_id
					);
			// OU si target_port_id = sending port id
//			gm_send_to_peer_with_callback();
#endif
PINFO()
		  }

	  ///  Fin du chronometrage 
	  tps_fin = GetTick();
	  

	  ///  Stockage du temps de transmission (aller simple) (en micro sec)  
	  tps_exec = tps_fin - tps_deb;
	  tab_result_tps[i] = tps_exec;
       } 
      

      ///  Moyennage, tri et formatage des donnees  
    ptr_extrema = min_max_moy (&tab_result_tps[1], (ECHANTILLON - 1),nb_envoi);

    printf("Latence  min : %.2f  moy : %.2f  max : %.2f ns",
       ptr_extrema->min, ptr_extrema->moy, ptr_extrema->max);

	}
	else
	{
		// receive 
		// gm_receive(...)
		// OU
		// gm_blocking_receive (...)
		// OU
		gm_provide_receive_buffer_with_tag( port, 
				message, 
				size_mem, 
				prio, 0);
		// ou
		// ...
	
	}
	PINFO()
	// Fin
#ifdef __GM_REGISTER__
	gm_deregister_memory(port, 
				&message,
				sizeof(message)
				);
#elif defined(__GM_DMA__)
	gm_dma_free(port, &message);
#else
	gm_free(&message);
#endif

	gm_close(port);
	gm_finalize();

  return 0;

}
#endif

int main(int argc,char **argv) 
{
	struct sched_param sparams;
	int prio = 1, type;
	char** argv2;

PINFO()	
	argv2 = (char**)malloc(10*sizeof(char*));
PINFO()
   if (argc <4 || argc > 5)
   		usage(argc,argv);
	else
	{
PINFO()
		type = atoi(argv[1]);
		argv2[0] = argv[2];		// taille mem
		argv2[1] = argv[3];		// nb envoi
		if (type == 2)
			argv[2] = argv[5];		// si GM, precise si envoi ou reception
	}
PINFO()
/*
	sparams.sched_priority = prio;
	if (sched_setscheduler (0, SCHED_FIFO, &sparams) < 0)
		perror ("sched_setscheduler");
	if ((mlockall (MCL_CURRENT | MCL_FUTURE)) < 0)
	{
		perror ("mlockall");
		exit (-2);
	}
*/
PINFO()
	switch (type)
	{
		case 1:
			test_mpi(2,argv2);
			break;
		case 2:
#if defined(__GM_REGISTER__) || defined(__GM_DMA__)
			test_gm(3,argv2);
#endif
			break;
		default:
			usage(argc,argv);
	}
PINFO()
  // deverrouillage
  munlockall ();
PINFO()
  return 0;
PINFO()
}