RE: EIF probe hanging

#!/usr/bin/perl 
# eif_loadtest.pl - Forking perl script for generating different types of loads on the EIF probe.
# jredus, Sep 2013
# 
# This script will generate (fork) X sub-processes, which will connect Y times to the EIF probe,
# and send Z events per connection, with configurable amounts of delays between connections
# as well as between events within connection. Total events sent = X * Y * Z. A test NAME can 
# be specified for the purpose of counting the number of entries in the eif log (to see if 
# all events made it). The events sent will all be 'HARMLESS' sev, and will all look the same except
# for the msg/Summary (they should all replace each other as duplicates).
# 
# 
# Run script with -h to see usage and default values.
# 
# 
# EX: This will start in 10 secs, fork 75 processes that each connect 2 times with 1 sec (1000000 usecs) 
# between connections, and send 20 events with 1/10 sec (100000 usecs) between events. 
# eif_loadtest.pl -s 10 -f 75 -c 2 -e 20 -m 1000000 -M 100000 -S 2.3.4.5
# 
# 
# EX: This will start in 5 secs, fork 40 processes that each send 0 events, but will sleep 300 secs (300000000 usecs)
# during each connection (this causes high loadavg on our Linux running EIF 11.x)
# eif_loadtest.pl -f 40 -s 5 -S 2.3.4.5 -M 300000000 -e 0
# 
# 

require 5.002;
use Socket;
use Time::HiRes;  # for usleep (microsec sleep)

$Port=9998;
$Server="myserver";
$Count=1;
$Events=1;
$Sequence=0;  # event counter for each child
$Name=time;
$Name = "TEST_$Name";
$Usleep_conn=0;  # microsecond sleep between connections
$Usleep_evt=0;   # microsecond sleep between evets (within connection)

require "getopts.pl";

&Getopts('hN:f:s:S:P:c:e:m:M:');

#############################################################################
my $Usage="

This script will put different types of load on the EIF probe by forking <N>
processes, each of which will connect <count> times to the EIF probe, and send
<events> events per connection, with variable delays between connections and 
between events within the connection. If you choose to send 0 events (-e 0) but
specify a sleep time 'between events' (<usleepe> != 0), then the worker processes
will still connect to EIF, do nothing for <usleepe> usecs, and then disconnect.

Usage:

$0  -f <N> -s <S> -S <server> [-P <port> -c <count> -e <events> -N <name> -m <usleepc> -M <usleepe>]

where
       -f N .......... # of child processes to fork off 
       -s S .......... number of seconds to wait before waking up and running test
       -S server ..... EIF probe server (default:$Server)
       -P Port ....... EIF probe port (default:$Port)
       -c count ...... the count of connections to make (default:$Count)
       -e events ..... the events to send per connection (default:$Events)
       -m usleepc .... sleep this many microseconds between connections (default:$Usleep_conn)
       -M usleepe .... sleep this many microseconds between events within connection (default:$Usleep_evt)
       -N name ....... a name for the test (default: TEST_<epoch-time>)

Example:  $0 -s 10 -f 75 -c 2 -e 20 -m 1000000 -M 100000 -S 2.3.4.5
This example waits 10 secs to start test, forks 75 worker processes which will each connect
twice to EIF, with 1 sec (1000000 usecs) between connections, and sends 20 events each, with 
.1 sec (100000 usecs) between events. The IP to send to is 2.3.4.5 (port=default=9998).


Note: 
100000 usec   = 1/10 sec
500000 usec   = 1/2 sec
1000000 usec  = 1 sec
10000000 usec = 10 sec

";
#############################################################################
# 

if ( $opt_h ) {
    print "$Usage"; exit;
}

if ( ! ($opt_f && $opt_s && $opt_S) ) {
    print "$Usage"; exit;
}

$Server=$opt_S if ($opt_S);
$Port=$opt_P if ($opt_P);
$Count=$opt_c if ($opt_c);
$Events=$opt_e if ($opt_e =~ m/^\d+$/);
$Name=$opt_N if ($opt_N);
$forks = $opt_f;
$secs = $opt_s;
$Usleep_conn = $opt_m if (($opt_m) && ($opt_m =~ m/^\d+$/));
$Usleep_evt = $opt_M if (($opt_M) && ($opt_M =~ m/^\d+$/));

$Count=1 if (!($Count =~ m/^\d+$/));

print "EIF Server=$Server\n";
print "EIF Port=$Port\n";
print "Forks=$opt_f\n";
print "Wakeup in $opt_s secs\n";
print "Sleeping $Usleep_conn microsecs between connections\n";
print "Sleeping $Usleep_evt microsecs between events\n";
print "Connections per worker=$Count\n";
print "Events per connection=$Events\n";
print "Test Name=$Name\n";

# exit;  

$wakeup=time() + $secs;

$Total_events=$forks * $Count * $Events;

if ( ($forks <= 0) || ($forks > 200)) {
    print "$Usage"; 
    print "N should be between 1 and 200\n"; 
    exit;
}

if ( ($secs <= 0) || ($secs > 900)) {
    print "$Usage"; 
    print "S should be between 1 and 900\n"; 
    exit;
}

print "forks=$forks,  secs=$secs.\n";

# alarm($secs); 

$children=0;

$SIG{CHLD} = \&REAPER;        # signal handlers .......
# $SIG{ALRM}='timer'; 

# Parent/Server forking child worker processes:
FORK: for ( $i = 0; $i<$forks; $i++) {
        if ($pid = fork) {          # parent (non-zero pid)
            $children++;
            # print "Parent($$) has $children children\n";
        } elsif (defined $pid) {    # child - but make sure defined
            &child_sub;
            exit;                   # don't forget this!
        } else {
            die("Wierd fork error $!\n")
        } # if
}  # for

# Parent gets here ..
print "Main process ($$) has started $forks sub-processes ...\n";


# Parent waits for start time (had some trouble using alarms for this):
while(1) {

    sleep(1); 
    $now=time;
    if ($now < $wakeup) {
        $diff=$wakeup - $now;
        print "Waiting $diff more seconds to start sending events ...\n";
    }else{
        $t0=time;
        last;
    }
}

# Parent waits for all children to exit:
while (1) {
    # the signal handler will cause the parent to run REAPER func as children terminate
    # until all children are gone.
    $t1=time;
    print "Parent ($$) has $children children left.\n";
    sleep(1);
    if ($children < 1) {
        print "Parent ($$) has no more children left - all done.\n";
        $now=time;
        $diff=$now-$wakeup;
        $diff=1 if ($diff == 0);  # no div by 0
        printf("Test=$Name:  $Total_events test events sent in $diff seconds. Rate=%.2f events/sec\n", ($Total_events/$diff));
        exit(0);
    }
}

###########################################################################
sub logg {
    my($string)=$_[0];

    print "$string\n";
    return;
}
###########################################################################
sub connect_to_EIF {
    my($remote, $port, $iaddr, $paddr, $proto);
    # pass arg1=server, arg2=port

    $remote=$_[0];
    $port=$_[1];

    # print "Connecting to EIF port $port, host $remote ...  \n";

    $iaddr = inet_aton($remote);
    $paddr = sockaddr_in($port, $iaddr);
    $proto = getprotobyname("tcp");

    if (! socket(EIFSOCK, PF_INET, SOCK_STREAM, $proto) ) {
        print "socket error: $!\n";
        return(undef);
    }
    if (! connect(EIFSOCK, $paddr) ) {
        print "connect error: $!\n";
        return(undef);
    }
    # print "Connected to EIF probe.\n";
    return(EIFSOCK);

}
###########################################################################
sub send_events {
    my($evt,$edata,$count,$buff);

    # logg("Checking connection to EIF");

    if (! defined connect_to_EIF($Server,$Port)) {
        logg("Child ($$) Error Connecting to EIF - Unable to send events.");
        exit(1);;
    }

    select EIFSOCK;
    $|=1;           # don't buffer events
    select STDOUT;

    # TEST stuff (create a sustained connection)
    # print "Child PID:$$ sleeping for a while ...\n";
    # sleep(900);
   
    $count=0;
    $buff="";
    for($i=0; $i<$Events; $i++) {
        $Sequence++;
        $evt="Load_test;source=PERL_SCRIPT;msg='Test:$Name,PID:$$,sequence:$Sequence';hostname=ignore_this;severity=HARMLESS;END\n";  # A small event, postemsg fmt.
        $len=length($evt);
        $edata="<START>>" . pack(N7,0,0,0,0,0,$len+1,$len+1) . $evt . pack(c,0x01);
        # $buff .= $edata;
        print EIFSOCK $edata;
        if($Usleep_evt) {
            Time::HiRes::usleep($Usleep_evt);  # sleep between events
        }
    }
    # print EIFSOCK $buff;

    # If we are sending zero events, but want to connect and sleep for a while, 
    # then do it. This way we can compare do-nothing connections between EIF and, for example, SSH. 
    if( $Events == 0) {
        if($Usleep_evt) {
            logg("Child PID:$$ has no events to send, but will sleep $Usleep_evt usecs while connected.");
            Time::HiRes::usleep($Usleep_evt);  
        }
    }

    close(EIFSOCK);

    return if ($Events == 0);  # skip confusing output if no events sent

    # Sleep between connections if we are sending events
    if($Usleep_conn) {
        logg("Child PID:$$ sent $Events more events ($Sequence total) - sleeping $Usleep_conn microseconds.");
        Time::HiRes::usleep($Usleep_conn);
    }else{
        logg("Child PID:$$ sent $Events more events ($Sequence total).");
    }

    return;

}
###########################################################################
sub timer () {

    print "Timer sub\n";
    return;
}
###########################################################################
sub child_sub
{
    # Spawned children processes run this sub.
    my($count, $i);

    print "Child($$) is waiting for start time ...\n";
    while(1) {
    
        sleep(1); 
        $now=time;
        if ($now < $wakeup) {
            $diff=$wakeup - $now;
            # print "Child($$) is waiting $diff more seconds for start time ...\n";
        }else{
            last;
        }
        
    }
    $tot=$Events * $Count;  # total events per worker
    print "Child($$) process sending $tot events ($Count connections, $Events events per connection).\n";  
    for ($i=0; $i<$Count; $i++) {
        send_events;
    }
    exit;

}

###########################################################################
sub REAPER {
    # Parent harvests children as they exit with non-blocking waitpid() 
    my $waitedpid = 0;
    $cnt=0;
    while( ($waitedpid = waitpid(-1, WNOHANG)) > 0) {
        $cnt++;
        $children--;
        $stat=$?>>8;
        # print "Parent($$): ($cnt) Reaped pid $waitedpid (status=$stat), $children left.\n";
    }
    $SIG{CHLD} = \&REAPER;

}