CPU spikes when migrating proprietary code from AS 2.1 to AS 3.0

Hello All,

We have observed CPU utilization spikes when porting our software from ou=
r current platform (Redhat AS 2.1) to our new development platform (Redha=
t AS 3.0). When running on the AS 2.1 multi-processor boxes, using kernel=
 2.4.9-e.3smp #1 SMP, everything behaves normally. However when running o=
n AS 3.0 multi-processor boxes, using kernel 2.4.21-27.ELsmp #1 SMP or 2.=
4.21-20.ELsmp #1 SMP, we noticed that periodically the CPU utilization wo=
uld spike to the point where one of the processors would be 100% consumed=
. This would occur whether hyper-threading was turned on or off. If we sw=
itch to the non SMP kernel (2.4.21-27.EL #1) there are no CPU spikes.

The attached program demonstrates the problem. It can be run as a daemon =
(default) or non-daemon (cmd opt -no-daemon). It will stat a non existent=
 file 100 times every 10 milliseconds, using select to sleep. (I have als=
o tried using nanosleep with the same results.) When run on the 2.4.21 SM=
P kernels it will quickly begin to accumulate CPU time, caused by these i=
ntermittent spikes as opposed to a steady build-up. When run on the 2.4.9=
 SMP kernel or the 2.4.21 NON - SMP kernel, there are no cpu spikes, and =
no accumulation of CPU time.

My guess is that the problem ultimately has to do with the frequent 10 ms=
 sleeps. However I would like to know why it only occurs with the 2.4.21 =
SMP kernels and not the 2.4.9 or 2.4.21 non smp. I'm not sure if it has a=
nything to do with it or not, however I have noticed that when running on=
 2.4.9 or 2.4.21 NON smp the test app retains a priority of 15. When runn=
ing on the 2.4.21 SMP kernels it ends up at 25.

The test program was compiled with gcc version 3.2.3, using the following=
 command lines:
gcc -o testd_c -x c testd.cpp
g++ -o testd_cpp testd.cpp

The test code is as follows:

#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <unistd.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <time.h>
#include <string.h>

void makeDaemon()
{
	pid_t pid =3D fork();
	if (pid < 0)
	{
		printf("failed to fork1 for Daemon: %d\n", errno);
		exit(-1);=09
	}
	else
	if ( pid > 0)
	{
		exit(0);
	}

	if (setsid() < 0)
	{
		printf("error: cannot disassociate from controlling TTY: %d\n", errno);
		exit(-1);
	}
=09
	pid =3D fork();
	if (pid < 0)
	{
		printf("failed to fork2 for Daemon: %d\n", errno);
		exit(-1);=09
	}
	else
	if ( pid > 0)
	{
		exit(0);
	}
=09
	// pid is zero surely meaning we are the child
	int i =3D open("/dev/null", O_RDWR );
    dup2(i, 0);
    dup2(i, 1);
    dup2(i, 2);
    close(i);
}

int checkFile(const char* fileName)
{
	struct stat tmpStat;
	memset(&tmpStat,0,sizeof(tmpStat));

	if (stat(fileName,&tmpStat) =3D=3D -1)
	{
		return 0;
	}

	return tmpStat.st_size;
}

int main(int argc,char* argv[])
{
	int noDaemon =3D 0;

	if ((argc > 1) && (0 =3D=3D strcmp(argv[1],"--no-daemon")))
	{
		printf("Forgo daemonization.\n");
		noDaemon =3D 1;
	}
	else
	{
		printf("Run as daemon.\n");
		makeDaemon();	=09
	}

	struct timeval tm;
	time_t now 			=3D 0;
	time_t lasttime 	=3D 0;
	time_t startTime 	=3D time(NULL);
	int i =3D 0;
	while(1)
	{
		for (i =3D 0; i < 99; i++)
		{
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
			checkFile("abcdefghikabcdefghik");
		}
	=09
		tm.tv_usec 	=3D 10000;
		tm.tv_sec 	=3D 0;
//		tm.tv_usec 	=3D 0;
//		tm.tv_sec 	=3D 60;
		select(0,NULL,NULL,NULL,&tm);
	=09
		now 			=3D time(NULL);
		if (noDaemon && ((now - lasttime) > 30))
		{
			double accum =3D ( clock() / CLOCKS_PER_SEC);
			printf("Running time: %ld, ", ((now - startTime)));
			printf("Accumulated time: %.02f\n",accum);
			lasttime =3D now;
		}
	}
=09
	return 0;
}


Regards,

Sean Kirkpatrick
Pipeline Trading Systems LLC
Software Engineer
(212) 370 - 8328
[email protected]