Thread profiling

Now and then I make scripts that runs though a filesystem and stats
some or all files in it. This will inherently be IO bound. But it's a
distributed filesystem, so it can handle lots of parallell
operations. Stat runs with the global lock disabled, so in theory
threads should be able to speed this up quite a bit. And they seem to
do, but it's not quite where it should be. I can still speed things up
by using separate processes.

So, do we have any way of profiling where there is contention and how
much is waiting with the lock disabled and how much is spent in Pike?

For your amusement, here's a one-shot script I wrote the other day
that takes a list of files, stats them all and produces a new file
with some stat information for each file. I should probably use
Thread.Farm instead of rolling my own, but I've never taken the time to
figure out how you wait for all async jobs to finish with Thread.Farm.

There is some obvious theoretical limitation with this code: If by chance
a majority of the threads stat the files on the same server backend at
the same time. That is however statistically unlikely to happen on
uniform and unloaded servers that I did this on. But if there was profiling,
we could know for sure. :)

There is also the fact that it uses a common log for all threads on the
Pike layer, something would be insane in general, but writing to the local
filesystem is several magnitudes faster than the remote stats, so
it's not something that should come in to play here. Profiling would say
for sure.


--------8<-----------------------------------------------------

int completed_threads; // Just used for stats
int max_threads = 50;
int active_threads = 0;
int paths_per_thread = 150;

void check_size(array(string) paths)
{
    foreach(paths, string path) {
        Stdio.Stat s = file_stat(path);
        if(!s) {
            // stats failed. File probably went away
            log->write("%s:%s\n",
                       "mis", replace(path, ":", "//"));
        } else {
            if(s->isreg)
                log->write("%s:%s:%d:%d:%d:%d:%d:%d\n",
                           s->type, replace(path, ":", "//"),
                           s->uid, s->gid, s->size, s->nlink,
s->mtime, s->ctime );
            else
                log->write("%s:%s:%d:%d\n",
                           s->type, replace(path, ":", "//"), s->uid, s->gid);
        }
    }

    completed_threads++;
    active_threads--;
}

int num_lines_done; // includes currently running
void print_status(int num_lines)
{
    while(1) {
        sleep(1);
        int done = completed_threads*paths_per_thread;
         werror("\r%d paths/s %d/%d  Est time remaining: %2fh (threads
active: %d/%d)    ",
                done,
                num_lines_done, num_lines,
                (num_lines - num_lines_done)/((float)done+0.00000001)/3600,
                active_threads, max_threads);
        completed_threads = 0;
    }
}

void spawn_checker(array paths)
{
    while(active_threads >= max_threads) {
        sleep(0.0001);
    }
    active_threads++;
    Thread.Thread(check_size, paths);
}

Stdio.File log;
int main(int argc, array argv)
{
    string filelist;
    if(argc > 1)
        filelist = argv[1];
    else
        filelist = "default-list-o-files";

    log = Stdio.File(filelist+".out", "wc");

    int num_lines;
    foreach( Stdio.File(filelist)->line_iterator(); int pathnr; string line )
        num_lines = pathnr;
    Thread.Thread(print_status, num_lines);

    array paths = ({});  // Aggregate a few files before spawning a
checker thread
    foreach( Stdio.File(filelist)->line_iterator(); int pathnr; string path ) {
        paths += ({ path });
        if(sizeof(paths) == paths_per_thread) {
            spawn_checker(paths);
            num_lines_done = pathnr;
            paths = ({});
        }
    }

    // Handle aggregation stragglers
    if(sizeof(paths)) {
        spawn_checker(paths);
        paths = ({});
    }

    while(active_threads)
        sleep(0.001);

    werror("\nDONE\n");
}

--------8<-----------------------------------------------------

Regards,
-- 
Peter Bortas