A log homogenization service - Clarify

###########################################################################
# $Id: clarify.conf,v 0.90 
###########################################################################

# You can put comments anywhere you want to.  They are effective for the
# rest of the line.

# this is in the format of <name> = <value>.  Whitespace at the beginning
# and end of the lines is removed.  Whitespace before and after the = sign
# is removed.  Everything is case *insensitive*.

# Yes = True  = On  = 1
# No  = False = Off = 0

Title = "Clarified syslog (filters out "visual noise" from log events)"

# Which logfile group...
LogFile = messages
LogFile = secure
LogFile = maillog
LogFile = cron

$term_limits = "kernel!audit:0 audit:10 dbus:4 avahi:1 dbus:1 ntpd:0 sendmail:6 userhelper:4"
$ignore_list = "krb eth0 sda sdb sdc sdd i386 i686"
$sort_by_count=0
$display_random_terms=1

#*RemoveHeaders

########################################################
# This was written and is maintained by:
#    Kirk Bauer <[email protected]>
#
# Please send all comments, suggestions, bug reports,
#    etc, to [email protected].
########################################################

# vi: shiftwidth=3 tabstop=3 et

#!/usr/bin/perl

# clarify.pl
#
# 3/1/07 [email protected]
# 
#  The Clarify.pl Perl script removes the "visual noise" from logs by first replacing stand-alone numbers with
#  the "#" token. Next, it replaces randomly generated terms with ellipses "..." Filtering reduces many log events
#  to a few making it possible for system administrators to systematically and reliably review their logs.
#
#  Note: randomly generated terms are identified  by splitting all log events into separate terms. Terms that include
#  puncutation and alphabet-only strings are ignored. The left-over terms only include alphanumeric strings, which are
#  typically randomly generated terms. Clarify then searches for the alphanumeric strings and replaces them with the
#  ellipsis. Filtering maps similar events into into single ones. The filtered events are displayed along with the count
#  (the number of mapped events). The filtered events are sorted by message by default but can be sorted by count or by
#  reverse count.
#
#  Clarify provides an optional pre-filter process designed to reduce especially noisy events created by kernel,
#  audit and sendmail events. The filter removes all but the first N terms from the specified log events before
#  passing them on to the stage 1 and 2 filters. Setting the $pre_filter variable to 1 enables the pre-filter and
#  sets N=1, which removes all but the first term from cron, kernel and sendmail events; setting $pre_filter to 2
#  sets N=2, etc. By default $prefilter is set to 0 and is not envoked.

#  This script can be used in a stand-alone fasion bay piping log file(s) to it. For instance,
#      cat /var/log/messages | ./clarify
#   or, with command-line options,
#      cat /var/log/messages | ./clarify -r -s 'kernel!audit:0 audit:4 sendmail:6'
#
#  Alternatively, the clarify script can be used as a logwatch module by placing it in /usr/share/logwatch/scripts/services
#  (to use with logwatch you also need to place the companion logjamm.conf file in
#  /usr/share/logwatch/default.conf/services/clarify.conf; you don't need the conf file for stand-alone mode.)


use warnings;
use strict;

our $debug=0;

# get operation variables from /usr/share/logwatch/default.conf/services/clarify.conf
my $term_limits = $ENV{'term_limits'} || '';
my $ignore_list = $ENV{'ignore_list'} || '';
my $sort_key = $ENV{'sort_by_count'} || 0;
my $display_random_terms = $ENV{'display_random_terms'} || 0;

# command-line options supercede clarify.conf
($sort_key,$display_random_terms,$term_limits)=get_args($sort_key,$display_random_terms,$term_limits);

my ($raw_count,$raw_logs)=get_raw_messages();

$raw_logs=pre_filter($term_limits,$raw_logs) if (length($term_limits)>0);

my $filtered_logs=first_filter($raw_logs);

($filtered_logs,my $random_terms)=second_filter($debug,$filtered_logs,$ignore_list);

display_output($sort_key,$filtered_logs,$random_terms,$raw_count,$display_random_terms);

exit(0);


################ primary subroutines #######################3

sub get_args {
   my ($sort_key)=shift;
   my ($display_random_terms)=shift;
   my ($term_limits)=shift;


   while ($#ARGV >= 0) {
      if ( $ARGV[0] eq '-d' ) {
         $debug=1;
         shift @ARGV;
      } elsif 
         ( $ARGV[0] eq '-s' ) {
         $sort_key=1;
         shift @ARGV;
      } elsif
         ( $ARGV[0] eq '-r' ) {
         $display_random_terms=1;
         shift @ARGV;
      } elsif  
         ($ARGV[0] eq '-h' || $ARGV[0] eq '--help') {
            print "Usage: $0 [-d] [-s] [-r] ['term-limits[!term-exclusions]N']\n";
            print "  -d = debug\n";
            print "  -s = numeric sort\n";
            print "  -r = display random (alphanumeric) exclusion terms\n";
            print "  'term limits...' \n";
            print "  term limit example: 'kernel!audit:1 audit:6 sendmail:8'\n";
            exit;
      } else {
         $term_limits= shift @ARGV;
      }
   }

   return $sort_key,$display_random_terms,$term_limits;
}


#
# Read raw log messages from standard input
# 
sub get_raw_messages {
   my @raw;
   my $i=0;
   while (defined(my $thisline = <STDIN>)) {
         chomp($thisline);

         # consolidate white space
         $thisline =~ s/\s+/ /g; 

         # strip date, time & machine name
         my @raw_terms = split /\s/,$thisline,5;
         $raw[$i++] = $raw_terms[4];
   }
   return $#raw+1,\@raw;
}

#
#  Reduce the terms of "chatty" event types to a set number.
#  Some event types, like from the kernel, don't filter well.
#  By only allowing the first N terms, these types of events 
#  can be reduced in number.
#
sub pre_filter {
   my ($term_limits)=shift;
   my ($events)=shift;

   my @out;
   my $event;
   my @terms;
   foreach $event (@$events) {
      LOOP: foreach (split / /,$term_limits) {
         my ($prefilter,$exception,$count)=get_prefilter($_);
         if ($event =~ $prefilter) {
            if (length($exception)>0) {
               next LOOP if ($event =~ $exception);
            }
            @terms=split / /,$event;
            foreach (0..($#terms > $count+1 ? $#terms-$count-1 : 0)) {
               pop(@terms);
            }
            $event=join(" ",@terms);
         }
      }
      push(@out,$event);
   }
   return \@out;
}

#
# First filter replaces stand-alone numbers, prefixed by a non-alphanumeric character,
#  with the "#" symbol. This filter alone greatly reduces the number of raw log events
#  without removing important, and potentially, interesting information.
#
sub first_filter {
   # remove all stand-alone numbers from events
   # stand-alone numbers are numeric-only strings prefixed 
   # by a non-alphanumeric character

   my ($rawlogs)=shift;
   my @out;
   foreach (@$rawlogs) {
      push(@out,replace_numbers_with_token($_));
   }
   return \@out;
}

#
#  The second filter identifies and replaces alphanumeric strings with the ellipses (...). 
#   Alphanumeric strings are generally random process identifiers (for instance, a message id
#   generated by sendmail) and create a great deal of clutter or "noise". Replacing alphanumeric
#   strings further reduces the number raw log events without removing interesting information.
#
sub second_filter {
   my ($debug)=shift;
   my ($events)=shift;
   my ($ignore_list)=shift;

   my %filtered_logs;
   my %random_terms;
   my $i=0;
   foreach my $event (@$events) {
      my @terms=split /[^0-9a-zA-Z]/,$event;
      foreach my $term (@terms) {
         if (!skip_term($term,$ignore_list)) {
            $event =~ s/$term/.../;
            $random_terms{$term}++;
         }
      }
      $filtered_logs{$event}++;
      $i++;
      print "[second]count=$i\n" if ($i % 10000 == 0);

   }

   return \%filtered_logs,\%random_terms;
}


sub display_output {
   my ($sort_by_count)=shift;
   my ($filtered_logs)=shift;
   my ($uniq_terms)=shift;
   my ($raw_count)=shift;
   my ($display_random_terms)=shift;

   my %umsgs=%$filtered_logs;
   my @temp=keys %umsgs;
   my $count=$#temp+1;

   # display header info
   print "# raw events: $raw_count; # filtered: $count\n";
   print "(Tokens: '#' = numbers, '...' = random terms)\n\n";

   # display messages sorted by count or message
   if ($sort_by_count) {
      foreach (sort {$umsgs{$a} <=> $umsgs{$b} } keys %umsgs) {
         printf "%-6s%s\n",$umsgs{$_},$_;
      }
   } else {
      foreach (sort keys %umsgs) {
         printf "%-6s%s\n",$umsgs{$_},$_;
      }
   }

   if ($display_random_terms) {
      print "\nRandom terms (mostly application-generated alphanumeric identifiers like the message ids generated\n";
      print "  by sendmail, but also catches user, host, device and protocols names. You can designate non-random\n";
      print "  terms using the ignore_terms variable in clarify.conf; for instance, set ignore_terms = 'ssh i386 eth0'.)\n";
      foreach my $uniq_term (sort {length($a) <=> length($b)} keys %$uniq_terms) {
         print "$uniq_term\n";
      }
   }

   return;
}


################ support functions ###################################

#
# Return 1 (skip) for alphabetic strings (words) and 0 for alphanumeric strings (random pids).
# The calling routine will skip words and record random pids for later identification 
# and replacement. Terms included in the white list, if the file clarify.wl exists, are
# not processed. 
#
sub skip_term {
   my ($term)=shift;
   my ($ignore_list)=shift;

   # skip if term is empty
   return 1 if (length($term)<=0);

   # skip if term found in whitelist
   foreach (split / /,$ignore_list) {
      return 1 if ($term =~ $_);
   }

   # skip if term does not include a number
   return 1 if ($term !~ /[0-9]+/);

   # skip only if term is alphanumeric
   if ($term =~ /[0-9]+/) {
      if ($term =~ /[a-zA-Z]+/) {
         return 0;
      } else {
         return 1;
      }
      return 1;
   }

   return 0;
}

#
# Identify stand-alone numbers, prefixed by non-alphanumeric character
#  and replace with the # token. Replace MAC addresses with "-MAC-" token.
#
sub replace_numbers_with_token {
   my $msg=shift;

   # tokenize MAC addresses 
   $msg =~ s/([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])/-MAC-/g;

   # tokenize all stand-alone numbers (bracketed by non-alphanumeric characters)
   #     [ ( { ' " - = , . : and white space
   # (this process removes the "visual noise" in an event)
   #$msg =~ s/([^0-9a-zA-Z])[0-9]+([^0-9a-zA-Z])/$1#$2/g;
   $msg =~ s/([^0-9a-zA-Z])[0-9]+/$1#/g;

   return $msg;
}

sub get_prefilter {
   my ($terms)=shift;

   my @prefilter;
   my $exception='';
   my $count;
   if ($terms =~ /!/) {
      @prefilter=(split /!/,$terms);
      $exception=(split /:/,$prefilter[1])[0];
      $count=(split /:/,$prefilter[1])[1];
   } else {
      @prefilter=split /:/,$terms;
      $count=$prefilter[1];
   }

   return $prefilter[0],$exception,$count;
}
      

# vi: shiftwidth=3 tabstop=3 syntax=perl et