Re: Train Spamassassin

#!/usr/bin/php
<?php
/** Konfiguration */
$imap_host     = '192.168.196.1';
$imap_user     = 'spamfilter';
$imap_pwd      = '****';
$imap_spam     = 'TRAIN-SPAM';
$imap_ham      = 'TRAIN-HAM';
$spamd_socket  = '/run/spamd-debug/spamassassin.sock';
$corpus_folder = '/var/lib/spamass-milter/training';
$train_script  = '/usr/bin/bash /scripts/sa-learn.sh';
$stats_script  = '/usr/bin/php /scripts/spamfilter/corpus-stats.php today';
$imap_delete   = true;
$training      = false;

/** Sicherstellen dass Extension geladen ist */
if(!function_exists('imap_fetch_overview'))
{
 exit("ERROR: Extension 'php-imap' nicht geladen\n");
}

/** Wir merken uns Hashes duchgelaufener Dateien um keine 1:1 Duplikate abzulegen */
$hash_list = [];

/**
 * SPAM-TRAINING
 *
 * IMAP-Verbindung aufbauen
 * Nachrichten abfragen
 * RAW-Messages in Temp-Files schreiben
 * Check gegen SpamAssassin
 * Wenn nicht mindestens BAYES_99 in Trainingsfolder verschieben
 * Wenn neue Samples im Trainings-Folder unser zentrales Learning-Skript aufrufen
*/
$new_spam_samples = [];
$result_types = ['BAYES_00', 'BAYES_05', 'BAYES_20', 'BAYES_40', 'BAYES_50', 'BAYES_60', 'BAYES_80', 'BAYES_95', 'BAYES_99', 'BAYES_999'];
$imap_connection = imap_open('{' . $imap_host . ':143/imap/novalidate-cert}' . $imap_spam, $imap_user, $imap_pwd, OP_SILENT, 5);
if($imap_connection)
{
 $imap_info = @imap_check($imap_connection);
 if($imap_info->Nmsgs > 0)
 {
  foreach(@imap_fetch_overview($imap_connection,"1:{$imap_info->Nmsgs}",0) as $msg)
  {
   if(!$msg->deleted)
   {
    $raw_headers = @imap_fetchheader($imap_connection, $msg->msgno);
    $raw_body    = @imap_body($imap_connection, $msg->msgno);
    if(!empty($raw_headers) && !empty($raw_body))
    {
     $raw_message = cleanup_sample(cleanup_mozilla_removed_attachments($raw_headers . $raw_body));
     unset($raw_headers, $raw_body);
     $tmp_file = '/tmp/' . strftime('%Y-%m-%d-%H-%M-%S') . '-' . hash('MD4', $raw_message) . '.eml';
     file_put_contents($tmp_file, $raw_message);
     $spamd_result = spamc_report(/**$socket_path*/$spamd_socket, /**$eml_path*/$tmp_file, /**$user*/'sa-milt');
     $bayes = '';
     foreach($result_types as $result_type)
     {
      if(strpos($spamd_result, ' ' . $result_type . ' ') !== false)
      {
       $bayes = $result_type;
      }
     }
     /** Bereits zweifelsfrei erkannte Samples ignorieren */
     if($bayes == 'BAYES_99' || $bayes == 'BAYES_999')
     {
      echo 'SPAM-IGNORE: ' .  str_replace('BAYES_999', 'BAYES_99', $bayes) . ' - ' . $tmp_file . "\n";
     }
     else
     {
      /** Sicherstellen dass zumindest in den aktuellen Samples keine echten Doubletten sind */
      $current_hash = hash_file('MD4', $tmp_file);
      if(!isset($hash_list[$current_hash]))
      {
       /** Terminal-Ausgabe */
       echo 'SPAM-TRAIN:  ' .  $bayes . ' - ' . $tmp_file . "\n";
       $current_basename = basename($tmp_file);
       /** Sample in Trainings-Ordner verschieben */
       rename($tmp_file, $corpus_folder . '/spam/' . $current_basename);
       $new_spam_samples[] = $corpus_folder . '/spam/' . $current_basename;
       $training = true;
       $hash_list[$current_hash] = 1;
      }
      /** Doublette loeschen */
      else
      {
       @unlink($tmp_file);
       echo 'SPAM-IGNORE: DUPLICATE - ' . $tmp_file . "\n";
      }
     }
    }
    /** Sample vom IMAP-Server loeschen */
    if($imap_delete)
    {
     @imap_delete($imap_connection, $msg->msgno);
    }
    @unlink($tmp_file);
   }
  }
  /** Sample vom IMAP-Server loeschen */
  if($imap_delete)
  {
   @imap_expunge($imap_connection);
  }
 }
 imap_close($imap_connection);
}

/**
 * HAM-TRAINING
 *
 * IMAP-Verbindung aufbauen
 * Nachrichten abfragen
 * RAW-Messages in Temp-Files schreiben
 * Wenn neue Samples im Trainings-Folder unser zentrales Learning-Skript aufrufen
 * Bei Ham-Samples trainieren wir ohne auf den aktuellen Bayes-Wert zu schauen, rein informell
*/
$new_ham_samples = [];
$imap_connection = imap_open('{' . $imap_host . ':143/imap/novalidate-cert}' . $imap_ham, $imap_user, $imap_pwd, OP_SILENT, 5);
if($imap_connection)
{
 $imap_info = @imap_check($imap_connection);
 if($imap_info->Nmsgs > 0)
 {
  foreach(@imap_fetch_overview($imap_connection,"1:{$imap_info->Nmsgs}",0) as $msg)
  {
   if(!$msg->deleted)
   {
    $raw_headers = @imap_fetchheader($imap_connection, $msg->msgno);
    $raw_body    = @imap_body($imap_connection, $msg->msgno);
    if(!empty($raw_headers) && !empty($raw_body))
    {
     $raw_message = cleanup_sample(cleanup_mozilla_removed_attachments($raw_headers . $raw_body));
     unset($raw_headers, $raw_body);
     $tmp_file = '/tmp/' . strftime('%Y-%m-%d-%H-%M-%S') . '-' . hash('MD4', $raw_message) . '.eml';
     file_put_contents($tmp_file, $raw_message);
     $spamd_result = spamc_report(/**$socket_path*/$spamd_socket, /**$eml_path*/$tmp_file, /**$user*/'sa-milt');
     $bayes = '';
     foreach($result_types as $result_type)
     {
      if(strpos($spamd_result, ' ' . $result_type . ' ') !== false)
      {
       $bayes = $result_type;
       break;
      }
     }
     $current_hash = hash_file('MD4', $tmp_file);
     if(!isset($hash_list[$current_hash]))
     {
      /** Terminal-Ausgabe */
      echo 'HAM-TRAIN:   ' .  $bayes . ' - ' . $tmp_file . "\n";
      $current_basename = basename($tmp_file);
      /** Sample in Trainings-Ordner verschieben */
      rename($tmp_file, $corpus_folder . '/ham/' . $current_basename);
      $new_ham_samples[] = $corpus_folder . '/ham/' . $current_basename;
      $training = true;
      $hash_list[$current_hash] = 1;
     }
     else
     {
      @unlink($tmp_file);
      echo 'HAM-IGNORE:  DUPLICATE - ' . $tmp_file . "\n";
     }
    }
    if($imap_delete)
    {
     @imap_delete($imap_connection, $msg->msgno);
    }
    @unlink($tmp_file);
   }
  }
  if($imap_delete)
  {
   @imap_expunge($imap_connection);
  }
 }
 imap_close($imap_connection);
}

/**
 * Unser globales Trainings-Skript am Ende aufrufen
 * Dieses ist auch fuer Permissions, Clenaups, Header-Stripping etc. verantwortlich
 * Wir spielen hier einfach nur Samples aus der Mailbox ein
 *
 * In den Arrays $new_spam_samples und $new_ham_samples liegen an dieser
 * Stelle Listen mit Dateinamen die wir am Ende gerne als non-root
 * auch durch 'bogofilter' jagen wurden nach dem das SA-Training
 * abgeschlossen und im Vorfeld Permissions und Cleanups sichergestellt
 * wurden
*/
if($training)
{
 echo "\n";
 passthru($train_script);
 if(!empty($stats_script))
 {
  echo "\n";
  echo "------------------------------------------------------------------------------\n\n";
  echo "BAYES-TEST AUF TAGESAKTUELLE SAMPLES:\n\n";
  passthru($stats_script);
  /** Bogofilter am Ende ebenfalls trainieren - Drop privileges */
  $sa_milt_user  = 189;
  $sa_milt_group = 188;
  if(!posix_initgroups('sa-milt', $sa_milt_group) || !posix_setgid($sa_milt_group) || !posix_setuid($sa_milt_user))
  {
   exit('BOGOFILTER - DROP PRIVILEGES FAILED' . "\n");
  }
  if(!empty($new_spam_samples))
  {
   passthru('/usr/bin/bogofilter -s -B ' . implode(' ', $new_spam_samples));
  }
  if(!empty($new_ham_samples))
  {
   passthru('/usr/bin/bogofilter -n -B ' . implode(' ', $new_ham_samples));
  }
 }
}

/**
 * Wenn per Thunderbird Attachments geloescht werden diese idiotischen
 * Pseudo-Header entfernen so dass am Ende die originale Mime-Struktur
 * einem leeren Content bestehen bleibt und die Attachments in
 * einem Mail-Client mit den korrekten Namen aber 0 Bytes
 * dargestellt werden
 *
 * @param  string $content
 * @return string
 * @access public
*/
function cleanup_mozilla_removed_attachments($content)
{
 /** Suche nach Vorkommen */
 $start_string = 'Content-Type: text/x-moz-deleted';
 $end_string   = 'Content-Type: ';
 $junk_hits = [];
 $offset = 0;
 $total_count = substr_count($content, $start_string);
 for($count=1; $count<=$total_count; $count++)
 {
  $start = strpos($content, $start_string, $offset);
  if($start !== false && $start > 1)
  {
   $end = stripos($content, $end_string, ($start+strlen($start_string)));
   if($end > $start)
   {
    $junk = trim(substr($content, $start, ($end-$start)));
    if(!empty($junk))
    {
     $junk_hits[] = $junk;
    }
   }
   $offset = $end + strlen($end_string);
  }
 }
 /** Entfernen */
 if(!empty($junk_hits))
 {
  foreach($junk_hits as $junk)
  {
   $content = str_replace($junk, '', $content);
  }
  $content = str_replace("\n\nContent-Type:", "\nContent-Type:", $content);
  $content = str_replace("\r\n\r\nContent-Type:", "\r\nContent-Type:", $content);
 }
 /** Ggf. bereinigte Version zurueck geben */
 return $content;
}

/**
 * Mail-Sample so aufraumen dass die Header-Anordnung weitgehend dem Original-Empfang
 * entspricht nachdem das meiste das wir importieren aus Milter-Rejects stammt
 * und in weiten Bereichen auch Subject-Tags enthaelt
 *
 * Die internen Received-Header veraendern oft das Bayes-Scoring und wuerde zwar beim Training
 * vorher Cleanup/Procmail-Scripts durchlaufen, wir wollen hier allerding bereits eine saubere
 * Entscheidung BAYES_99-Samples zu ignorieren und gar nicht erst zu trainieren
 *
 * @param  string $content
 * @return string
 * @access public
*/
function cleanup_sample($content)
{
 $content = str_replace('Subject: [SPAM] ', 'Subject: ', $content);
 $content = str_replace('Subject: [SPAM]', 'Subject: ', $content);
 $content = str_replace('esx1.thelounge.net', 'example.com', $content);
 $local_header_start = strpos($content, 'Received: by mail-gw.thelounge.net (THELOUNGE GATEWAY, from userid 189)');
 if($local_header_start > 0)
 {
  $local_header_stop = strpos($content, 'Received:', ($local_header_start+1));
  if($local_header_stop > $local_header_start)
  {
   $content = substr($content, $local_header_stop);
  }
 }
 $local_header_start = strpos($content, 'Received: from mail-gw.thelounge.net (mail-gw.thelounge.net [10.0.0.19])');
 if($local_header_start !== false)
 {
  $local_header_stop = strpos($content, 'X-AT-Envelope-From', ($local_header_start+1));
  if($local_header_stop > $local_header_start)
  {
   $content = substr($content, $local_header_stop);
  }
 }
 $local_header_start = strpos($content, 'Received: from caladan.thelounge.net (caladan.thelounge.net [10.0.0.17])');
 if($local_header_start !== false)
 {
  $local_header_stop = strpos($content, 'X-AT-Envelope-From', ($local_header_start+1));
  if($local_header_stop > $local_header_start)
  {
   $content = substr($content, $local_header_stop);
  }
 }
 return trim($content) . "\n";
}

/**
 * Scan RAW-Message with spamd
 *
 * @param  string $socket_path spamd-socket
 * @param  string $eml_path    sample-file
 * @param  string $user        client-uid
 * @return string
 * @access public
*/
function spamc_report($socket_path, $eml_path, $user='sa-milt')
{
 $socket = false;
 while(!$socket)
 {
  $socket = @fsockopen('unix://' . $socket_path);
 }
 $raw_eml = trim(file_get_contents($eml_path)) . "\r\n";
 fwrite($socket, 'REPORT SPAMC/1.5' . "\r\n");
 fwrite($socket, 'Content-length: ' . strlen($raw_eml) . "\r\n");
 fwrite($socket, 'User: ' . $user . "\r\n");
 fwrite($socket, "\r\n");
 fwrite($socket, $raw_eml);
 fwrite($socket, "\r\n");
 $response = '';
 while(true)
 {
  $buffer = fgets($socket, 128);
  $response .= $buffer;
  if(feof($socket))
  {
   break;
  }
 }
 fclose($socket);
 return substr($response, strpos($response, 'Content analysis details'));
}