More patches for dspam.cgi: delivery of false positives from quarantine
"Tod A. Sandman" <[email protected]> Mon, 7 Jan 2013 19:24:00 -0600
| Newsgroups | gmane.mail.spam.dspam.devel |
|---|---|
| Message-ID | <[email protected]> |
--pf9I7BMVVzbSWLtt
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Here is another patch for dspam.cgi that deals with the delivery of false positives from the quarantine. It is a patch on top of the recent quarantine file locking patch I submitted. It allows for specifying in configure.pl an external command for delivering false positives. If this command is not defined, the patch has no effect, other than some optional additional logging.
For us, this is a script that first attempts to retrain/deliver with dspam as normal. But then the script tries to verify if the message was actually delivered, and if not, it will try to deliver the command directly (for instance, with procmail). This external script is quite site specific.
Background:
If the DSPAM signature of a message cannot be found in the database (most often because it has expired and has been cleaned from the database), the dspam command will exit with a successful status even though it does not deliver the message.
Long ago, we had several unhappy professors after they returned from sabbatical and attempted to deliver old messages in their quarantines.
Tod Sandman
Sr. Systems Administrator
Middleware Development & Integration
Rice University
--pf9I7BMVVzbSWLtt
Content-Type: text/plain; charset=us-ascii
Content-Disposition: attachment; filename="patch.deliver_fp"
--- configure.pl.orig 2012-12-10 19:06:22.000000000 -0600
+++ configure.pl 2013-01-07 18:51:33.838892200 -0600
@@ -38,6 +38,11 @@
$CONFIG{'DSPAM_LOGDIR'} = "/var/log/dspam";
$CONFIG{'CGI_LOG'} = "$CONFIG{'DSPAM_LOGDIR'}/cgi.log";
+## Define an optional command used to deliver false positives that have been quarantined
+## (for instance, when the user clicks "Deliver Checked" in the Quarantine Tab of the CGI).
+## If not defined, the DSPAM command will be used directly.
+#$CONFIG{'DELIVER_FP'} = "/usr/site/dspam_train/bin/dspam_fp --user %CURRENT_USER%";
+
# Default DSPAM display
#$CONFIG{'DATE_FORMAT'} = "%d.%m.%Y %H:%M"; # Date format in strftime style
# if undefined use default DSPAM display format
--- dspam.cgi.orig 2012-12-10 19:06:22.000000000 -0600
+++ dspam.cgi 2012-12-11 19:13:52.377436371 -0600
@@ -160,6 +160,7 @@
}
$CONFIG{'DSPAM_ARGS'} =~ s/%CURRENT_USER%/$CURRENT_USER/g;
+($CONFIG{'DELIVER_FP'} =~ s/%CURRENT_USER%/$CURRENT_USER/g) if ($CONFIG{'DELIVER_FP'});
# Current Store
do {
@@ -885,10 +886,16 @@
}
sub ProcessFalsePositive {
- my(@buffer, %head, $found);
+ my(@buffer, $found);
if ($FORM{'signatureID'} eq "") {
&error("$CONFIG{'LANG'}->{$LANGUAGE}->{'error_no_sigid'}");
}
+
+ ## ProcessFalsePositive gets called via the History tab as well as the
+ ## Quarantine tab. MAILBOX may or may not exist. Avoid useless error logs.
+ ## We won't indent for patching's sake.
+ if (-f $MAILBOX) {
+
open(FILE, "<$MAILBOX");
while(<FILE>) {
s/\r?\n$//;
@@ -897,51 +904,28 @@
close(FILE);
while($#buffer>=0) {
- my($buff, $mode, @temp);
- $mode = 0;
+ my($buff, @temp, %head);
@temp = ();
- while(($buff !~ /^From /) && ($#buffer>=0)) {
- $buff = $buffer[0];
- if ($buff =~ /^From /) {
- if ($mode == 0) { $mode = 1; }
- else { next; }
- }
- $buff = shift(@buffer);
- if ($buff !~ /^From /) {
- push(@temp, $buff);
- }
- next;
- }
- foreach(@temp) {
- last if ($_ eq "");
- my($key, $val) = split(/\: ?/, $_, 2);
- $head{$key} = $val;
- }
+ GetNextEmail (\@buffer, \@temp, \%head, 1);
+
if ($head{'X-DSPAM-Signature'} eq $FORM{'signatureID'}) {
$found = 1;
- open(PIPE, "|$CONFIG{'DSPAM'} $CONFIG{'DSPAM_ARGS'} >$TMPFILE 2>&1") || &error($!);
- foreach(@temp) {
- print PIPE "$_\n";
- }
- close(PIPE);
+ my $err = &DeliverFalsePositive (\@temp, \%head);
+ &error ($err) if $err;
+ last; ## No sense wading through the rest.
}
}
+ } ## if (-f $MAILBOX)
# Couldn't find the message, so just retrain on signature
if (!$found) {
+ ## Quarantining is not being used. Or else the user has gone click happy in the History tab.
+ my $signature = $FORM{'signatureID'};
+ Log ("Re-training false positive for user $CURRENT_USER, signature $signature");
system("$CONFIG{'DSPAM'} --source=error --class=innocent --signature=" . quotemeta($FORM{'signatureID'}) . " --user " . quotemeta("$CURRENT_USER"));
+ Log ("FAILURE retraining with signature for user $CURRENT_USER, signature $signature") if ($?);
}
- if ($?) {
- my(@log);
- open(LOG, "<$TMPFILE");
- @log = <LOG>;
- close(LOG);
- unlink("$TMPFILE");
- &error("<PRE>".join('', @log)."</PRE>");
- }
-
- unlink("$TMPFILE");
$FORM{$FORM{'signatureID'}} = "on";
&Quarantine_DeleteSpam();
return;
@@ -964,34 +948,11 @@
open(RETRAIN, ">>$USER.retrain.log");
while($#buffer>=0) {
- my($buff, $mode, @temp, %head, $delivered);
- $mode = 0;
- while(($buff !~ /^From /) && ($#buffer>=0)) {
- $buff = $buffer[0];
- if ($buff =~ /^From /) {
- if ($mode == 0) {
- $mode = 1;
- $buff = shift(@buffer);
- push(@temp, $buff);
- $buff = "";
- next;
- } else {
- next;
- }
- }
- $buff = shift(@buffer);
- push(@temp, $buff);
- next;
- }
- foreach(@temp) {
- last if ($_ eq "");
- my($key, $val) = split(/\: ?/, $_, 2);
- $head{$key} = $val;
- }
+ my($buff, @temp, %head, $delivered);
+ GetNextEmail (\@buffer, \@temp, \%head);
$delivered = 0;
if ($FORM{$head{'X-DSPAM-Signature'}} ne "") {
- my($err);
- $err = &Deliver(@temp);
+ my $err = &DeliverFalsePositive (\@temp, \%head);
if ($err eq "") {
$delivered = 1;
} else {
@@ -1014,16 +975,6 @@
return;
}
-sub Deliver {
- my(@temp) = @_;
- open(PIPE, "|$CONFIG{'DSPAM'} $CONFIG{'DSPAM_ARGS'}") || return $!;
- foreach(@temp) {
- print PIPE "$_\n" || return $!;
- }
- close(PIPE) || return $!;
- return "";
-}
-
sub Quarantine_ViewMessage {
my(@buffer);
@@ -1041,26 +992,9 @@
close(FILE);
while($#buffer>=0) {
- my($buff, $mode, @temp, %head);
- $mode = 0;
+ my($buff, @temp, %head);
@temp = ();
- while(($buff !~ /^From /) && ($#buffer>=0)) {
- $buff = $buffer[0];
- if ($buff =~ /^From /) {
- if ($mode == 0) { $mode = 1; }
- else { next; }
- }
- $buff = shift(@buffer);
- if ($buff !~ /^From /) {
- push(@temp, $buff);
- }
- next;
- }
- foreach(@temp) {
- last if ($_ eq "");
- my($key, $val) = split(/\: ?/, $_, 2);
- $head{$key} = $val;
- }
+ GetNextEmail (\@buffer, \@temp, \%head, 1);
if ($head{'X-DSPAM-Signature'} eq $FORM{'signatureID'}) {
foreach(@temp) {
s/</\<\;/g;
@@ -1115,30 +1049,8 @@
truncate(FILE, 0);
while($#buffer>=0) {
- my($buff, $mode, @temp, %head);
- $mode = 0;
- while(($buff !~ /^From /) && ($#buffer>=0)) {
- $buff = $buffer[0];
- if ($buff =~ /^From /) {
- if ($mode == 0) {
- $mode = 1;
- $buff = shift(@buffer);
- push(@temp, $buff);
- $buff = "";
- next;
- } else {
- next;
- }
- }
- $buff = shift(@buffer);
- push(@temp, $buff);
- next;
- }
- foreach(@temp) {
- last if ($_ eq "");
- my($key, $val) = split(/\: ?/, $_, 2);
- $head{$key} = $val;
- }
+ my($buff, @temp, %head);
+ GetNextEmail (\@buffer, \@temp, \%head);
if ($FORM{$head{'X-DSPAM-Signature'}} eq "") {
foreach(@temp) {
print FILE "$_\n";
@@ -1262,6 +1174,7 @@
$mode = "";
next;
}
+ close (FILE);
my($sortBy) = $FORM{'sortby'};
if ($sortBy eq "") {
@@ -2108,3 +2021,111 @@
close ($fh);
}
+
+###################################################################
+#
+# GetNextEmail pops the next mail message off of the specified
+# mailbox.
+#
+# Replaces replicated code in the these routines:
+# ProcessFalsePositive: GetNextEmail (\@buffer, \@temp, \%head, 1);
+# Quarantine_DeleteSpam: GetNextEmail (\@buffer, \@temp, \%head);
+# Quarantine_ManyNotSpam: GetNextEmail (\@buffer, \@temp, \%head);
+# Quarantine_ViewMessage: GetNextEmail (\@buffer, \@temp, \%head, 1);
+#
+# PARAMETERS
+# mbox: ref to array containing the mail box.
+# email: ref to array where the next mail message is placed.
+# headers: optional ref to header hash.
+# delete_fromline: throw out an initital "From " line if it exists.
+#
+###################################################################
+
+sub GetNextEmail
+{
+ my ($mbox, $email, $headers, $delete_fromline) = @_;
+
+ unless (@$mbox) { return 1; }
+
+ if ($delete_fromline) {
+ shift (@$mbox) if ($$mbox[0] =~ /^From /);
+ }
+ else {
+ push (@$email, shift (@$mbox));
+ }
+
+ while (@$mbox) {
+ last if ($$mbox[0] =~ /^From /);
+ push (@$email, shift (@$mbox));
+ }
+
+ if ($headers) {
+ foreach (@$email) {
+ last if ($_ eq "");
+ my($key, $val) = split(/\: ?/, $_, 2);
+ $headers->{$key} = $val;
+ }
+ }
+ return 1;
+}
+
+
+
+###################################################################
+#
+# Deliver a false positive.
+#
+# PARAMETERS
+# email: ref to array containing the lines of the email.
+# headers: optional ref to headers hash, used only for logging.
+# outfile: optional filename to which the deliver pipe's STDOUT and
+# STDERR is written. A successful DSPAM exit status does not
+# assure the message was delivered. DSPAM outputting something
+# is a good sign that something went wrong.
+#
+###################################################################
+
+sub DeliverFalsePositive
+{
+ my ($email, $headers, $outfile) = @_;
+ my ($syserr, $unlink) = ("", 0);
+ my $signature = "unknown";
+ my @output = ();
+ my $command = $CONFIG{'DELIVER_FP'} ? $CONFIG{'DELIVER_FP'} :
+ "$CONFIG{'DSPAM'} $CONFIG{'DSPAM_ARGS'}";
+
+ unless ($outfile) {
+ $outfile = $TMPFILE;
+ $unlink = 1;
+ }
+
+ $syserr = DeliverEmail ($command, $email, $outfile);
+ unless ($syserr || -s $outfile) { ## Success!
+ unlink $outfile if $unlink;
+ }
+ else {
+ open (DELIVER_OUT, "<$outfile"); @output = <DELIVER_OUT>; close (DELIVER_OUT);
+ ($signature = $headers->{'X-DSPAM-Signature'}) if ($headers && $headers->{'X-DSPAM-Signature'});
+ Log ("FAILURE retraining with signature:", "User: $CURRENT_USER", "Signature: $signature",
+ "Command error: $syserr", "Command output:", @output);
+ }
+ return $syserr;
+}
+
+
+sub DeliverEmail
+{
+ my ($command, $email, $outfile) = @_;
+ my $syserr = "";
+
+ ## Strip initial "From " line. NO!! Let the pipe command decide.
+ #shift (@$email) if ($$email[0] =~ /^From /);
+
+ open(PIPE, "|$command >$outfile 2>&1") || return $!;
+ foreach (@$email) { print PIPE "$_\n" || return $!; }
+ unless (close(PIPE)) {
+ $syserr = $! ? $! : "pipe exit status: $?";
+ }
+ return $syserr;
+}
+
--pf9I7BMVVzbSWLtt
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
------------------------------------------------------------------------------
Master SQL Server Development, Administration, T-SQL, SSAS, SSIS, SSRS
and more. Get SQL Server skills now (including 2012) with LearnDevNow -
200+ hours of step-by-step video tutorials by Microsoft MVPs and experts.
SALE $99.99 this month only - learn more at:
http://p.sf.net/sfu/learnmore_122512
--pf9I7BMVVzbSWLtt
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
_______________________________________________
Dspam-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/dspam-devel
--pf9I7BMVVzbSWLtt--