Author: gbechis
Date: Mon Mar 2 07:29:27 2026
New Revision: 1932097
Log:
improve algorithm if training set is unbalanced
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm Mon Mar 2 07:01:06 2026 (r1932096)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm Mon Mar 2 07:29:27 2026 (r1932097)
@@ -44,7 +44,7 @@ use strict;
use warnings;
use re 'taint';
-my $VERSION = 0.2;
+my $VERSION = 0.3;
use AI::FANN qw(:all);
use Storable qw(store retrieve);
@@ -147,7 +147,7 @@ scan, and launches the auto-learn proces
in the same way as during the manual learning.
Value 0 at this option disables the auto-learn process for this plugin.
-=item neuralnetwork_dsn (default: none)
+=item neuralnetwork_dsn (default: none)
The DBI dsn of the database to use.
@@ -265,7 +265,7 @@ SQLite.
default => FANN_TRAIN_RPROP,
code => sub {
my ($self, $key, $value, $line) = @_;
- my %algorithm_map = (
+ my %algorithm_map = (
'FANN_TRAIN_QUICKPROP' => FANN_TRAIN_QUICKPROP,
'FANN_TRAIN_RPROP' => FANN_TRAIN_RPROP,
'FANN_TRAIN_BATCH' => FANN_TRAIN_BATCH,
@@ -646,11 +646,47 @@ sub learn_message {
$network->learning_momentum($momentum);
$network->training_algorithm($train_algorithm);
- # Use multiple-epoch incremental training for each feature vector to increase learning effect
- my $epochs = $train_epochs;
- for my $e (1 .. $epochs) {
+ # Load the current corpus counts so we can compute how skewed the training
+ # history is.
+ my %vocab_for_balance;
+ if (defined $conf->{neuralnetwork_dsn} && $self->{dbh}) {
+ my $vocab_ref = $self->_load_vocabulary_from_sql($self->{main}->{username});
+ %vocab_for_balance = %{$vocab_ref} if ref($vocab_ref) eq 'HASH';
+ }
+ if (!keys %{$vocab_for_balance{terms} || {}}) {
+ my $vocab_path = File::Spec->catfile($nn_data_dir, 'vocabulary-' . lc($self->{main}->{username}) . '.data');
+ if (-f $vocab_path) {
+ eval {
+ my $ref = retrieve($vocab_path);
+ %vocab_for_balance = %{$ref} if ref $ref eq 'HASH';
+ 1;
+ } or do {
+ dbg("Could not load vocabulary for balance check: " . ($@ || 'unknown'));
+ };
+ }
+ }
+ my $spam_docs = $vocab_for_balance{_spam_count} || 1;
+ my $ham_docs = $vocab_for_balance{_ham_count} || 1;
+
+ # class_weight > 1 means this message belongs to the minority class and
+ # should be trained harder; < 1 means it belongs to the majority class.
+ my $class_weight;
+ if ($isspam) {
+ $class_weight = $ham_docs / $spam_docs; # < 1 when spam dominates
+ } else {
+ $class_weight = $spam_docs / $ham_docs; # < 1 when ham dominates
+ }
+ $class_weight = 0.5 if $class_weight < 0.5;
+ $class_weight = 5.0 if $class_weight > 5.0;
+
+ my $weighted_epochs = int($train_epochs * $class_weight) || 1;
+ dbg("Incremental training: weighted_epochs=$weighted_epochs " .
+ "(base=$train_epochs, class_weight=$class_weight, " .
+ "spam_docs=$spam_docs, ham_docs=$ham_docs, isspam=$isspam)");
+
+ for my $e (1 .. $weighted_epochs) {
for my $i (0 .. $#$feature_vectors) {
- my $input = $feature_vectors->[$i];
+ my $input = $feature_vectors->[$i];
my $output = [$labels[$i] ? 1 : 0];
eval { $network->train($input, $output); 1 } or dbg("Training step failed: " . ($@ || 'unknown'));
}
@@ -718,14 +754,14 @@ sub forget_message {
sub check_neuralnetwork_spam {
my ($self, $pms) = @_;
-
+
_check_neuralnetwork($self, $pms);
return $pms->{neuralnetwork_spam};
}
sub check_neuralnetwork_ham {
my ($self, $pms) = @_;
-
+
_check_neuralnetwork($self, $pms);
return $pms->{neuralnetwork_ham};
}
@@ -786,9 +822,9 @@ sub _retrain_from_vocabulary {
return unless defined $nn_data_dir && $vocab_size > 0;
- my $learning_rate = $conf->{neuralnetwork_learning_rate};
- my $momentum = $conf->{neuralnetwork_momentum};
- my $train_epochs = $conf->{neuralnetwork_train_epochs};
+ my $learning_rate = $conf->{neuralnetwork_learning_rate};
+ my $momentum = $conf->{neuralnetwork_momentum};
+ my $train_epochs = $conf->{neuralnetwork_train_epochs};
my $train_algorithm = $conf->{neuralnetwork_train_algorithm};
my %vocabulary;
@@ -817,19 +853,23 @@ sub _retrain_from_vocabulary {
my $actual_size = scalar @vocab_keys;
return unless $actual_size == $vocab_size;
- # Build synthetic spam and ham TF-IDF vectors from vocabulary statistics
- my $N = $vocabulary{_doc_count} || 1;
+ # Build synthetic spam and ham TF-IDF vectors normalised by class
+ # document count.
+ my $N = $vocabulary{_doc_count} || 1;
+ my $spam_docs = $vocabulary{_spam_count} || 1;
+ my $ham_docs = $vocabulary{_ham_count} || 1;
+
my (@spam_vec, @ham_vec);
for my $i (0 .. $#vocab_keys) {
- my $w = $vocab_keys[$i];
+ my $w = $vocab_keys[$i];
my $td = $terms->{$w};
- my $df = $td->{docs} || 0;
- my $idf = log(($N + 1) / ($df + 1)) + 1;
- my $spam_freq = $td->{spam} || 0;
- my $ham_freq = $td->{ham} || 0;
- my $total = $spam_freq + $ham_freq || 1;
- $spam_vec[$i] = ($spam_freq / $total) * $idf;
- $ham_vec[$i] = ($ham_freq / $total) * $idf;
+ my $df = $td->{docs} || 0;
+ my $spam_freq = $td->{spam} || 0;
+ my $ham_freq = $td->{ham} || 0;
+ my $idf = log(($N + 1) / ($df + 1)) + 1;
+
+ $spam_vec[$i] = ($spam_freq / $spam_docs) * $idf;
+ $ham_vec[$i] = ($ham_freq / $ham_docs) * $idf;
}
# L2-normalize both vectors
@@ -840,6 +880,18 @@ sub _retrain_from_vocabulary {
@$vec = map { $_ / $norm } @$vec;
}
+ my $spam_reps = 1;
+ my $ham_reps = 1;
+ if ($spam_docs > $ham_docs) {
+ $ham_reps = int($spam_docs / $ham_docs + 0.5) || 1;
+ $ham_reps = 10 if $ham_reps > 10;
+ } elsif ($ham_docs > $spam_docs) {
+ $spam_reps = int($ham_docs / $spam_docs + 0.5) || 1;
+ $spam_reps = 10 if $spam_reps > 10;
+ }
+ dbg("Retraining from vocabulary: spam_docs=$spam_docs, ham_docs=$ham_docs, " .
+ "spam_reps=$spam_reps, ham_reps=$ham_reps, epochs=$train_epochs");
+
# Create and train new network
my $num_hidden = int(sqrt($vocab_size)) || 1;
my $network = AI::FANN->new_standard($vocab_size, $num_hidden, 1);
@@ -850,8 +902,12 @@ sub _retrain_from_vocabulary {
$network->training_algorithm($train_algorithm);
for my $e (1 .. $train_epochs) {
- eval { $network->train(\@spam_vec, [1]); 1 } or dbg("Retrain spam step failed: " . ($@ || 'unknown'));
- eval { $network->train(\@ham_vec, [0]); 1 } or dbg("Retrain ham step failed: " . ($@ || 'unknown'));
+ for (1 .. $spam_reps) {
+ eval { $network->train(\@spam_vec, [1]); 1 } or dbg("Retrain spam step failed: " . ($@ || 'unknown'));
+ }
+ for (1 .. $ham_reps) {
+ eval { $network->train(\@ham_vec, [0]); 1 } or dbg("Retrain ham step failed: " . ($@ || 'unknown'));
+ }
}
return $network;
@@ -969,7 +1025,7 @@ sub _init_sql_connection {
return if $self->{dbh};
return if !$conf->{neuralnetwork_dsn};
- my $dsn = $conf->{neuralnetwork_dsn};
+ my $dsn = $conf->{neuralnetwork_dsn};
my $username = $conf->{neuralnetwork_username} || '';
my $password = $conf->{neuralnetwork_password} || '';
@@ -1039,7 +1095,7 @@ sub _save_msgid_to_neural_seen {
# Flag: 'S' for spam, 'H' for ham
my $flag = $isspam ? 'S' : 'H';
my $username = lc($self->{main}->{username}) || 'default';
-
+
# Use INSERT IGNORE to avoid duplicate key errors
my $insert_sql;
@@ -1147,9 +1203,9 @@ sub _save_vocabulary_to_sql {
lc($username),
$keyword,
$term_data->{total} || 0,
- $term_data->{docs} || 0,
- $term_data->{spam} || 0,
- $term_data->{ham} || 0
+ $term_data->{docs} || 0,
+ $term_data->{spam} || 0,
+ $term_data->{ham} || 0
);
$count++;
}
@@ -1216,12 +1272,12 @@ sub _load_vocabulary_from_sql {
my ($keyword, $total, $docs, $spam, $ham) = @{$row};
$vocabulary{terms}{$keyword} = {
total => $total,
- docs => $docs,
- spam => $spam,
- ham => $ham
+ docs => $docs,
+ spam => $spam,
+ ham => $ham
};
- $vocabulary{_doc_count}++ if($docs eq 1);
- $vocabulary{_ham_count}++ if($ham eq 1);
+ $vocabulary{_doc_count}++ if($docs eq 1);
+ $vocabulary{_ham_count}++ if($ham eq 1);
$vocabulary{_spam_count}++ if($spam eq 1);
$count++;
}
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.