Author: gbechis
Date: Sun Mar 8 08:39:06 2026
New Revision: 1932217
Log:
update to latest version
- Run TF-IDF transform only when needed
- change sql schema to fix retraining
- Improve locking and SQL queries
Added:
spamassassin/trunk/sql/neural_sqlite.sql
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
spamassassin/trunk/sql/neural_mysql.sql
spamassassin/trunk/sql/neural_pg.sql
spamassassin/trunk/t/neuralnetwork.t
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm Sun Mar 8 08:30:18 2026 (r1932216)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm Sun Mar 8 08:39:06 2026 (r1932217)
@@ -44,10 +44,11 @@ use strict;
use warnings;
use re 'taint';
-my $VERSION = 0.3;
+my $VERSION = 0.5.1;
use AI::FANN qw(:all);
use Storable qw(store retrieve);
+use Fcntl qw(:flock);
use File::Spec;
use Mail::SpamAssassin;
@@ -349,6 +350,7 @@ sub finish_parsing_end {
}
my $dataset_path = File::Spec->catfile($nn_data_dir, 'fann-' . lc($self->{main}->{username}) . '.model');
+ $dataset_path = Mail::SpamAssassin::Util::untaint_file_path($dataset_path);
if (-f $dataset_path) {
eval {
$self->{neural_model} = AI::FANN->new_from_file($dataset_path);
@@ -364,7 +366,7 @@ sub finish_parsing_end {
# Converts a list of raw text strings into a list of
# numerical feature vectors (dense arrays), suitable for Neural Networks training.
sub _text_to_features {
- my ($self, $conf, $nn_data_dir, $train, $label, @emails) = @_;
+ my ($self, $conf, $nn_data_dir, $train, $label, $target_vocab_ref, @emails) = @_;
my $min_word_len = $conf->{neuralnetwork_min_word_len};
my $max_word_len = $conf->{neuralnetwork_max_word_len};
@@ -501,11 +503,13 @@ sub _text_to_features {
}
}
- # Build vocabulary index (stable sorted order)
- my @vocab_keys = sort keys %{ $vocabulary{terms} };
+ # Build vocabulary index
+ my @vocab_keys = ($target_vocab_ref && @$target_vocab_ref)
+ ? @$target_vocab_ref
+ : sort keys %{ $vocabulary{terms} };
my %vocab_index = map { $vocab_keys[$_] => $_ } 0..$#vocab_keys;
my $vocab_size = scalar @vocab_keys;
- return ([], 0) unless $vocab_size > 0;
+ return ([], 0, []) unless $vocab_size > 0;
# Precompute IDF: log((N+1)/(df+1)) + 1 smoothing
my $N = $vocabulary{_doc_count} || 1;
@@ -541,7 +545,7 @@ sub _text_to_features {
push @feature_vectors, { vec => \@vec, hits => $hits };
}
- return \@feature_vectors, $vocab_size;
+ return \@feature_vectors, $vocab_size, \@vocab_keys;
}
sub learn_message {
@@ -615,7 +619,7 @@ sub learn_message {
my $update_vocab = 1;
# Convert email text to numerical feature vectors
- my ($feature_vectors, $vocab_size) = _text_to_features($self, $self->{main}->{conf}, $nn_data_dir, $update_vocab, $isspam, @email_texts);
+ my ($feature_vectors, $vocab_size, $vocab_keys_ref) = _text_to_features($self, $self->{main}->{conf}, $nn_data_dir, $update_vocab, $isspam, undef, @email_texts);
return unless $feature_vectors && @$feature_vectors;
@@ -628,19 +632,17 @@ sub learn_message {
my $num_output_neurons = 1;
# Reload model from disk if cache has expired
- my $ttl = $self->{main}->{conf}->{neuralnetwork_cache_ttl} || 0;
- my $model_age = defined $self->{_neural_model_load_time} ? time() - $self->{_neural_model_load_time} : undef;
- if (defined $model_age && $ttl > 0 && $model_age >= $ttl && -f $dataset_path) {
- dbg("Model cache expired (age: ${model_age}s, ttl: ${ttl}s), reloading before training");
- eval {
- $self->{neural_model} = AI::FANN->new_from_file($dataset_path);
- $self->{_neural_model_load_time} = time();
- 1;
- } or do {
- dbg("Failed to reload model: " . ($@ || 'unknown'));
- undef $self->{neural_model};
- };
- }
+ my $lock_path = $dataset_path . '.lock';
+ $lock_path = Mail::SpamAssassin::Util::untaint_file_path($lock_path);
+ open(my $lock_fh, '>', $lock_path) or do {
+ info("Cannot open lock file '$lock_path': $!");
+ return;
+ };
+ flock($lock_fh, LOCK_EX) or do {
+ info("Cannot acquire lock on '$lock_path': $!");
+ close($lock_fh);
+ return;
+ };
my $network;
if(defined $self->{neural_model} && $self->{neural_model}->num_inputs() == $num_input) {
@@ -658,10 +660,17 @@ sub learn_message {
}
if (defined $existing_network) {
- # Vocabulary grew: preserve the trained model by adjusting the training vectors
+ # Vocabulary grew: rebuild training vectors using the model's original word -> index mapping
my $model_size = $existing_network->num_inputs();
- dbg("Vocabulary size changed ($num_input vs model $model_size), adjusting training vectors");
- $feature_vectors = [ map { my $v = _adjust_vector_size($_->{vec}, $model_size); { vec => $v, hits => scalar grep { $_ != 0 } @$v } } @$feature_vectors ];
+ dbg("Vocabulary size changed ($num_input vs model $model_size), rebuilding training vectors with model vocabulary");
+ my $stored_vocab_ref = $self->_load_model_vocab($nn_data_dir);
+ if (defined $stored_vocab_ref && scalar(@$stored_vocab_ref) == $model_size) {
+ ($feature_vectors, undef) = _text_to_features($self, $self->{main}->{conf}, $nn_data_dir, 0, undef, $stored_vocab_ref, @email_texts);
+ $vocab_keys_ref = $stored_vocab_ref;
+ } else {
+ dbg("Model vocabulary file not found or mismatched, falling back to vector adjustment");
+ $feature_vectors = [ map { my $v = _adjust_vector_size($_->{vec}, $model_size); { vec => $v, hits => scalar grep { $_ != 0 } @$v } } @$feature_vectors ];
+ }
$num_input = $model_size;
$network = $existing_network;
} else {
@@ -688,6 +697,7 @@ sub learn_message {
}
if (!keys %{$vocab_for_balance{terms} || {}}) {
my $vocab_path = File::Spec->catfile($nn_data_dir, 'vocabulary-' . lc($self->{main}->{username}) . '.data');
+ $vocab_path = Mail::SpamAssassin::Util::untaint_file_path($vocab_path);
if (-f $vocab_path) {
eval {
my $ref = retrieve($vocab_path);
@@ -710,7 +720,7 @@ sub learn_message {
$class_weight = $spam_docs / $ham_docs; # < 1 when ham dominates
}
$class_weight = 0.5 if $class_weight < 0.5;
- $class_weight = 5.0 if $class_weight > 5.0;
+ $class_weight = 2.0 if $class_weight > 2.0;
my $weighted_epochs = int($train_epochs * $class_weight) || 1;
dbg("Incremental training: weighted_epochs=$weighted_epochs " .
@@ -725,6 +735,18 @@ sub learn_message {
}
}
+ # Train once on vocabulary-derived representative spam/ham
+ # vectors.
+ if (keys %{$vocab_for_balance{terms} || {}} && defined $vocab_keys_ref) {
+ my ($svec, $hvec) = _build_class_tfidf_vectors(\%vocab_for_balance, $vocab_keys_ref);
+ if ($svec) {
+ eval { $network->train($svec, [1]); 1 } or dbg("Replay spam step failed: " . ($@ || 'unknown'));
+ eval { $network->train($hvec, [0]); 1 } or dbg("Replay ham step failed: " . ($@ || 'unknown'));
+ dbg("Replay: spam_docs=" . ($vocab_for_balance{_spam_count} || 1) .
+ ", ham_docs=" . ($vocab_for_balance{_ham_count} || 1));
+ }
+ }
+
if (scalar(@$feature_vectors) == 1) {
my $pred_after = eval { $network->run($feature_vectors->[0]{vec}) };
$pred_after = ref($pred_after) ? $pred_after->[0] : $pred_after;
@@ -733,7 +755,12 @@ sub learn_message {
# Save the model
eval {
- $network->save($dataset_path) or die "save failed";
+ $network->save($dataset_path) or die "model save failed";
+ if (defined $self->{main}->{conf}->{neuralnetwork_dsn} && $self->{dbh}) {
+ $self->_save_model_vocab_to_sql($vocab_keys_ref);
+ } else {
+ $self->_save_model_vocab($vocab_keys_ref, $nn_data_dir);
+ }
1;
} and do {
dbg("Model saved to '$dataset_path' (input:$num_input)");
@@ -749,6 +776,7 @@ sub learn_message {
} or do {
info("Cannot save model to '$dataset_path' (" . ($@ || 'unknown') . ")");
};
+ close($lock_fh);
return;
}
@@ -849,6 +877,33 @@ sub _adjust_vector_size {
return \@v;
}
+# Build L2-normalised TF-IDF spam and ham vectors from a vocabulary hash.
+sub _build_class_tfidf_vectors {
+ my ($vocabulary, $vocab_keys) = @_;
+ return () unless ref($vocabulary) eq 'HASH' && ref($vocab_keys) eq 'ARRAY' && @$vocab_keys;
+
+ my $terms = $vocabulary->{terms} || {};
+ my $N = $vocabulary->{_doc_count} || 1;
+ my $spam_docs = $vocabulary->{_spam_count} || 1;
+ my $ham_docs = $vocabulary->{_ham_count} || 1;
+
+ my (@spam_vec, @ham_vec);
+ for my $i (0 .. $#$vocab_keys) {
+ my $w = $vocab_keys->[$i];
+ my $td = $terms->{$w} // {};
+ my $idf = log(($N + 1) / (($td->{docs} || 0) + 1)) + 1;
+ $spam_vec[$i] = (($td->{spam} || 0) / $spam_docs) * $idf;
+ $ham_vec[$i] = (($td->{ham} || 0) / $ham_docs) * $idf;
+ }
+
+ for my $vec (\@spam_vec, \@ham_vec) {
+ my $norm = sqrt(do { my $s = 0; $s += $_ * $_ for @$vec; $s }) || 1;
+ @$vec = map { $_ / $norm } @$vec;
+ }
+
+ return (\@spam_vec, \@ham_vec);
+}
+
# Create a baseline model from vocabulary statistics when vocab size has changed.
sub _retrain_from_vocabulary {
my ($self, $conf, $nn_data_dir, $vocab_size) = @_;
@@ -886,32 +941,12 @@ sub _retrain_from_vocabulary {
my $actual_size = scalar @vocab_keys;
return unless $actual_size == $vocab_size;
- # Build synthetic spam and ham TF-IDF vectors normalised by class
- # document count.
- my $N = $vocabulary{_doc_count} || 1;
my $spam_docs = $vocabulary{_spam_count} || 1;
my $ham_docs = $vocabulary{_ham_count} || 1;
- my (@spam_vec, @ham_vec);
- for my $i (0 .. $#vocab_keys) {
- my $w = $vocab_keys[$i];
- my $td = $terms->{$w};
- my $df = $td->{docs} || 0;
- my $spam_freq = $td->{spam} || 0;
- my $ham_freq = $td->{ham} || 0;
- my $idf = log(($N + 1) / ($df + 1)) + 1;
-
- $spam_vec[$i] = ($spam_freq / $spam_docs) * $idf;
- $ham_vec[$i] = ($ham_freq / $ham_docs) * $idf;
- }
-
- # L2-normalize both vectors
- for my $vec (\@spam_vec, \@ham_vec) {
- my $norm = 0;
- $norm += $_ * $_ for @$vec;
- $norm = sqrt($norm) || 1;
- @$vec = map { $_ / $norm } @$vec;
- }
+ my ($spam_vec_ref, $ham_vec_ref) = _build_class_tfidf_vectors(\%vocabulary, \@vocab_keys);
+ return unless $spam_vec_ref;
+ my (@spam_vec, @ham_vec) = (@$spam_vec_ref, @$ham_vec_ref);
my $spam_reps = 1;
my $ham_reps = 1;
@@ -977,11 +1012,22 @@ sub _check_neuralnetwork {
return;
}
+ my $dataset_path = File::Spec->catfile($nn_data_dir, 'fann-' . lc($self->{main}->{username}) . '.model');
+ if(not -f $dataset_path) {
+ $pms->{neuralnetwork_prediction} = undef;
+ dbg("Can't predict without a trained model, $dataset_path cannot be read");
+ return;
+ }
+
+ # Load the vocabulary the model was trained on so the feature vector dimensions
+ # are always aligned with the model, regardless of subsequent vocabulary growth.
+ my $stored_vocab_ref = $self->_load_model_vocab($nn_data_dir);
+
# Do not update the vocabulary
my $update_vocab = 0;
- # Convert email to feature vector using the same vocabulary
- my ($feature_vectors, $vocab_size) = _text_to_features($self, $conf, $nn_data_dir, $update_vocab, undef, $email_to_predict);
+ # Convert email to feature vector using the model's vocabulary
+ my ($feature_vectors, $vocab_size) = _text_to_features($self, $conf, $nn_data_dir, $update_vocab, undef, $stored_vocab_ref, $email_to_predict);
unless ($feature_vectors && @$feature_vectors) {
$pms->{neuralnetwork_prediction} = undef;
dbg("Not enough tokens found");
@@ -997,13 +1043,6 @@ sub _check_neuralnetwork {
}
my $input_vector = $feature_vectors->[0]{vec};
- my $dataset_path = File::Spec->catfile($nn_data_dir, 'fann-' . lc($self->{main}->{username}) . '.model');
- if(not -f $dataset_path) {
- $pms->{neuralnetwork_prediction} = undef;
- dbg("Can't predict without a trained model, $dataset_path cannot be read");
- return;
- }
-
my $ttl = $conf->{neuralnetwork_cache_ttl} || 0;
my $model_age = defined $self->{_neural_model_load_time} ? time() - $self->{_neural_model_load_time} : undef;
my $model_expired = defined $model_age && $ttl > 0 && $model_age >= $ttl;
@@ -1025,7 +1064,8 @@ sub _check_neuralnetwork {
my $expected_size = $network->num_inputs();
if (scalar(@$input_vector) != $expected_size) {
- dbg("Vocabulary size changed (got ".scalar(@$input_vector).", model expects ".$expected_size."), adjusting input vector");
+ # Fallback for models created before vocab tracking was introduced
+ dbg("Input vector size mismatch (got ".scalar(@$input_vector).", model expects ".$expected_size."), adjusting");
$input_vector = _adjust_vector_size($input_vector, $expected_size);
unless (defined $input_vector && scalar(@$input_vector) == $expected_size) {
$pms->{neuralnetwork_prediction} = undef;
@@ -1103,6 +1143,7 @@ sub _create_vocabulary_table {
docs_count INTEGER NOT NULL DEFAULT 0,
spam_count INTEGER NOT NULL DEFAULT 0,
ham_count INTEGER NOT NULL DEFAULT 0,
+ model_position INTEGER DEFAULT NULL,
UNIQUE (username, keyword)
)
");
@@ -1238,6 +1279,7 @@ sub _save_vocabulary_to_sql {
my $sth_upsert = $self->{dbh}->prepare($upsert_sql);
my $count = 0;
+ $self->{dbh}->begin_work();
foreach my $keyword (keys %{$terms}) {
my $term_data = $terms->{$keyword};
$sth_upsert->execute(
@@ -1250,6 +1292,7 @@ sub _save_vocabulary_to_sql {
);
$count++;
}
+ $self->{dbh}->commit();
dbg("Saved $count vocabulary terms to SQL for user: $username");
@@ -1260,6 +1303,7 @@ sub _save_vocabulary_to_sql {
}
1;
} or do {
+ eval { $self->{dbh}->rollback() };
my $err = $@ || 'unknown';
dbg("Failed to save vocabulary to SQL: $err");
};
@@ -1339,4 +1383,90 @@ sub _load_vocabulary_from_sql {
return \%vocabulary;
}
+sub _save_model_vocab_to_sql {
+ my ($self, $vocab_keys_ref, $username) = @_;
+ return unless $self->{dbh} && defined $vocab_keys_ref;
+
+ $username ||= $self->{main}->{username};
+
+ eval {
+ $self->{dbh}->begin_work();
+ $self->{dbh}->do(
+ "UPDATE neural_vocabulary SET model_position = NULL WHERE username = ?",
+ undef, lc($username)
+ );
+ my $sth = $self->{dbh}->prepare(
+ "UPDATE neural_vocabulary SET model_position = ? WHERE username = ? AND keyword = ?"
+ );
+ for my $i (0 .. $#$vocab_keys_ref) {
+ $sth->execute($i, lc($username), $vocab_keys_ref->[$i]);
+ }
+ $self->{dbh}->commit();
+ dbg("Saved model vocabulary (" . scalar(@$vocab_keys_ref) . " terms) to SQL for user: $username");
+ 1;
+ } or do {
+ eval { $self->{dbh}->rollback() };
+ dbg("Failed to save model vocabulary to SQL: " . ($@ || 'unknown'));
+ };
+}
+
+sub _load_model_vocab_from_sql {
+ my ($self, $username) = @_;
+ return undef unless $self->{dbh};
+
+ $username ||= $self->{main}->{username};
+
+ my $vocab_ref;
+ eval {
+ my $sth = $self->{dbh}->prepare(
+ "SELECT keyword FROM neural_vocabulary
+ WHERE username = ? AND model_position IS NOT NULL
+ ORDER BY model_position"
+ );
+ $sth->execute(lc($username));
+ my $rows = $sth->fetchall_arrayref();
+ $vocab_ref = [ map { $_->[0] } @$rows ] if @$rows;
+ 1;
+ } or do {
+ dbg("Failed to load model vocabulary from SQL: " . ($@ || 'unknown'));
+ };
+ return $vocab_ref;
+}
+
+sub _model_vocab_path {
+ my ($self, $nn_data_dir) = @_;
+ return File::Spec->catfile($nn_data_dir, 'model-vocab-' . lc($self->{main}->{username}) . '.data');
+}
+
+sub _save_model_vocab {
+ my ($self, $vocab_keys_ref, $nn_data_dir) = @_;
+ my $vocab_path = $self->_model_vocab_path($nn_data_dir);
+ $vocab_path = Mail::SpamAssassin::Util::untaint_file_path($vocab_path);
+ eval {
+ store($vocab_keys_ref, $vocab_path) or die "store failed";
+ 1;
+ } or do {
+ dbg("Failed to save model vocabulary to file: " . ($@ || 'unknown'));
+ };
+}
+
+sub _load_model_vocab {
+ my ($self, $nn_data_dir) = @_;
+ if (defined $self->{main}->{conf}->{neuralnetwork_dsn} && $self->{dbh}) {
+ return $self->_load_model_vocab_from_sql();
+ } else {
+ my $vocab_path = $self->_model_vocab_path($nn_data_dir);
+ $vocab_path = Mail::SpamAssassin::Util::untaint_file_path($vocab_path);
+ return undef unless -f $vocab_path;
+ my $vocab_ref;
+ eval {
+ $vocab_ref = retrieve($vocab_path);
+ 1;
+ } or do {
+ dbg("Failed to load model vocabulary from file: " . ($@ || 'unknown'));
+ };
+ return $vocab_ref;
+ }
+}
+
1;
Modified: spamassassin/trunk/sql/neural_mysql.sql
==============================================================================
--- spamassassin/trunk/sql/neural_mysql.sql Sun Mar 8 08:30:18 2026 (r1932216)
+++ spamassassin/trunk/sql/neural_mysql.sql Sun Mar 8 08:39:06 2026 (r1932217)
@@ -12,5 +12,7 @@ CREATE TABLE neural_vocabulary (
docs_count int(11) NOT NULL DEFAULT '0',
spam_count int(11) NOT NULL DEFAULT '0',
ham_count int(11) NOT NULL DEFAULT '0',
- PRIMARY KEY neural_vocab_idx1 (username, keyword)
+ model_position int(11) DEFAULT NULL,
+ PRIMARY KEY neural_vocab_idx1 (username, keyword),
+ KEY neural_vocab_model_pos_idx (username, model_position)
) ENGINE=InnoDB;
Modified: spamassassin/trunk/sql/neural_pg.sql
==============================================================================
--- spamassassin/trunk/sql/neural_pg.sql Sun Mar 8 08:30:18 2026 (r1932216)
+++ spamassassin/trunk/sql/neural_pg.sql Sun Mar 8 08:39:06 2026 (r1932217)
@@ -19,6 +19,7 @@ CREATE TABLE neural_vocabulary (
docs_count INTEGER NOT NULL DEFAULT 0,
spam_count INTEGER NOT NULL DEFAULT 0,
ham_count INTEGER NOT NULL DEFAULT 0,
+ model_position INTEGER DEFAULT NULL,
UNIQUE (username, keyword)
);
@@ -26,3 +27,4 @@ CREATE INDEX neural_vocabulary_username_
CREATE INDEX neural_vocabulary_keyword_idx ON neural_vocabulary(keyword);
CREATE INDEX neural_vocabulary_spam_count_idx ON neural_vocabulary(spam_count DESC);
CREATE INDEX neural_vocabulary_total_count_idx ON neural_vocabulary(total_count DESC);
+CREATE INDEX neural_vocabulary_model_position_idx ON neural_vocabulary(username, model_position);
Added: spamassassin/trunk/sql/neural_sqlite.sql
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/sql/neural_sqlite.sql Sun Mar 8 08:39:06 2026 (r1932217)
@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS neural_seen (
+ username VARCHAR(200) NOT NULL DEFAULT 'default',
+ msgid VARCHAR(200) NOT NULL DEFAULT '',
+ flag CHAR(1) NOT NULL DEFAULT '',
+ UNIQUE (username, msgid)
+);
+
+CREATE TABLE IF NOT EXISTS neural_vocabulary (
+ username VARCHAR(200) NOT NULL DEFAULT '',
+ keyword VARCHAR(256) NOT NULL DEFAULT '',
+ total_count INTEGER NOT NULL DEFAULT 0,
+ docs_count INTEGER NOT NULL DEFAULT 0,
+ spam_count INTEGER NOT NULL DEFAULT 0,
+ ham_count INTEGER NOT NULL DEFAULT 0,
+ model_position INTEGER DEFAULT NULL,
+ UNIQUE (username, keyword)
+);
+
+CREATE INDEX IF NOT EXISTS neural_vocabulary_username_idx ON neural_vocabulary(username);
+CREATE INDEX IF NOT EXISTS neural_vocabulary_model_position_idx ON neural_vocabulary(username, model_position);
Modified: spamassassin/trunk/t/neuralnetwork.t
==============================================================================
--- spamassassin/trunk/t/neuralnetwork.t Sun Mar 8 08:30:18 2026 (r1932216)
+++ spamassassin/trunk/t/neuralnetwork.t Sun Mar 8 08:39:06 2026 (r1932217)
@@ -30,6 +30,7 @@ tstprefs("
neuralnetwork_data_dir $userstate/NN
neuralnetwork_min_spam_count 0
neuralnetwork_min_ham_count 0
+ neuralnetwork_min_vocab_hits 5
body NN_SPAM eval:check_neuralnetwork_spam()
describe NN_SPAM Email considered as spam by Neural Network
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.