svn commit: r1936252 - in spamassassin/trunk: . lib/Mail lib/Mail/SpamAssassin lib/Mail/SpamAssassin/Plugin t

Author: gbechis
Date: Fri Jul 17 10:36:54 2026
New Revision: 1936252

Log:
make NeuralNetwork plugin work even without Bayes plugin enabled
add a `--plugin` option to sa-learn in order to dump NeuralNetwork stats

Modified:
   spamassassin/trunk/lib/Mail/SpamAssassin.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
   spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
   spamassassin/trunk/sa-learn.raw
   spamassassin/trunk/t/neuralnetwork.t

Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin.pm	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/lib/Mail/SpamAssassin.pm	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -849,6 +849,37 @@ sub dump_bayes_db {
   $self->{bayes_scanner}->dump_bayes_db(@opts) if $self->{bayes_scanner};
 }
 
+=item $f-E<gt>dump_neuralnetwork_db()
+
+Dump the contents of the NeuralNetwork plugin's data/stats.
+
+=cut
+
+sub dump_neuralnetwork_db {
+  my($self, $magic, $toks, $regex) = @_;
+  return $self->call_plugins("neuralnetwork_dump_database",
+        { magic => $magic, toks => $toks, regex => $regex });
+}
+
+=item $f-E<gt>learner_scoreset_active()
+
+Returns true if a learner plugin, (Bayes and/or NeuralNetwork),  is enabled
+and has enough data to actively classify mail.
+
+=cut
+
+sub learner_scoreset_active {
+  my $self = shift;
+
+  return 1 if $self->{bayes_scanner} && $self->{bayes_scanner}->is_scan_available()
+              && $self->{conf}->{use_bayes_rules};
+
+  return 1 if $self->{conf}->{use_neuralnetwork}
+              && $self->call_plugins("neuralnetwork_is_scan_available");
+
+  return 0;
+}
+
 =item $f-E<gt>signal_user_changed ( [ { opt =E<gt> val, ... } ] )
 
 Signals that the current user has changed (possibly using C<setuid>), meaning
@@ -907,8 +938,11 @@ sub signal_user_changed {
   }
 
   # reopen bayes dbs for this user
+  # Note: this dispatcher object is shared by all learner plugins (Bayes and
+  # others), so it is gated on the generic use_learner switch, not use_bayes;
+  # each plugin's hooks are responsible for checking their own specific switch.
   $self->{bayes_scanner}->finish() if $self->{bayes_scanner};
-  if ($self->{conf}->{use_bayes}) {
+  if ($self->{conf}->{use_learner}) {
       require Mail::SpamAssassin::Bayes;
       $self->{bayes_scanner} = Mail::SpamAssassin::Bayes->new($self);
   } else {
@@ -919,7 +953,7 @@ sub signal_user_changed {
   $self->{'learn_to_journal'} = $self->{conf}->{bayes_learn_to_journal};
 
   $set |= 1 unless $self->{local_tests_only};
-  $set |= 2 if $self->{bayes_scanner} && $self->{bayes_scanner}->is_scan_available() && $self->{conf}->{use_bayes_rules};
+  $set |= 2 if $self->learner_scoreset_active();
 
   $self->{conf}->set_score_set ($set);
 
@@ -1878,8 +1912,9 @@ sub init {
     die "config: no rules were found!  Do you need to run 'sa-update'?\n";
   }
 
-  # Initialize the Bayes subsystem
-  if ($self->{conf}->{use_bayes}) {
+  # Initialize the learner subsystem dispatcher; shared by all learner
+  # plugins (Bayes and others).
+  if ($self->{conf}->{use_learner}) {
       require Mail::SpamAssassin::Bayes;
       $self->{bayes_scanner} = Mail::SpamAssassin::Bayes->new($self);
   }
@@ -1888,7 +1923,7 @@ sub init {
   # Figure out/set our initial scoreset
   my $set = 0;
   $set |= 1 unless $self->{local_tests_only};
-  $set |= 2 if $self->{bayes_scanner} && $self->{bayes_scanner}->is_scan_available() && $self->{conf}->{use_bayes_rules};
+  $set |= 2 if $self->learner_scoreset_active();
   $self->{conf}->set_score_set ($set);
 
   if ($self->{only_these_rules}) {

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -206,11 +206,12 @@ for a test.
 
 If four valid scores are listed, then the score that is used depends
 on how SpamAssassin is being used. The first score is used when
-both Bayes and network tests are disabled (score set 0). The second
-score is used when Bayes is disabled, but network tests are enabled
-(score set 1). The third score is used when Bayes is enabled and
-network tests are disabled (score set 2). The fourth score is used
-when Bayes is enabled and network tests are enabled (score set 3).
+both a learner (Bayes and/or NeuralNetwork) and network tests are disabled
+(score set 0). The second score is used when no learner is active, but
+network tests are enabled (score set 1). The third score is used when a
+learner (Bayes and/or NeuralNetwork) is active and network tests are
+disabled (score set 2). The fourth score is used when a learner is active
+and network tests are enabled (score set 3).
 
 Setting a rule's score to 0 will disable that rule from running.
 

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -417,12 +417,12 @@ sub check_timed {
   $self->{msg}->delete_header('X-Spam-.*');
 
   # Resident Mail::SpamAssassin code will possibly never change score
-  # sets, even if bayes becomes available.  So we should do a quick check
-  # to see if we should go from {0,1} to {2,3}.  We of course don't need
-  # to do this switch if we're already using bayes ... ;)
+  # sets, even if a learner (Bayes or NeuralNetwork) becomes available.  So
+  # we should do a quick check to see if we should go from {0,1} to {2,3}.
+  # We of course don't need to do this switch if we're already using one ... ;)
   my $set = $self->{conf}->get_score_set();
-  if (($set & 2) == 0 && $self->{main}->{bayes_scanner} && $self->{main}->{bayes_scanner}->is_scan_available() && $self->{conf}->{use_bayes_rules}) {
-    dbg("check: scoreset $set but bayes is available, switching scoresets");
+  if (($set & 2) == 0 && $self->{main}->learner_scoreset_active()) {
+    dbg("check: scoreset $set but a learner is available, switching scoresets");
     $self->{conf}->set_score_set ($set|2);
   }
   dbg("check: using scoreset $set in M:S:Pms"); 

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/NeuralNetwork.pm	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -31,10 +31,11 @@ This plugin checks emails using Neural N
 
 =head1 CAVEATS
 
-The SpamAssassin learning subsystem routes all training through the Bayes
-scanner infrastructure.  As a result, C<Mail::SpamAssassin::Plugin::Bayes>
-must be loaded and C<use_bayes 1> must be set for this plugin's training to
-be triggered.
+Training, forgetting and prediction only require the generic C<use_learner 1>
+switch (on by default) and C<use_neuralnetwork 1>.
+
+C<sa-learn --dump --plugin NeuralNetwork> can be used to display this
+plugin's stats and vocabulary data.
 
 =cut
 
@@ -61,6 +62,7 @@ use Storable qw(store retrieve);
 use File::Copy qw(copy);
 use File::Spec;
 use Errno qw(EXDEV);
+use Encode qw(encode);
 
 use Mail::SpamAssassin;
 use Mail::SpamAssassin::Plugin;
@@ -1055,6 +1057,99 @@ sub forget_message {
   return 0;
 }
 
+# Plugin hook, invoked via Mail::SpamAssassin::dump_neuralnetwork_db
+# (sa-learn's `--dump --plugin NeuralNetwork`). Deliberately a distinct
+# hook name from Bayes's "learner_dump_database" so plain `sa-learn --dump`
+# never picks this up.
+sub neuralnetwork_dump_database {
+  my ($self, $params) = @_;
+  my $conf = $self->{main}->{conf};
+
+  return 0 unless $conf->{use_neuralnetwork};
+
+  my $magic = $params->{magic};
+  my $toks  = $params->{toks};
+  my $regex = $params->{regex};
+
+  my $nn_data_dir = $conf->{neuralnetwork_data_dir};
+  unless (defined $nn_data_dir) {
+    dbg("neuralnetwork_data_dir not set, nothing to dump");
+    return 0;
+  }
+  $nn_data_dir = Mail::SpamAssassin::Util::untaint_file_path($nn_data_dir);
+  unless (-d $nn_data_dir) {
+    info("Cannot access directory $nn_data_dir");
+    return 0;
+  }
+
+  $self->_init_sql_connection($conf) if defined $conf->{neuralnetwork_dsn};
+
+  my $vocabulary = $self->_load_vocabulary($conf, $nn_data_dir, 0);
+  my $meta       = $self->_load_meta($conf);
+
+  my $template = '%3.3f %10u %10u %10u  %s'."\n";
+
+  if ($magic) {
+    printf($template, 0.0, 0, $vocabulary->{_spam_count}, 0,
+        'non-token data: neuralnetwork nspam') or die "Error writing: $!";
+    printf($template, 0.0, 0, $vocabulary->{_ham_count}, 0,
+        'non-token data: neuralnetwork nham') or die "Error writing: $!";
+    printf($template, 0.0, 0, $vocabulary->{_doc_count}, 0,
+        'non-token data: neuralnetwork ndocs') or die "Error writing: $!";
+    printf($template, 0.0, 0, scalar(keys %{$vocabulary->{terms}}), 0,
+        'non-token data: neuralnetwork nvocab') or die "Error writing: $!";
+    printf($template, 0.0, 0, $meta->{_learns_since_retrain}, 0,
+        'non-token data: neuralnetwork learns since retrain') or die "Error writing: $!";
+    printf($template, 0.0, 0, scalar(@{$meta->{_tbuf}{spam} || []}), 0,
+        'non-token data: neuralnetwork training buffer spam') or die "Error writing: $!";
+    printf($template, 0.0, 0, scalar(@{$meta->{_tbuf}{ham} || []}), 0,
+        'non-token data: neuralnetwork training buffer ham') or die "Error writing: $!";
+
+    my $dataset_path = Mail::SpamAssassin::Util::untaint_file_path($self->_model_path($nn_data_dir));
+    if ($HAS_AI_FANN && -f $dataset_path) {
+      my $mtime = (stat($dataset_path))[9] || 0;
+      my $inputs = eval { AI::FANN->new_from_file($dataset_path)->num_inputs() } || 0;
+      printf($template, 0.0, 0, $mtime, 0,
+          'non-token data: neuralnetwork model mtime') or die "Error writing: $!";
+      printf($template, 0.0, 0, $inputs, 0,
+          'non-token data: neuralnetwork model inputs') or die "Error writing: $!";
+    }
+  }
+
+  if ($toks) {
+    my $terms = $vocabulary->{terms} || {};
+    foreach my $term (sort keys %$terms) {
+      next if defined $regex && $term !~ /$regex/o;
+      my $t = $terms->{$term} || {};
+      my $out_term = utf8::is_utf8($term) ? encode('UTF-8', $term) : $term;
+      printf($template, 0.0, $t->{total} || 0, $t->{spam} || 0, $t->{docs} || 0,
+          $out_term) or die "Error writing: $!";
+    }
+  }
+
+  return 1;
+}
+
+# Plugin hook, invoked via Mail::SpamAssassin::learner_scoreset_active to
+# decide whether score sets 2/3 (the "learner enabled" scoresets) should be
+# used. True when this plugin is enabled and has a usable trained model for
+# the current user.
+sub neuralnetwork_is_scan_available {
+  my ($self, $params) = @_;
+  my $conf = $self->{main}->{conf};
+
+  return 0 unless $conf->{use_neuralnetwork};
+  return 0 unless $HAS_AI_FANN;
+
+  my $nn_data_dir = $conf->{neuralnetwork_data_dir};
+  return 0 unless defined $nn_data_dir;
+  $nn_data_dir = Mail::SpamAssassin::Util::untaint_file_path($nn_data_dir);
+  return 0 unless -d $nn_data_dir;
+
+  my $dataset_path = Mail::SpamAssassin::Util::untaint_file_path($self->_model_path($nn_data_dir));
+  return -f $dataset_path ? 1 : 0;
+}
+
 sub check_neuralnetwork_spam {
   my ($self, $pms) = @_;
 

Modified: spamassassin/trunk/sa-learn.raw
==============================================================================
--- spamassassin/trunk/sa-learn.raw	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/sa-learn.raw	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -121,6 +121,7 @@ GetOptions(
   'version|V'       => \$opt{'version'},
 
   'dump:s' => \$opt{'dump'},
+  'plugin=s' => \$opt{'plugin'},
   'import' => \$opt{'import'},
 
   'backup'    => \$opt{'backup'},
@@ -153,6 +154,17 @@ if (defined $opt{'debug'}) {
   $opt{'debug'} ||= 'all';
 }
 
+# --plugin selects which classifier plugin --dump (and friends) operate on;
+# defaults to Bayes for backwards compatibility.
+$opt{'plugin'} = 'Bayes' unless defined $opt{'plugin'};
+if ($opt{'plugin'} =~ /^bayes$/i) {
+  $opt{'plugin'} = 'Bayes';
+} elsif ($opt{'plugin'} =~ /^neuralnetwork$/i) {
+  $opt{'plugin'} = 'NeuralNetwork';
+} else {
+  usage( 0, "Unknown --plugin '" . $opt{'plugin'} . "', must be Bayes or NeuralNetwork" );
+}
+
 if ( $opt{'force-expire'} ) {
   $synconly = 1;
 }
@@ -207,9 +219,11 @@ if ( defined $bayes_override_path ) {
 }
 
 # These options require bayes_scanner, which requires "use_bayes 1", but
-# that's not necessary for these commands.
-if (defined $opt{'dump'} || defined $opt{'import'} || defined $opt{'clear'} ||
-    defined $opt{'backup'} || defined $opt{'restore'}) {
+# that's not necessary for these commands. --plugin NeuralNetwork redirects
+# --dump to the NeuralNetwork plugin instead, so it's excluded here.
+if (defined $opt{'import'} || defined $opt{'clear'} ||
+    defined $opt{'backup'} || defined $opt{'restore'} ||
+    (defined $opt{'dump'} && $opt{'plugin'} eq 'Bayes')) {
   $post_config .= "use_bayes 1\n";
 }
 
@@ -269,9 +283,21 @@ if ( defined $opt{'dump'} ) {
     exit 1;
   }
 
-  if (!$spamtest->dump_bayes_db( $magic, $toks, $opt{'regexp'}) ) {
+  # --plugin selects which plugin --dump displays data for; the
+  # NeuralNetwork plugin's own stats/vocabulary instead of the Bayes
+  # database.
+  if ($opt{'plugin'} eq 'NeuralNetwork' && !$spamtest->{conf}->{use_neuralnetwork}) {
     $spamtest->finish_learner();
-    die "ERROR: Bayes dump returned an error, please re-run with -D for more information\n";
+    die "ERROR: --plugin NeuralNetwork requires 'use_neuralnetwork 1' in the configuration\n";
+  }
+
+  my ( $dump_ok, $what ) = $opt{'plugin'} eq 'NeuralNetwork'
+    ? ( $spamtest->dump_neuralnetwork_db( $magic, $toks, $opt{'regexp'} ), 'NeuralNetwork' )
+    : ( $spamtest->dump_bayes_db( $magic, $toks, $opt{'regexp'} ), 'Bayes' );
+
+  if (!$dump_ok) {
+    $spamtest->finish_learner();
+    die "ERROR: $what dump returned an error, please re-run with -D for more information\n";
   }
 
   $spamtest->finish_learner();
@@ -281,6 +307,23 @@ if ( defined $opt{'dump'} ) {
   exit 0;
 }
 
+# --plugin only changes the behaviour of --dump; the NeuralNetwork plugin
+# has no equivalent of these Bayes-store-specific operations.
+if ($opt{'plugin'} eq 'NeuralNetwork' &&
+    (defined $opt{'import'} || defined $opt{'clear'} ||
+     defined $opt{'backup'} || defined $opt{'restore'})) {
+  $spamtest->finish_learner();
+  die "ERROR: --plugin NeuralNetwork is not supported together with --import, --clear, --backup or --restore\n";
+}
+
+# --import/--clear/--backup/--restore operate directly on the Bayes
+# on-disk store, so the Bayes plugin must actually be loaded for them.
+if ((defined $opt{'import'} || defined $opt{'clear'} || defined $opt{'backup'} ||
+     defined $opt{'restore'}) && !$spamtest->{bayes_scanner}->{store}) {
+  $spamtest->finish_learner();
+  die "ERROR: this command requires the Bayes plugin to be loaded and enabled\n";
+}
+
 if ( defined $opt{'import'} ) {
   my $ret = $spamtest->{bayes_scanner}->{store}->perform_upgrade();
   $spamtest->finish_learner();
@@ -337,8 +380,8 @@ if (defined $opt{'restore'}) {
   exit 0;
 }
 
-if ( !$spamtest->{conf}->{use_bayes} ) {
-  warn "ERROR: configuration specifies 'use_bayes 0', sa-learn disabled\n";
+if ( !$spamtest->{conf}->{use_learner} ) {
+  warn "ERROR: configuration specifies 'use_learner 0', sa-learn disabled\n";
   exit 1;
 }
 
@@ -631,8 +674,12 @@ Options:
  --force-expire        Force a database sync and expiry run
  --dbpath <path>       Allows commandline override (in bayes_path form)
                        for where to read the Bayes DB from
- --dump [all|data|magic]  Display the contents of the Bayes database
-                       Takes optional argument for what to display
+ --dump [all|data|magic]  Display the contents of the selected plugin's
+                       database (see --plugin). Takes optional argument
+                       for what to display
+ --plugin <name>       Which plugin --dump displays data for: Bayes
+                       (default) or NeuralNetwork. Not supported with
+                       --import, --clear, --backup or --restore.
   --regexp <re>        For dump only, specifies which tokens to
                        dump based on a regular expression.
  -f file, --folders=file  Read list of files/directories from file
@@ -775,14 +822,26 @@ Allows a commandline override of the I<b
 
 =item B<--dump> I<option>
 
-Display the contents of the Bayes database.  Without an option or with
-the I<all> option, all magic tokens and data tokens will be displayed.
-I<magic> will only display magic tokens, and I<data> will only display
-the data tokens.
+Display the contents of the database of the plugin selected with
+B<--plugin> (Bayes by default). Without an option or with the I<all>
+option, all magic tokens and data tokens will be displayed. I<magic> will
+only display magic tokens, and I<data> will only display the data tokens.
 
 Can also use the B<--regexp> I<RE> option to specify which tokens to
 display based on a regular expression.
 
+=item B<--plugin> I<name>
+
+Selects which plugin's data B<--dump> displays: I<Bayes> (the default) or
+I<NeuralNetwork>. With I<NeuralNetwork>, I<magic> shows spam/ham/doc
+counts, vocabulary size and model info, and I<data> shows the vocabulary
+terms (requires C<use_neuralnetwork 1>).
+
+B<--plugin> only affects B<--dump>, and I<NeuralNetwork> is not supported
+together with B<--import>, B<--clear>, B<--backup> or B<--restore> -- the
+NeuralNetwork plugin has no equivalent of those Bayes-store-specific
+operations.
+
 =item B<--clear>
 
 Clear an existing Bayes database by removing all traces of the database.

Modified: spamassassin/trunk/t/neuralnetwork.t
==============================================================================
--- spamassassin/trunk/t/neuralnetwork.t	Fri Jul 17 10:19:26 2026	(r1936251)
+++ spamassassin/trunk/t/neuralnetwork.t	Fri Jul 17 10:36:54 2026	(r1936252)
@@ -10,7 +10,7 @@ use Test::More;
 
 use constant HAS_SQLITE => eval { require DBD::SQLite; 1 };
 
-plan tests => 18;
+plan tests => 29;
 
 sub nn_reinit {
   my $extra = shift || '';
@@ -152,3 +152,43 @@ SKIP: {
   }
   is($correct_ham, 2, 'SQLite backend: all ham classified correctly');
 }
+
+# Verify NeuralNetwork training/classification works fully independent of
+# Bayes (use_bayes 0). Regression test for the bayes_scanner dispatcher
+# being gated on the generic use_learner switch instead of use_bayes --
+# previously, use_bayes 0 silently prevented learn_message/forget_message
+# from ever firing for ANY plugin, not just Bayes.
+
+nn_reinit("use_bayes 0");
+ok(salearnrun("-L --spam data/spam/001", \&check_examined_token));
+ok(salearnrun("-L --ham  data/nice/001", \&check_examined_token));
+
+%patterns    = ( q{ 1.0 NN_SPAM }, '' );
+%anti_patterns = ( q{ -1.0 NN_HAM }, '' );
+sarun("-L -t < data/spam/001", \&patterns_run_cb);
+ok_all_patterns();
+
+%patterns    = ( q{ -1.0 NN_HAM }, '' );
+%anti_patterns = ( q{ 1.0 NN_SPAM }, '' );
+sarun("-L -t < data/nice/001", \&patterns_run_cb);
+ok_all_patterns();
+
+# --dump --plugin NeuralNetwork must show NeuralNetwork stats, and plain
+# --dump (--plugin defaults to Bayes) must stay Bayes-only regardless of
+# whether the NeuralNetwork plugin is enabled.
+
+nn_reinit();
+salearnrun("--spam data/spam/001", undef);
+salearnrun("--ham  data/nice/001", undef);
+
+my $nn_dump_output = '';
+ok(salearnrun("--dump magic --plugin NeuralNetwork", sub { $nn_dump_output = join('', <IN>); }));
+like($nn_dump_output, qr/non-token data: neuralnetwork nspam/,
+     '--dump --plugin NeuralNetwork shows nspam stat');
+like($nn_dump_output, qr/non-token data: neuralnetwork nham/,
+     '--dump --plugin NeuralNetwork shows nham stat');
+
+my $bayes_dump_output = '';
+ok(salearnrun("--dump magic", sub { $bayes_dump_output = join('', <IN>); }));
+unlike($bayes_dump_output, qr/neuralnetwork/i,
+       '--dump defaults to Bayes-only, no neuralnetwork markers leak in');