svn commit: r1935564 - in spamassassin/trunk: . lib/Mail/SpamAssassin t t/data t/data/nice
[email protected] Tue, 23 Jun 2026 04:10:28 -0000
| Newsgroups | gmane.mail.spam.spamassassin.cvs |
|---|---|
| Message-ID | <178218782819.116971.13409031342498569177@svn03-he-fi> |
Author: fkento
Date: Tue Jun 23 04:10:27 2026
New Revision: 1935564
Log:
Add MIME-part handler framework for plugins
Plugins can register a method as the handler for a content-type pattern
via register_handler(). Each matching MIME part is dispatched to its
handler, which can inject extracted text and return synthetic child
parts that are dispatched recursively. Dispatch is bounded by new
handler_max_depth, handler_max_parts, handler_max_bytes, and
handler_time_limit options.
Added:
spamassassin/trunk/t/data/nice/handler_textplain
spamassassin/trunk/t/data/testhandler.pm
spamassassin/trunk/t/handler.t
Modified:
spamassassin/trunk/UPGRADE
spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin.pm
Modified: spamassassin/trunk/UPGRADE
==============================================================================
--- spamassassin/trunk/UPGRADE Tue Jun 23 01:59:58 2026 (r1935563)
+++ spamassassin/trunk/UPGRADE Tue Jun 23 04:10:27 2026 (r1935564)
@@ -68,6 +68,17 @@ Note for Users Upgrading to SpamAssassin
server in order to catch more redirectors that uses Javascript or
other tricks.
+- New MIME-part handler framework. Plugins can now register a method
+ as the handler for a content-type pattern (an exact type such as
+ "image/jpeg" or a major-type glob such as "image/*") via
+ $plugin->register_handler(). During message processing, each matching
+ MIME part is dispatched to its handler, which can inject extracted
+ text into the part and return synthetic child parts that are
+ dispatched recursively (for example, nested archives or embedded
+ images). Dispatch is bounded by the new handler_max_depth,
+ handler_max_parts, handler_max_bytes, and handler_time_limit options
+ to guard against deeply nested or oversized content.
+
Note for Users Upgrading to SpamAssassin 4.0.2
----------------------------------------------
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Tue Jun 23 01:59:58 2026 (r1935563)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Conf.pm Tue Jun 23 04:10:27 2026 (r1935564)
@@ -4492,6 +4492,50 @@ the filesystem.
}
});
+=item handler_max_depth n (default: 8)
+
+Maximum handler chain depth (guards against deeply nested archives).
+
+=item handler_max_parts n (default: 1000)
+
+Maximum total synthetic parts produced per message.
+
+=item handler_max_bytes n (default: 50000000)
+
+Total extracted-bytes budget across the handler chain for one message.
+
+=item handler_time_limit n (default: 10)
+
+Per-invocation timeout, in seconds, for a single handler running on a single
+part. The whole-pass budget is the existing scan-wide time limit, not a
+separate setting.
+
+=cut
+
+ push (@cmds, {
+ setting => 'handler_max_depth',
+ default => 8,
+ type => $CONF_TYPE_NUMERIC,
+ });
+
+ push (@cmds, {
+ setting => 'handler_max_parts',
+ default => 1000,
+ type => $CONF_TYPE_NUMERIC,
+ });
+
+ push (@cmds, {
+ setting => 'handler_max_bytes',
+ default => 50_000_000,
+ type => $CONF_TYPE_NUMERIC,
+ });
+
+ push (@cmds, {
+ setting => 'handler_time_limit',
+ default => 10,
+ type => $CONF_TYPE_NUMERIC,
+ });
+
=item ignore_always_matching_regexps (Default: 0)
Ignore any rule which contains a regexp which always matches.
@@ -5477,6 +5521,41 @@ sub load_plugin {
$self->{main}->{plugins}->load_plugin($package, $path, $silent);
}
+# Register a plugin method as the MIME-part handler for a content-type pattern.
+# Called via Mail::SpamAssassin::Plugin::register_handler. $pattern is an exact
+# type ("image/jpeg") or a major-type glob ("image/*"); $method is the name of
+# the method to invoke on $obj for each matching part.
+sub register_handler {
+ my ($self, $obj, $pattern, $method) = @_;
+
+ unless (defined $pattern && $pattern =~ m{^[\w.+-]+/(?:[\w.+-]+|\*)$}) {
+ warn "handler: ignoring invalid handler pattern '".($pattern//'')."'\n";
+ return;
+ }
+ unless (defined $method && $method ne '') {
+ warn "handler: register_handler for $pattern requires a method name\n";
+ return;
+ }
+
+ my $key = lc $pattern;
+ if (exists $self->{handlers}->{$key}) {
+ warn "handler: overwriting handler for $pattern: " .
+ ref($self->{handlers}->{$key}->[0]) . " replaced by " . ref($obj) . "\n";
+ }
+ $self->{handlers}->{$key} = [ $obj, $method ];
+ dbg("handler: registered %s->%s for %s", ref $obj, $method, $pattern);
+}
+
+# Resolve the [handler_obj, method] pair for a content type, or undef. Try the
+# exact type first, then fall back to the "<major>/*" glob
+sub get_handler_for_type {
+ my ($conf, $type) = @_;
+ $type = lc $type;
+ return $conf->{handlers}->{$type} if exists $conf->{handlers}->{$type};
+ (my $glob = $type) =~ s{/.*}{/*}s; # "image/jpeg" -> "image/*"
+ return $conf->{handlers}->{$glob};
+}
+
sub load_plugin_succeeded {
my ($self, $plugin, $package, $path) = @_;
$self->{plugins_loaded}->{$package} = 1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Tue Jun 23 01:59:58 2026 (r1935563)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message.pm Tue Jun 23 04:10:27 2026 (r1935564)
@@ -54,6 +54,7 @@ use Mail::SpamAssassin::Message::Node;
use Mail::SpamAssassin::Message::Metadata;
use Mail::SpamAssassin::Constants qw(:sa);
use Mail::SpamAssassin::Logger;
+use Mail::SpamAssassin::Timeout;
our @ISA = qw(Mail::SpamAssassin::Message::Node);
@@ -1268,6 +1269,117 @@ sub _find_part_by_type {
return undef;
}
+###########################################################################
+# MIME-part handler dispatch.
+#
+# apply_handlers() walks every part once, invokes the registered handler for
+# each part's content type, and recursively dispatches any synthetic child
+# parts the handler produces. Handlers inject text via $node->set_rendered and
+# accumulate metadata on $pms; their only return value is an arrayref of
+# child-part specs ({ type, data, name }). Synthetic children live in a
+# separate {handler_parts} array (never {body_parts}), so the MIME tree shape is
+# untouched -- only get_body_text_array_common reads {handler_parts}.
+
+sub apply_handlers {
+ my ($self, $permsgstatus) = @_;
+ return if $self->{handlers_applied}++;
+
+ my $conf = $permsgstatus->{conf};
+ return unless $conf->{handlers} && %{$conf->{handlers}};
+
+ $self->parse_body() if exists $self->{'parse_queue'};
+
+ my $ctx = {
+ permsgstatus => $permsgstatus,
+ bytes_budget => $conf->{handler_max_bytes} || 50_000_000,
+ parts_budget => $conf->{handler_max_parts} || 1000,
+ deadline => $permsgstatus->{master_deadline}, # scan-wide; may be undef
+ part_secs => $conf->{handler_time_limit} || 10,
+ max_depth => $conf->{handler_max_depth} || 8,
+ seen => {}, # content fingerprint -> seen (cycle/dedup guard)
+ };
+
+ # Seed the worklist with every parsed node (containers included). Synthetic
+ # children are appended as they are produced, driving recursion.
+ my @work = map { [$_, 0] } $self->find_parts(qr/./, 0);
+
+ while (my $item = shift @work) {
+ my ($node, $depth) = @$item;
+ last if $ctx->{deadline} && time > $ctx->{deadline}; # undef => no cap
+ last if $ctx->{parts_budget} <= 0;
+ next if $depth > $ctx->{max_depth};
+
+ my $handler =
+ Mail::SpamAssassin::Conf::get_handler_for_type($conf, $node->effective_type);
+ next unless $handler; # [ $plugin_obj, $methodname ]
+
+ my $parts = $self->_invoke_handler($handler, $node, $ctx);
+ next unless $parts && @$parts;
+
+ $self->_attach_parts($node, $parts, $ctx, \@work, $depth);
+ }
+}
+
+# Run one handler method under a timeout and eval, so a hung or dying handler
+# degrades to "no result" and never aborts the scan. Side effects (set_rendered
+# on the node, writes to $pms) have already happened by the time it returns; the
+# only consumed return value is the child-parts arrayref.
+sub _invoke_handler {
+ my ($self, $handler, $node, $ctx) = @_;
+ my ($obj, $method) = @$handler;
+ my $pms = $ctx->{permsgstatus};
+ my $parts;
+ my $t = Mail::SpamAssassin::Timeout->new({ secs => $ctx->{part_secs} });
+ $t->run(sub {
+ eval { $parts = $obj->$method($node, $pms); 1 }
+ or dbg("handler: %s->%s died on %s: %s",
+ ref $obj, $method, $node->{type}, $@);
+ });
+ dbg("handler: %s->%s timed out on %s", ref $obj, $method, $node->{type})
+ if $t->timed_out;
+ return $parts;
+}
+
+# Build a Message::Node for each returned child-part spec, append it to the
+# part's own {handler_parts} array, and queue it for recursive dispatch.
+sub _attach_parts {
+ my ($self, $node, $parts, $ctx, $work, $depth) = @_;
+
+ for my $cp (@$parts) {
+ last if $ctx->{bytes_budget} <= 0;
+ next unless ref $cp eq 'HASH';
+ my $data = ref $cp->{data} ? ${$cp->{data}} : $cp->{data};
+ $ctx->{bytes_budget} -= length($data // '');
+
+ # content fingerprint: identical bytes can't loop forever, and identical
+ # repeated parts are processed once. filename is deliberately excluded.
+ my $key = ($cp->{type} // '') . ':' . length($data // '')
+ . ':' . substr($data // '', 0, 64);
+ next if $ctx->{seen}{$key}++;
+
+ my $child = $self->_synth_node($cp);
+ push @{$node->{handler_parts}}, $child; # separate array, ordered
+ $ctx->{parts_budget}--;
+ push @$work, [ $child, $depth + 1 ]; # drive recursion via worklist
+ }
+}
+
+# Turn a child-part spec into a real (leaf) Message::Node carrying its bytes
+# pre-decoded.
+sub _synth_node {
+ my ($self, $cp) = @_;
+ my $data = ref $cp->{data} ? ${$cp->{data}} : $cp->{data};
+ my $n = Mail::SpamAssassin::Message::Node->new({ normalize => $self->{normalize} });
+ $n->{type} = $cp->{type};
+ $n->{name} = $cp->{name} if defined $cp->{name};
+ $n->{synthetic} = 1;
+ $n->{decoded} = $data; # decode() returns this verbatim
+ $n->{raw} = [ $data ];
+ return $n;
+}
+
+###########################################################################
+
# common code for get_rendered_body_text_array,
# get_visible_rendered_body_text_array, get_invisible_rendered_body_text_array
#
@@ -1298,6 +1410,10 @@ sub get_body_text_array_common {
my @queue = ($self);
my $text = '';
while (my $p = shift @queue) {
+ # Descend into handler-produced synthetic children. They render in document
+ # order, after this part's own text below.
+ my $hparts = $p->{'handler_parts'};
+
if (!$p->is_leaf()) {
if ($preferred_alt && $p->{'type'} eq 'multipart/alternative') {
my $preferred_part = _find_part_by_type($p, $preferred_alt);
@@ -1309,9 +1425,12 @@ sub get_body_text_array_common {
} else {
unshift @queue, @{$p->{'body_parts'}};
}
+ unshift @queue, @$hparts if $hparts;
next;
}
+ unshift @queue, @$hparts if $hparts;
+
my($type, $rnd) = $p->$method_name(); # decode this part
# Only text/* types are rendered ...
if (defined $rnd) {
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Tue Jun 23 01:59:58 2026 (r1935563)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/PerMsgStatus.pm Tue Jun 23 04:10:27 2026 (r1935564)
@@ -2047,6 +2047,13 @@ sub extract_message_metadata {
$self->set_tag('RELAYSEXTERNAL', $self->{relays_external_str});
$self->set_tag('LANGUAGES', $self->{msg}->get_metadata("X-Languages"));
+ # Run MIME-part handlers (OCR, archive expansion, etc.) BEFORE the body text
+ # is assembled and cached below -- handler-injected text must be present when
+ # get_decoded_stripped_body_text_array() builds (and caches) the body array,
+ # and before the URI list is frozen by the first get_uri_detail_list().
+ # apply_handlers() itself no-ops when no handlers are registered.
+ $self->{msg}->apply_handlers($self);
+
# This should happen before we get called, but just in case.
if (!defined $self->{msg}->{metadata}->{html}) {
$self->get_decoded_stripped_body_text_array();
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin.pm Tue Jun 23 01:59:58 2026 (r1935563)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin.pm Tue Jun 23 04:10:27 2026 (r1935564)
@@ -1069,6 +1069,29 @@ sub register_eval_rule {
$self->{main}->{conf}->register_eval_rule ($self, $nameofsub, $ruletype);
}
+=item $plugin-E<gt>register_handler ($mime_pattern, $nameofsub)
+
+Register one of this plugin's methods as the MIME-part handler for a
+content-type pattern. C<$mime_pattern> is an exact type (C<image/jpeg>) or a
+major-type glob (C<image/*>); the most specific match wins. C<$nameofsub> is
+the name of a method on this plugin that will be called as
+C<< $plugin->$nameofsub($node, $permsgstatus) >> for each matching MIME part,
+during message metadata extraction (before body rules run and before the URI
+list is frozen).
+
+The method may inject extracted text into the part with
+C<< $node->set_rendered($text, $type) >>, accumulate per-message findings on
+C<$permsgstatus>, and return an arrayref of synthetic child-part specs
+(C<< { type => ..., data => $bytes, name => ... } >>) which are dispatched
+recursively -- or C<undef>/C<[]> for none.
+
+=cut
+
+sub register_handler {
+ my ($self, $mime_pattern, $nameofsub) = @_;
+ $self->{main}->{conf}->register_handler ($self, $mime_pattern, $nameofsub);
+}
+
=item $plugin-E<gt>register_generated_rule_method ($nameofsub)
In certain circumstances, plugins may find it useful to compile
Added: spamassassin/trunk/t/data/nice/handler_textplain
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/t/data/nice/handler_textplain Tue Jun 23 04:10:27 2026 (r1935564)
@@ -0,0 +1,9 @@
+From: [email protected]
+To: [email protected]
+Subject: Handler test message
+Message-Id: <[email protected]>
+Date: Tue, 10 Jun 2025 12:00:00 +0000
+Content-Type: text/plain; charset=us-ascii
+
+This is the original body text of the message.
+ORIGINAL_BODY_MARKER appears here.
Added: spamassassin/trunk/t/data/testhandler.pm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/t/data/testhandler.pm Tue Jun 23 04:10:27 2026 (r1935564)
@@ -0,0 +1,74 @@
+=head1 testhandler.pm
+
+A minimal plugin used by t/handler.t to exercise the MIME-part handler
+framework end-to-end: text injection, per-message metadata + an eval rule, and
+child-part chaining/recursion. It also exercises registering two different
+handler methods for two different types via register_handler.
+
+To try it out:
+
+ loadplugin myTestHandler ../../../data/testhandler.pm
+ body HANDLER_A /HANDLER_SENTINEL_A/
+ body HANDLER_B /HANDLER_SENTINEL_B/
+ header HANDLER_EVAL eval:check_test_handler()
+
+=cut
+
+package myTestHandler;
+
+use strict;
+use warnings;
+
+use Mail::SpamAssassin::Plugin;
+use Mail::SpamAssassin::Logger;
+
+our @ISA = qw(Mail::SpamAssassin::Plugin);
+
+sub new {
+ my ($class, $main) = @_;
+ $class = ref($class) || $class;
+ my $self = $class->SUPER::new($main);
+ bless ($self, $class);
+
+ # Two handler methods for two types -- proves register_handler's per-method
+ # signature.
+ $self->register_handler('text/plain', 'handle_text');
+ $self->register_handler('application/x-sa-test', 'handle_child');
+
+ $self->register_eval_rule('check_test_handler');
+
+ return $self;
+}
+
+sub handle_text {
+ my ($self, $node, $pms) = @_;
+
+ # Append sentinel A to the part's rendered text and emit a synthetic child of
+ # a second type to drive chaining. Record a per-message flag for the eval
+ # rule to read off $pms (never off $self -- the plugin is a singleton).
+ my (undef, $rnd) = $node->rendered();
+ $rnd = '' unless defined $rnd;
+ $node->set_rendered($rnd . "\nHANDLER_SENTINEL_A\n");
+ $pms->{handlers}{TestHandler}{test_fired} = 1;
+ dbg("handler: testhandler fired on text/plain, emitting child");
+ return [ { type => 'application/x-sa-test',
+ data => "child payload bytes",
+ name => 'child.dat' } ];
+}
+
+sub handle_child {
+ my ($self, $node, $pms) = @_;
+
+ # This part only exists because handle_text emitted it and the dispatcher
+ # re-queued it -- proving chaining/recursion.
+ $node->set_rendered("HANDLER_SENTINEL_B\n");
+ dbg("handler: testhandler fired on chained application/x-sa-test");
+ return [];
+}
+
+sub check_test_handler {
+ my ($self, $pms) = @_;
+ return $pms->{handlers}{TestHandler}{test_fired} ? 1 : 0;
+}
+
+1;
Added: spamassassin/trunk/t/handler.t
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/t/handler.t Tue Jun 23 04:10:27 2026 (r1935564)
@@ -0,0 +1,49 @@
+#!/usr/bin/perl -T
+
+use lib '.'; use lib 't';
+use SATest; sa_t_init("handler");
+
+use Test::More tests => 5;
+
+# ---------------------------------------------------------------------------
+# Exercise the MIME-part handler framework end-to-end:
+# - a plugin registers handle_text for text/plain, which injects
+# HANDLER_SENTINEL_A into the body (proves set_rendered text reaches body
+# rules through the renderer),
+# - it emits a synthetic application/x-sa-test child, dispatched to a second
+# registered method handle_child which injects HANDLER_SENTINEL_B (proves
+# child-part dispatch / chaining / recursion AND per-method registration),
+# - it sets a per-message flag on $pms that the plugin's eval rule reads
+# (proves a plugin can be both a handler and an eval-rule provider).
+
+tstpre ('
+ loadplugin myTestHandler ../../../data/testhandler.pm
+');
+
+tstlocalrules ('
+ body HANDLER_A /HANDLER_SENTINEL_A/
+ score HANDLER_A 1.0
+ describe HANDLER_A handler-injected text reached body rules
+
+ body HANDLER_B /HANDLER_SENTINEL_B/
+ score HANDLER_B 1.0
+ describe HANDLER_B chained handler-injected text reached body rules
+
+ body HANDLER_ORIG /ORIGINAL_BODY_MARKER/
+ score HANDLER_ORIG 1.0
+ describe HANDLER_ORIG original body text is preserved
+
+ header HANDLER_EVAL eval:check_test_handler()
+ score HANDLER_EVAL 1.0
+ describe HANDLER_EVAL handler-owned eval rule fired
+');
+
+%patterns = (
+ ' 1.0 HANDLER_A ', 'text_injected',
+ ' 1.0 HANDLER_B ', 'chained_child_injected',
+ ' 1.0 HANDLER_ORIG ', 'original_body_preserved',
+ ' 1.0 HANDLER_EVAL ', 'handler_eval_rule',
+);
+
+ok (sarun ("-L -t < data/nice/handler_textplain", \&patterns_run_cb));
+ok_all_patterns();