svn commit: r1936530 - in spamassassin/trunk: . lib/Mail/SpamAssassin/Message t

Author: fkento
Date: Thu Jul 23 23:04:51 2026
New Revision: 1936530

Log:
Message::Node: fix BOM'd UTF-16 decoding in _normalize

detect_utf16() returned undef when the data began with a UTF-16 BOM
("let perl figure it out from the BOM"), but its only caller --
_normalize()'s UTF-16 branch -- used solely the decoder it returned.
So BOM'd UTF-16 fell through to the Windows-1252 last resort and was
mangled (the BOM bytes ff fe / fe ff decoded to c3 bf c3 be, with the
interleaved NUL bytes preserved).

Return the BOM-aware UTF-16 decoder from detect_utf16() on a BOM instead
of undef; it consumes the BOM and picks endianness from it. detect_utf16()
now returns undef only when the data does not look like UTF-16 at all.

While here, bound the endianness heuristic to a 1024-byte prefix instead
of scanning the whole part: the check is statistical, so a prefix gives
the same verdict while avoiding the unpack() of a multi-MB body into
per-nibble arrays. The BOM pre-check now tests $_[0] directly (only the
first two bytes matter) rather than copying the body first.

Add t/node_utf16.t: direct unit tests covering detect_utf16() and
_normalize('UTF-16') for BOM'd and BOM-less UTF-16 of either endianness,

Added:
   spamassassin/trunk/t/node_utf16.t
Modified:
   spamassassin/trunk/MANIFEST
   spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm

Modified: spamassassin/trunk/MANIFEST
==============================================================================
--- spamassassin/trunk/MANIFEST	Thu Jul 23 20:10:03 2026	(r1936529)
+++ spamassassin/trunk/MANIFEST	Thu Jul 23 23:04:51 2026	(r1936530)
@@ -671,6 +671,7 @@ t/missing_hb_separator.t
 t/mkrules.t
 t/mkrules_else.t
 t/neuralnetwork.t
+t/node_utf16.t
 t/nonspam.t
 t/olevbmacro.t
 t/originating_ip_hdr.t

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm	Thu Jul 23 20:10:03 2026	(r1936529)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Message/Node.pm	Thu Jul 23 23:04:51 2026	(r1936530)
@@ -450,7 +450,6 @@ sub decode {
 
 # Detect endianness of UTF-16 encoded data
 sub detect_utf16 {
-	my $data = $_[0];  # could not avoid copying large strings
 	my $utf16le_clues = 0;
 	my $utf16be_clues = 0;
 	my $sum_h_e = 0;
@@ -459,12 +458,21 @@ sub detect_utf16 {
 	my $sum_l_o = 0;
 	my $decoder = undef;
 
-	# avoid scan if BOM present
-	if( $data =~ /^(?:\xff\xfe|\xfe\xff)/ ) {
+	# A BOM already declares the encoding and endianness, so skip the heuristic
+	# scan and return the plain UTF-16 decoder, which consumes the BOM and picks
+	# the endianness from it.  Only the first two bytes matter, so test $_[0]
+	# directly rather than copying the (possibly multi-MB) body.
+	if( $_[0] =~ /^(?:\xff\xfe|\xfe\xff)/ ) {
 		dbg( "message: detect_utf16: found BOM" );
-		return;	# let perl figure it out from the BOM
+		return Encode::find_encoding("UTF-16");
 	}
-	
+
+	# The endianness heuristic below is statistical, so a bounded prefix gives the
+	# same verdict as the whole string while avoiding the unpack() of a large body
+	# into per-nibble arrays.  1024 is even, so the slice ends on a UTF-16 pair
+	# boundary.
+	my $data = substr($_[0], 0, 1024);
+
 	my @msg_h = unpack 'H' x length( $data ), $data;
 	my @msg_l = unpack 'h' x length( $data ), $data;
 
@@ -594,6 +602,10 @@ sub _normalize {
     # or declaring endianness as reported at:
     # https://bz.apache.org/SpamAssassin/show_bug.cgi?id=7252
 
+    # detect_utf16() sniffs the endianness of BOM-less UTF-16, and returns the
+    # BOM-aware UTF-16 decoder when a BOM is present.  (It returns undef only when
+    # the data does not look like UTF-16 at all, in which case we fall through to
+    # the guesswork below.)
     my $decoder = detect_utf16( $_[0] );
     if (defined $decoder) {
       if (eval { $rv = $decoder->decode($_[0], Encode::FB_CROAK | Encode::LEAVE_SRC); defined $rv }) {

Added: spamassassin/trunk/t/node_utf16.t
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ spamassassin/trunk/t/node_utf16.t	Thu Jul 23 23:04:51 2026	(r1936530)
@@ -0,0 +1,84 @@
+#!/usr/bin/perl
+use strict;
+use warnings FATAL => 'all';
+use lib 'blib/lib'; use lib '../blib/lib';
+use Test::More;
+use Encode qw();
+use Mail::SpamAssassin::Message::Node;
+
+# ---------------------------------------------------------------------------
+# Direct unit tests for the UTF-16 handling in Mail::SpamAssassin::Message::Node:
+#
+#   * detect_utf16($bytes) - returns an Encode decoder for UTF-16 input
+#     (BOM'd or BOM-less, either endianness).
+#   * _normalize($bytes, 'UTF-16', 0, 0) - transcodes UTF-16 octets to UTF-8
+#     octets via that decoder.
+#
+# Regression guard: detect_utf16() used to return undef when a BOM was present
+# ("let perl figure it out from the BOM"), but its only caller only ever used the
+# decoder it returned -- so BOM'd UTF-16 dropped through to the Windows-1252 last
+# resort in _normalize() and was mangled.  The BOM'd cases below cover that.
+# ---------------------------------------------------------------------------
+
+my $sample = "The quick brown fox jumps over the lazy dog.\n";
+
+my $le_bom   = "\xff\xfe" . Encode::encode('UTF-16LE', $sample);
+my $be_bom   = "\xfe\xff" . Encode::encode('UTF-16BE', $sample);
+my $le_nobom = Encode::encode('UTF-16LE', $sample);
+my $be_nobom = Encode::encode('UTF-16BE', $sample);
+
+my $utf8  = Encode::encode('UTF-8', "caf\x{e9} \x{4e16}\x{754c}\n");
+my $ascii = "plain ascii, no nulls here\n";
+
+plan tests => 6;
+
+# --- detect_utf16: returns the CORRECT decoder for every UTF-16 variant ------
+# The BOM'd cases are the regression guard (before the fix these returned undef).
+# A BOM declares its own endianness, so both BOM'd inputs must resolve to the
+# BOM-aware 'UTF-16' codec -- NOT UTF-16LE/UTF-16BE, which would ignore the BOM
+# and mis-decode the opposite endianness.  BOM-less inputs must resolve to the
+# specific endianness the heuristic sniffed.  We check both the decoder's name
+# and that it actually decodes the sample back to the original text.
+my @detect = (
+  ['BOM LE',    $le_bom,    'UTF-16'],
+  ['BOM BE',    $be_bom,    'UTF-16'],
+  ['no-BOM LE', $le_nobom,  'UTF-16LE'],
+  ['no-BOM BE', $be_nobom,  'UTF-16BE'],
+);
+for my $c (@detect) {
+  my ($name, $bytes, $want) = @$c;
+  subtest "detect_utf16: $name -> $want" => sub {
+    plan tests => 2;
+    my $dec = Mail::SpamAssassin::Message::Node::detect_utf16($bytes);
+    is(defined $dec ? $dec->name : undef, $want,
+       "returns the $want decoder");
+    is(defined $dec ? $dec->decode($bytes, Encode::LEAVE_SRC) : undef, $sample,
+       "that decoder round-trips the sample text");
+  };
+}
+
+# --- _normalize('UTF-16'): every variant transcodes to clean UTF-8 ----------
+# Result must be UTF-8 octets: the ASCII text is contiguous again (a BOM'd input
+# that fell through to Windows-1252 would keep its interleaved NUL bytes and the
+# mangled BOM, so both checks would fail).
+subtest '_normalize decodes all UTF-16 variants to UTF-8' => sub {
+  plan tests => 4;
+  for my $c (['BOM LE', $le_bom], ['BOM BE', $be_bom],
+             ['no-BOM LE', $le_nobom], ['no-BOM BE', $be_nobom]) {
+    my ($name, $bytes) = @$c;
+    my $out = Mail::SpamAssassin::Message::Node::_normalize($bytes, 'UTF-16', 0, 0);
+    ok($out =~ /\QThe quick brown fox\E/ && index($out, "\x00") < 0,
+       "$name decoded (BOM consumed, no NULs)");
+  }
+};
+
+# --- _normalize leaves already-UTF-8 and plain ASCII unchanged --------------
+# Each input is declared with the charset it actually is, so we exercise the
+# matching decode path rather than relying on the "try UTF-8 first" fallback.
+subtest '_normalize leaves UTF-8 / ASCII unchanged' => sub {
+  plan tests => 2;
+  is(Mail::SpamAssassin::Message::Node::_normalize($utf8, 'UTF-8', 0, 0), $utf8,
+     'valid UTF-8 octets unchanged');
+  is(Mail::SpamAssassin::Message::Node::_normalize($ascii, 'us-ascii', 0, 0), $ascii,
+     'plain ASCII unchanged');
+};