svn commit: r1936591 - in spamassassin/trunk: . lib/Mail/SpamAssassin t

Author: fkento
Date: Sun Jul 26 01:38:14 2026
New Revision: 1936591

Log:
Bug 8404: fix NBSP false positives in HTML obfuscation detection

 Add t/html_obfuscation_ratio.t to guard the obfuscation-ratio calculation
 against regressions. Writing it uncovered a new false positive: html_text()
 counts a text node's trailing character and the next node's leading
 character as "obfuscation" when both are word characters (a word split
 across a tag), but its character class treated NBSP as a word character.

 A non-breaking space renders as whitespace and, via the
 <br>-before-closing-block rewrite in html_tag(), is inserted by the parser
 itself, so NBSP-separated runs were wrongly counted as split words. Exclude
 \xa0 from the class, taking care not to confuse it with legitimate text:

   - a leading UTF-8 NBSP is skipped with (?!\xc2\xa0), without excluding
     the shared \xc2 lead byte (© ® ° ...);
   - a trailing \xa0 still counts when it is the tail byte of a multibyte
     letter (à, Š, CJK, ...) rather than an NBSP.

 The word-character class is factored into $wordchar with the leading and
 trailing variants built from it. The test exercises these cases directly
 against Mail::SpamAssassin::HTML in both character-semantics modes.

Added:
   spamassassin/trunk/t/html_obfuscation_ratio.t
Modified:
   spamassassin/trunk/MANIFEST
   spamassassin/trunk/lib/Mail/SpamAssassin/HTML.pm

Modified: spamassassin/trunk/MANIFEST
==============================================================================
--- spamassassin/trunk/MANIFEST	Sun Jul 26 00:00:16 2026	(r1936590)
+++ spamassassin/trunk/MANIFEST	Sun Jul 26 01:38:14 2026	(r1936591)
@@ -650,6 +650,7 @@ t/hashbl.t
 t/html_colors.t
 t/html_nested_anchors.t
 t/html_obfu.t
+t/html_obfuscation_ratio.t
 t/html_utf8.t
 t/html_visibility.t
 t/html_whitespace.t

Modified: spamassassin/trunk/lib/Mail/SpamAssassin/HTML.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/HTML.pm	Sun Jul 26 00:00:16 2026	(r1936590)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/HTML.pm	Sun Jul 26 01:38:14 2026	(r1936591)
@@ -1147,18 +1147,38 @@ sub html_text {
     # 1. using \w or [A-Za-z] instead of \S or non-punctuation
     # 2. exempting certain tags
     # no re "strict";  # since perl 5.21.8: Ranges of ASCII printables...
-    # Bug 8404: \x00 is the paragraph-break marker pushed by html_whitespace() for block
-    # tags (div, p, ...).  It is not in \s, so it is included here explicitly;
-    if ($text =~ /^[^\s\x00\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]/s &&
-	$self->{text}->[-1] =~ /[^\s\x00\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]\z/s)
+    # Bug 8404: $wordchar excludes \s and ASCII punctuation but otherwise treats
+    # everything (incl. non-ASCII bytes) as a "word" character, so the synthetic
+    # marker bytes the parser inserts to represent whitespace/breaks would
+    # wrongly read as word content and make an ordinary block break look like a
+    # word split across a tag.  Exclude them from the class:
+    #   \x00   paragraph-break marker (html_whitespace, div/p/...)
+    #   \xa0   NBSP -- lone \x{a0} (char-semantics) or the trailing byte of
+    #          UTF-8 \xc2\xa0 (octet mode); NBSP renders as whitespace, and
+    #          html_tag() rewrites a <br> before a closing block to NBSP.
+    my $wordchar = qr/[^\s\x00\xa0\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]/;
+
+    # \xa0 is also a legitimate byte in UTF-8 text -- it is the trailing byte of
+    # ~2000 letters (à=\xc3\xa0, Š=\xc5\xa0, 1400+ CJK) -- so excluding it from
+    # $wordchar would stop counting a word split right after one of those.  The
+    # anchors below add that back where it matters:
+    #   - leading (^): a run never *starts* with a bare \xa0 for a real letter
+    #     (that would be its lead byte, \xc2..\xf4), so just skip a leading UTF-8
+    #     NBSP with (?!\xc2\xa0).
+    #   - trailing (\z): a trailing \xa0 is ambiguous, so also accept an \xa0
+    #     that is the tail of a multibyte char (preceded by a UTF-8 continuation
+    #     or non-\xc2 lead byte); the NBSP forms (\xc2\xa0 / lone \x{a0}) are not.
+    my $lead  = qr/(?!\xc2\xa0)$wordchar/;
+    my $trail = qr/(?:$wordchar|(?<=[\x80-\xbf\xc3-\xf4])\xa0)/;
+
+    if ($text =~ /^$lead/s && $self->{text}->[-1] =~ /$trail\z/s)
     {
       $self->{obfuscation}++;
     }
-    if ($self->{text}->[-1] =~
-	/\b([^\s\x00\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]{1,7})\z/s)
+    if ($self->{text}->[-1] =~ /\b($trail{1,7})\z/s)
     {
       my $start = length($1);
-      if ($text =~ /^([^\s\x00\x21-\x2f\x3a-\x40\x5b-\x60\x7b-\x7e]{1,7})\b/s) {
+      if ($text =~ /^($lead{1,7})\b/s) {
 	$self->{backhair}->{$start . "_" . length($1)}++;
       }
     }

Added: spamassassin/trunk/t/html_obfuscation_ratio.t
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ spamassassin/trunk/t/html_obfuscation_ratio.t	Sun Jul 26 01:38:14 2026	(r1936591)
@@ -0,0 +1,80 @@
+#!/usr/bin/perl -T
+use strict;
+use warnings;
+use lib '.'; use lib 't';
+use lib 'lib';
+use Mail::SpamAssassin::HTML;
+use Test::More;
+
+# Bug 8404: the obfuscation counter (HTML.pm html_text) increments when the
+# tail of one text node and the head of the next are both "word" characters,
+# meaning a word looks split across a tag (V<b></b>iagra).  The synthetic
+# marker bytes the parser inserts for whitespace/breaks -- \x00 (block break)
+# and \xa0 (NBSP, incl. the <br>-before-closing-block rewrite) -- are not \s,
+# so they used to read as word chars and made ordinary block breaks and NBSP
+# separated runs look obfuscated (HTML_OBFUSCATE_* false positives).
+
+# triples of (label, HTML, expected obfuscation count)
+my @tests = (
+
+    '<br> counts as whitespace -- no obfuscation',
+    'Foo<br>bar',
+    0,
+
+    'Block elements count as whitespace -- no obfuscation',
+    '<div dir="auto">Bonjour Salomon</div><div dir="auto">Je reviens vers vous</div><div dir="auto">La vente</div>',
+    0,
+
+    '<br> immediately before a closing block -- no obfuscation',
+    '<div>alpha<br></div><div>bravo<br></div><div>charlie</div>',
+    0,
+
+    'Bare NBSP (\xa0) at the end of a text block -- no obfuscation',
+    "foo\xa0<b>bar</b>",
+    0,
+
+    'Bare NBSP (\xa0) at the beginning of a text block -- no obfuscation',
+    "foo<b>\xa0bar</b>",
+    0,
+
+    'A run starting with UTF-8 NBSP (\xc2\xa0) -- no obfuscation',
+    "word<b>\xc2\xa0next</b>",
+    0,
+
+    'A run ending with UTF-8 NBSP (\xc2\xa0) -- no obfuscation',
+    "word\xc2\xa0<b>next</b>",
+    0,
+
+    'Genuine mid-word split -- obfuscation',
+    "Vi<b>a</b>gra",
+    2,
+
+    # \xa0 is also the trailing byte of many UTF-8 letters; a genuine split
+    # right after one must still count, i.e. the NBSP exclusion must not swallow
+    # these letter tails.  Two shapes: a 2-byte char whose lead byte precedes
+    # the \xa0 (à=\xc3\xa0), and a 3-byte char whose continuation byte does
+    # (CJK U+4E20=\xe4\xb8\xa0).
+    'Split after 2-byte UTF-8 letter ending in \xa0 (à) -- obfuscation',
+    "voil\xc3\xa0<b>x</b>",
+    1,
+
+    'Split after 3-byte CJK char ending in \xa0 -- obfuscation',
+    "\xe4\xb8\xa0<b>x</b>",
+    1,
+
+);
+
+plan tests => scalar(@tests) / 3 * 2;   # (@tests/3) cases x 2 semantics modes
+
+while (@tests) {
+    my $label = shift @tests;
+    my $html     = shift @tests;
+    my $expected = shift @tests;
+
+    for my $character_semantics (0,1) {
+        my $obj = Mail::SpamAssassin::HTML->new($character_semantics, 1);
+        $obj->parse($html);
+        is($obj->{obfuscation} || 0, $expected,
+           "[char_semantics=$character_semantics] $label");
+    }
+}