svn commit: r1933395 - in spamassassin/trunk: . lib/Mail/SpamAssassin lib/Mail/SpamAssassin/FuzzyHash lib/Mail/SpamAssassin/Plugin t
[email protected] Mon, 27 Apr 2026 16:31:52 -0000
| Newsgroups | gmane.mail.spam.spamassassin.cvs |
|---|---|
| Message-ID | <177730751224.3614805.9071582161050352778@svn03-he-fi> |
Author: gbechis
Date: Mon Apr 27 16:31:51 2026
New Revision: 1933395
Log:
Add FuzzyHash::ZOrder MinHash/LSH near-duplicate body detection
Integrate into HashBL plugin via check_hashbl_bodyfuzzy(), which
queries a DNS blocklist for indexed body digests and fires when
similarity (0-100 Hamming-based score) meets the configured
sim_threshold.
This new rbl type matches on emails similar to known spam messages.
Added:
spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/
spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash.pm
spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/Util.pm
spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/ZOrder.pm
Modified:
spamassassin/trunk/MANIFEST
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/HashBL.pm
spamassassin/trunk/t/hashbl.t
Modified: spamassassin/trunk/MANIFEST
==============================================================================
--- spamassassin/trunk/MANIFEST Mon Apr 27 16:14:42 2026 (r1933394)
+++ spamassassin/trunk/MANIFEST Mon Apr 27 16:31:51 2026 (r1933395)
@@ -49,6 +49,9 @@ lib/Mail/SpamAssassin/Constants.pm
lib/Mail/SpamAssassin/DBBasedAddrList.pm
lib/Mail/SpamAssassin/Dns.pm
lib/Mail/SpamAssassin/DnsResolver.pm
+lib/Mail/SpamAssassin/FuzzyHash.pm
+lib/Mail/SpamAssassin/FuzzyHash/Util.pm
+lib/Mail/SpamAssassin/FuzzyHash/ZOrder.pm
lib/Mail/SpamAssassin/GeoDB.pm
lib/Mail/SpamAssassin/Header.pm
lib/Mail/SpamAssassin/Header/ArcAuthenticationResults.pm
Added: spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash.pm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash.pm Mon Apr 27 16:31:51 2026 (r1933395)
@@ -0,0 +1,52 @@
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </@LICENSE>
+
+package Mail::SpamAssassin::FuzzyHash;
+
+use strict;
+use warnings;
+
+our $VERSION = '1.0';
+
+=head1 NAME
+
+Mail::SpamAssassin::FuzzyHash - Fuzzy-hash similarity detection for SpamAssassin
+
+=head1 DESCRIPTION
+
+This is the root namespace for fuzzy-hash algorithms used by SpamAssassin
+to detect near-duplicate spam messages.
+
+=head1 AVAILABLE MODULES
+
+=over 4
+
+=item L<Mail::SpamAssassin::FuzzyHash::ZOrder>
+
+MinHash-based fuzzy similarity using exactly 4 DNS TXT lookups.
+Detects texts with approximately 90% token overlap efficiently.
+
+=item L<Mail::SpamAssassin::FuzzyHash::Util>
+
+Shared utility functions used by fuzzy-hash implementations:
+C<normalize_tokens()> and C<hamming_distance()>.
+
+=back
+
+=cut
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/Util.pm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/Util.pm Mon Apr 27 16:31:51 2026 (r1933395)
@@ -0,0 +1,102 @@
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </@LICENSE>
+
+package Mail::SpamAssassin::FuzzyHash::Util;
+
+use strict;
+use warnings;
+use Unicode::Normalize qw(NFKC);
+use Exporter 'import';
+
+our $VERSION = '1.0';
+our @EXPORT_OK = qw(normalize_tokens hamming_distance);
+
+=head1 NAME
+
+Mail::SpamAssassin::FuzzyHash::Util - Shared utilities for FuzzyHash algorithms
+
+=head1 SYNOPSIS
+
+ use Mail::SpamAssassin::FuzzyHash::Util qw(normalize_tokens hamming_distance);
+
+ my @tokens = normalize_tokens($text);
+ my $dist = hamming_distance($hex_a, $hex_b);
+
+=head1 FUNCTIONS
+
+=head2 normalize_tokens($text)
+
+Normalises C<$text> and returns a list of unique tokens suitable for
+fuzzy-hash computation.
+
+Steps: ensure UTF-8 flag is set, apply NFKC normalisation (folds
+full-width ASCII spam chars and ligatures), lowercase, insert space
+boundaries around CJK characters, strip anything that is not a Unicode
+letter, digit, or space, collapse whitespace, split and deduplicate.
+
+Returns C<('__empty__')> for texts that produce no tokens after
+normalisation.
+
+=cut
+
+sub normalize_tokens {
+ my ($text) = @_;
+
+ # Set Perl's Unicode flag is set so \p{} properties will work correctly
+ my $str = $text;
+ utf8::decode($str) unless utf8::is_utf8($str);
+
+ # NFKC normalisation folds full-width ASCII spam chars and ligatures
+ my $norm = NFKC(lc $str);
+
+ # Insert space boundaries around each CJK/ideographic character so that
+ # scripts without whitespace word-separators like Chinese
+ # are tokenised character-by-character instead of becoming a single token
+ $norm =~ s/([\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}])/ $1 /g;
+
+ # Strip anything that is not a Unicode letter, digit, or space
+ $norm =~ s/[^\p{L}\p{N} ]/ /g;
+ $norm =~ s/\s+/ /g;
+ $norm =~ s/^\s+|\s+$//g;
+
+ my (%seen, @words);
+ for my $w (split / /, $norm) {
+ push @words, $w if $w ne '' && !$seen{$w}++;
+ }
+ push @words, '__empty__' unless @words;
+ return @words;
+}
+
+=head2 hamming_distance($hex_a, $hex_b)
+
+Returns the Hamming distance (number of differing bits) between two
+equal-length lowercase hex strings. The strings must have even length.
+
+=cut
+
+sub hamming_distance {
+ my ($a, $b) = @_;
+ my $nbytes = length($a) / 2;
+ my $hamming = 0;
+ for my $i (0 .. $nbytes - 1) {
+ my $xor = hex(substr($a, $i*2, 2)) ^ hex(substr($b, $i*2, 2));
+ while ($xor) { $hamming++; $xor &= $xor - 1 }
+ }
+ return $hamming;
+}
+
+1;
Added: spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/ZOrder.pm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/FuzzyHash/ZOrder.pm Mon Apr 27 16:31:51 2026 (r1933395)
@@ -0,0 +1,276 @@
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </@LICENSE>
+
+package Mail::SpamAssassin::FuzzyHash::ZOrder;
+
+use strict;
+use warnings;
+use Digest::MD5 qw(md5);
+use Exporter 'import';
+use Mail::SpamAssassin::FuzzyHash::Util qw(normalize_tokens hamming_distance);
+
+our $VERSION = '1.0';
+our $PROTO = '2';
+our @EXPORT_OK = qw(
+ zorder_digest
+ zorder_compare
+ zorder_dns_keys
+ zorder_body_band_labels
+ zorder_explain
+);
+
+=head1 NAME
+
+FuzzyHash::ZOrder - Fuzzy text similarity via DNS, using exactly 4 queries
+
+=head1 SYNOPSIS
+
+ use FuzzyHash::ZOrder qw(zorder_digest zorder_compare zorder_dns_keys);
+
+ my $query_digest = zorder_digest($incoming_text);
+ for my $key (zorder_dns_keys($incoming_text, 'fbl.example.com')) {
+ for my $stored_digest (dns_txt_lookup($key)) {
+ my $sim = zorder_compare($query_digest, $stored_digest);
+ if ($sim >= 85) { # rule fires }
+ }
+ }
+
+=head1 DESCRIPTION
+
+FuzzyHash::ZOrder detects texts with ~90% token overlap using exactly
+B<4 DNS TXT lookups> followed by a B<local> similarity check via
+C<zorder_compare()>.
+
+=head2 Algorithm
+
+=over 4
+
+=item Normalisation
+
+Lowercase, replace non-alphanumeric with spaces, collapse whitespace,
+deduplicate tokens.
+
+=item 32-bit MinHash signature
+
+32 independent hash functions are applied to the deduplicated token set.
+Each function finds the token with the minimum hash value; the LSB parity
+of that minimum becomes one output bit. The result is a 32-bit signature.
+Each bit agrees between two texts with probability exactly equal
+to their Jaccard similarity.
+
+=item 4 bands x 8 bits
+
+The 32-bit signature is split into 4 consecutive 8-bit bands. Two texts
+share a band key when all 8 bits in that band agree: probability S^8 at
+Jaccard similarity S.
+
+=item Local similarity verification
+
+Each TXT response carries the full 32-bit digest of the indexed string.
+C<zorder_compare()> verifies similarity locally using all 32 bits.
+Unrelated texts will have a score of ~50%.
+
+=back
+
+=cut
+
+# 32 precomputed salts, one per MinHash function.
+# Each salt = first 4 bytes of MD5("mhsalt_N") as big-endian uint32.
+my @SALT = map { unpack('N', substr(md5("mhsalt_$_"), 0, 4)) } 0 .. 31;
+
+use constant C1 => 0x9e3779b9;
+use constant C2 => 0x85ebca6b;
+use constant C3 => 0xc2b2ae35;
+
+# _minhash($text) -> arrayref of 32 bits
+#
+# For each of 32 hash functions: mix each unique token's base hash with
+# the function's salt, keep the minimum, emit LSB parity as one bit.
+# LSB parity of the minimum agrees between two texts with P = Jaccard(A,B).
+
+sub _minhash {
+ my ($text) = @_;
+
+ my @words = normalize_tokens($text);
+
+ # encode each token back before hashing using md5()
+ my @wh = map { my $w = $_; utf8::encode($w); unpack('N', substr(md5($w), 0, 4)) } @words;
+
+ my @bits;
+ for my $k (0 .. 31) {
+ my $salt = $SALT[$k];
+ my $min = 0xFFFFFFFF;
+ for my $w (@wh) {
+ my $h = (($w ^ $salt) * C1) & 0xFFFFFFFF;
+ $h ^= ($h >> 16);
+ $h = ($h * C2) & 0xFFFFFFFF;
+ $h ^= ($h >> 13);
+ $h = ($h * C3) & 0xFFFFFFFF;
+ $h ^= ($h >> 16);
+ $min = $h if $h < $min;
+ }
+ push @bits, ($min ^ ($min >> 1)) & 1;
+ }
+ return \@bits;
+}
+
+# _bits_to_hex(\@bits) -> 8-char hex (4 bytes)
+sub _bits_to_hex {
+ my ($bits) = @_;
+ my $hex = '';
+ for my $i (0 .. 3) {
+ my $byte = 0;
+ $byte |= ($bits->[$i * 8 + $_] << (7 - $_)) for 0 .. 7;
+ $hex .= sprintf('%02x', $byte);
+ }
+ return $hex;
+}
+
+# _band_keys(\@bits, $domain) -> list of 4 FQDNs
+# Format: "fz2BVV.domain" fz=algo, 2=version, B=band 0-3, VV=byte 00-ff
+sub _band_keys {
+ my ($bits, $domain) = @_;
+ my @keys;
+ for my $b (0 .. 3) {
+ my $val = 0;
+ $val |= ($bits->[$b * 8 + $_] << (7 - $_)) for 0 .. 7;
+ push @keys, sprintf('fz' . $PROTO . '%d%02x.%s', $b, $val, $domain);
+ }
+ return @keys;
+}
+
+=head1 FUNCTIONS
+
+=head2 zorder_digest($text)
+
+Computes the 32-bit MinHash signature of C<$text>.
+Returns an 8-character lowercase hex string.
+
+This is the value stored in each DNS TXT record at index time and used
+for local similarity verification at query time.
+
+=cut
+
+sub zorder_digest {
+ my ($text) = @_;
+ return _bits_to_hex(_minhash($text));
+}
+
+=head2 zorder_dns_keys($text, $domain)
+
+Returns a list of exactly B<4 DNS FQDNs> for C<$text> under C<$domain>.
+
+Use for both indexing (store digest as TXT at each key) and querying
+(look up each key, verify TXT responses locally with zorder_compare).
+
+=cut
+
+sub zorder_dns_keys {
+ my ($text, $domain) = @_;
+ return _band_keys(_minhash($text), $domain);
+}
+
+=head2 zorder_body_band_labels($text)
+
+Returns a list of exactly B<4 bare band labels> without a trailing domain,
+each of the form C<fz2BVV>.
+
+=cut
+
+sub zorder_body_band_labels {
+ my ($text) = @_;
+ my $bits = _minhash($text);
+ my @labels;
+ for my $b (0 .. 3) {
+ my $val = 0;
+ $val |= ($bits->[$b * 8 + $_] << (7 - $_)) for 0 .. 7;
+ push @labels, sprintf('fz' . $PROTO . '%d%02x', $b, $val);
+ }
+ return @labels;
+}
+
+=head2 zorder_compare($digest_a, $digest_b)
+
+Compares two 8-char hex digests. Returns integer similarity 0-100,
+computed as C<100 * (1 - hamming_distance / 32)>.
+
+ 100 = identical token sets (0 bits differ)
+ 91 = ~3 bits differ -> very high overlap
+ 85 = ~5 bits differ -> high overlap
+ 75 = ~8 bits differ -> moderate overlap
+ 50 = ~16 bits differ -> noise floor for unrelated texts
+
+The noise floor at ~50% provides a clean gap below the suggested 90% threshold,
+so coincidental DNS band hits from unrelated indexed strings are reliably
+rejected without any additional DNS queries.
+
+=cut
+
+sub zorder_compare {
+ my ($a, $b) = @_;
+ die "zorder_compare: digest A must be 8 hex chars (got " . length($a) . ")\n"
+ unless length($a) == 8;
+ die "zorder_compare: digest B must be 8 hex chars (got " . length($b) . ")\n"
+ unless length($b) == 8;
+ return int((1.0 - hamming_distance($a, $b) / 32.0) * 100 + 0.5);
+}
+
+=head2 zorder_explain($digest_a, $digest_b)
+
+Returns a human-readable breakdown: Hamming distance, similarity score,
+shared band count, firing decision, and a per-band bit-agreement table.
+Used only for debugging RBL entries and tuning the similarity threshold.
+
+=cut
+
+sub zorder_explain {
+ my ($digest_a, $digest_b) = @_;
+
+ my $hamming = hamming_distance($digest_a, $digest_b);
+ my $sim = int((1 - $hamming / 32) * 100 + 0.5);
+
+ my $shared = 0;
+ for my $b (0 .. 3) {
+ $shared++ if hex(substr($digest_a, $b*2, 2)) == hex(substr($digest_b, $b*2, 2));
+ }
+
+ my $out = '';
+ $out .= "Digest A : $digest_a\n";
+ $out .= "Digest B : $digest_b\n";
+ $out .= sprintf("Hamming dist : %d / 32 bits differ\n", $hamming);
+ $out .= sprintf("Similarity : %d%%\n", $sim);
+ $out .= sprintf("Shared bands : %d / 4 (DNS hits at query time)\n", $shared);
+ $out .= sprintf("Fires rule : %s (local sim %d%% %s threshold 85%%)\n\n",
+ $sim >= 95 ? 'YES' : 'no', $sim, $sim >= 95 ? '>=' : '<');
+ $out .= sprintf(" %-6s %-10s %-10s %-14s %s\n",
+ 'Band', 'A (bits)', 'B (bits)', 'Agreement', 'DNS key match?');
+ $out .= " " . "-" x 58 . "\n";
+ for my $b (0 .. 3) {
+ my $ba = hex(substr($digest_a, $b*2, 2));
+ my $bb = hex(substr($digest_b, $b*2, 2));
+ my $xor = $ba ^ $bb;
+ my $diff = 0; my $x = $xor; while ($x) { $diff++; $x &= $x-1 }
+ my $ok = 8 - $diff;
+ $out .= sprintf(" band%d %08b %08b %s %d/8 bits %s\n",
+ $b, $ba, $bb,
+ ('|' x $ok) . ('.' x $diff), $ok,
+ $ba == $bb ? 'yes (TXT returned)' : 'no');
+ }
+ return $out;
+}
+
+1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/HashBL.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/HashBL.pm Mon Apr 27 16:14:42 2026 (r1933394)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/HashBL.pm Mon Apr 27 16:31:51 2026 (r1933395)
@@ -55,6 +55,15 @@ HashBL - query hashed (and unhashed) DNS
# Query the tag value as is from a DNSBL
header HASHBL_TAG eval:check_hashbl_tag('idbl.example.invalid/A', 'raw', 'XSOMEID', '^127\.')
+ # Fuzzy body hash lookup
+ # Issues exactly 4 DNS TXT lookups. Each TXT response carries the digest
+ # of the indexed string; similarity is verified locally via zorder_compare().
+ # sim_threshold sets the required similarity percentage.
+ #
+ body HASHBL_FUZZY eval:check_hashbl_bodyfuzzy('fbl.example.invalid/TXT', 'sim_threshold=90')
+ describe HASHBL_FUZZY Message body is similar to a known spam template
+ tflags HASHBL_FUZZY net
+
=head1 DESCRIPTION
This plugin supports multiple types of hashed or unhashed DNS blocklist queries.
@@ -225,6 +234,37 @@ Specific mime types can be skipped with
=over 4
+=item body RULE check_hashbl_bodyfuzzy('bl.example.invalid/TXT', 'OPTS')
+
+Computes a 32-bit MinHash signature of the full message body, splits it
+into 4 LSH bands of 8 bits each, and issues exactly B<4 DNS TXT lookups>
+of the form:
+
+ fz2BVV.<list> IN TXT "<8-char-digest>"
+
+where C<fz2> is the algorithm+version prefix, C<B> is the band index (0-3),
+and C<VV> is the 8-bit band value in hex.
+
+Each TXT response carries the 8-char hex digest of the indexed string.
+The rule fires when C<zorder_compare(query_digest, stored_digest)> meets
+the configured C<sim_threshold>.
+
+Supported OPTS:
+
+ sim_threshold=N integer 50-100, similarity % required to fire (default: 95)
+
+Example rules at different thresholds:
+
+ # Near-identical bodies only
+ body FUZZ_100 eval:check_hashbl_bodyfuzzy('fbl.example.invalid/TXT', 'sim_threshold=100')
+
+ # Near-duplicates (~90%+ token overlap)
+ body FUZZ_90 eval:check_hashbl_bodyfuzzy('fbl.example.invalid/TXT', 'sim_threshold=90')
+
+=back
+
+=over 4
+
=item hashbl_ignore value [value...]
Skip any type of query, if either the hash or original value (email for
@@ -250,6 +290,7 @@ use Digest::SHA qw(sha1_hex sha256);
use Mail::SpamAssassin::Plugin;
use Mail::SpamAssassin::Constants qw(:ip);
+use Mail::SpamAssassin::FuzzyHash::ZOrder qw(zorder_digest zorder_body_band_labels zorder_compare);
use Mail::SpamAssassin::Util qw(compile_regexp is_fqdn_valid reverse_ip_address
base32_encode);
@@ -278,6 +319,7 @@ sub new {
'check_hashbl_bodyre' => $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS,
'check_hashbl_tag' => $Mail::SpamAssassin::Conf::TYPE_HEAD_EVALS,
'check_hashbl_attachments' => $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS,
+ 'check_hashbl_bodyfuzzy' => $Mail::SpamAssassin::Conf::TYPE_BODY_EVALS,
};
while (my ($func, $type) = each %{$self->{evalfuncs}}) {
$self->register_eval_rule($func, $type);
@@ -1066,6 +1108,218 @@ sub check_hashbl_attachments {
return; # return undef for async status
}
+=head2 check_hashbl_bodyfuzzy($list, $opts_str, $subtest)
+
+Issues exactly B<4> async DNS TXT lookups for the message body using the
+ZOrder MinHash+LSH scheme (4 bands x 8 bits, 32-bit digest).
+
+Each TXT response carries the 8-char hex digest of the indexed string.
+Similarity is verified locally: C<zorder_compare(query_digest, stored_digest)>
+must be >= C<sim_threshold> for the rule to fire.
+
+=cut
+
+sub check_hashbl_bodyfuzzy {
+ my ($self, $pms, $bodyref, $list, $opts_str, $subtest) = @_;
+
+ return 0 if !$self->{hashbl_available};
+ return 0 if !$pms->is_dns_available();
+
+ my $rulename = $pms->get_current_eval_rule_name();
+
+ if (!defined $list) {
+ warn "HashBL: $rulename blocklist argument missing\n";
+ return 0;
+ }
+
+ my $opts = _parse_opts($opts_str || 'sim_threshold=95');
+
+ # Validate sim_threshold: must be an integer in a useful range
+ my $sim_threshold = int($opts->{sim_threshold});
+ if ($sim_threshold < 50 || $sim_threshold > 100) {
+ warn "HashBL: $rulename sim_threshold=$sim_threshold out of range (50-100)\n";
+ return;
+ }
+
+ my $body_text;
+ if (ref $bodyref eq 'ARRAY') {
+ $body_text = join(' ', @$bodyref)
+ } elsif (ref $bodyref eq 'SCALAR') {
+ $body_text = $$bodyref
+ } else {
+ $body_text = $bodyref // '';
+ }
+
+ if (length($body_text) < 64) {
+ dbg("$rulename: body too short, skipping");
+ return 0;
+ }
+
+ my ($list_domain, $qtype) = split(m{/}, $list, 2);
+ $qtype = uc($qtype || 'TXT');
+ $qtype = 'TXT' if $qtype eq '';
+ if ($qtype ne 'TXT') {
+ dbg("$rulename: warning - fuzzyhash lookups are TXT; got $qtype");
+ }
+
+ my @labels = Mail::SpamAssassin::FuzzyHash::ZOrder::zorder_body_band_labels($body_text);
+ my $qdigest = Mail::SpamAssassin::FuzzyHash::ZOrder::zorder_digest($body_text);
+
+ dbg("$rulename: body qdigest=$qdigest sim_threshold=$sim_threshold%% list=$list_domain");
+ dbg("$rulename: band labels: " . join(', ', @labels));
+
+ my $state_key = "hashbl_fuzzy:$rulename";
+ $pms->{$state_key} ||= {
+ pending => scalar(@labels),
+ query_digest => $qdigest,
+ sim_threshold => $sim_threshold,
+ fired => 0,
+ best_sim => 0,
+ best_stored => undef,
+ };
+ my $state = $pms->{$state_key};
+
+ # Register in per-message fuzzy tracker for cross-rule highest-sim selection
+ $pms->{hashbl_fuzzy_tracker} ||= {};
+ $pms->{hashbl_fuzzy_tracker}{$rulename} = $state_key;
+
+ my $launched = 0;
+ for my $label (@labels) {
+ my $host = "$label.$list_domain";
+ my $key = "hashbl_fuzzy:$qtype:$host";
+
+ dbg("$rulename: querying $host");
+
+ my $ent = {
+ rulename => $rulename,
+ type => 'HASHBL-FUZZY-TXT',
+ zone => $list_domain,
+ key => $key,
+ };
+
+ $ent = $pms->{async}->bgsend_and_start_lookup(
+ $host, $qtype, undef, $ent,
+ sub {
+ my ($ent2, $pkt) = @_;
+ $self->_finish_fuzzy_lookup($pms, $ent2, $pkt, $subtest, $state_key);
+ },
+ master_deadline => $pms->{master_deadline},
+ );
+
+ $launched++ if defined $ent;
+ }
+
+ if (!$launched) {
+ dbg("$rulename: no queries launched");
+ return 0;
+ }
+ return;
+}
+
+sub _finish_fuzzy_lookup {
+ my ($self, $pms, $ent, $pkt, $subtest, $state_key) = @_;
+
+ my $rulename = $ent->{rulename};
+ my $state = $pms->{$state_key};
+ return unless $state;
+
+ $state->{pending}--;
+
+ if ($pkt) {
+ foreach my $answer ($pkt->answer) {
+ next unless $answer && $answer->type eq 'TXT';
+ next if $state->{fired};
+
+ my $rdatastr = $answer->rdstring;
+ $rdatastr =~ s/^"|"$//g;
+ $rdatastr =~ s/^\s+|\s+$//g;
+
+ # TXT format: "<8-char-digest>" or "<digest>:<annotation>"
+ my ($stored_digest) = split(/:/, $rdatastr, 2);
+ next unless defined $stored_digest
+ && length($stored_digest) == 8
+ && $stored_digest =~ /^[0-9a-f]{8}$/i;
+
+ if (defined $subtest && $rdatastr !~ /$subtest/) {
+ dbg("$rulename: fuzzy TXT '$rdatastr' did not match subtest /$subtest/");
+ next;
+ }
+
+ $stored_digest = lc $stored_digest;
+
+ # Local similarity verification, uses all 32 bits of both digests.
+ # Corroboration here is semantic (bit-level agreement across the full
+ # MinHash signature).
+ my $sim = eval {
+ Mail::SpamAssassin::FuzzyHash::ZOrder::zorder_compare($state->{query_digest}, $stored_digest)
+ };
+ if ($@) {
+ dbg("$rulename: zorder_compare error: $@");
+ next;
+ }
+
+ my $sim_threshold = $state->{sim_threshold};
+
+ dbg("$rulename: fuzzy candidate - stored=$stored_digest "
+ . "sim=$sim% (threshold=$sim_threshold%)");
+
+ if ($sim >= $sim_threshold && $sim > $state->{best_sim}) {
+ $state->{best_sim} = $sim;
+ $state->{best_stored} = $stored_digest;
+ }
+ }
+ }
+
+ # When this rule's lookups are all done, check if all fuzzy rules are done
+ if ($state->{pending} <= 0) {
+ $self->_resolve_fuzzy_rules($pms);
+ }
+}
+
+sub _resolve_fuzzy_rules {
+ my ($self, $pms) = @_;
+ my $tracker = $pms->{hashbl_fuzzy_tracker};
+ return unless $tracker;
+
+ # Check if all registered fuzzy rules have completed their lookups
+ for my $state_key (values %$tracker) {
+ my $state = $pms->{$state_key};
+ return if $state && $state->{pending} > 0;
+ }
+
+ # All done, find the rule with the highest sim_threshold that matched
+ my ($best_rule, $best_threshold) = (undef, 0);
+ while (my ($rulename, $state_key) = each %$tracker) {
+ my $state = $pms->{$state_key};
+ next unless $state && defined $state->{best_stored};
+ if ($state->{sim_threshold} > $best_threshold) {
+ $best_threshold = $state->{sim_threshold};
+ $best_rule = $rulename;
+ }
+ }
+
+ # Fire only the most restrictive rule, mark all others ready
+ while (my ($rulename, $state_key) = each %$tracker) {
+ my $state = $pms->{$state_key};
+ next if $state->{fired};
+ if (defined $best_rule && $rulename eq $best_rule) {
+ $state->{fired} = 1;
+ dbg("$rulename: best fuzzy match sim=$state->{best_sim}%");
+ $pms->rule_ready($rulename);
+ $pms->test_log("sim=$state->{best_sim}%", $rulename);
+ $pms->got_hit($rulename, "HashBL-Fuzzy: ",
+ ruletype => 'body',
+ );
+ } else {
+ dbg("$rulename: no fuzzy match (best match was " . ($best_rule // 'none') . ")");
+ $pms->rule_ready($rulename);
+ }
+ }
+
+ # Prevent re-processing
+ delete $pms->{hashbl_fuzzy_tracker};
+}
+
sub _hash {
my ($self, $opts, $value) = @_;
@@ -1158,5 +1412,6 @@ sub has_hashbl_attachments { 1 }
sub has_hashbl_email_domain { 1 } # user/host/domain option for emails
sub has_hashbl_email_domain_alias { 1 } # hashbl_email_domain_alias
sub has_hashbl_alldomains { 1 }
+sub has_hashbl_fuzzyhash { 1 }
1;
Modified: spamassassin/trunk/t/hashbl.t
==============================================================================
--- spamassassin/trunk/t/hashbl.t Mon Apr 27 16:14:42 2026 (r1933394)
+++ spamassassin/trunk/t/hashbl.t Mon Apr 27 16:31:51 2026 (r1933395)
@@ -9,7 +9,7 @@ plan skip_all => "Can't use Net::DNS Saf
# run many times to catch some random natured failures
my $iterations = 5;
-plan tests => 14 * $iterations;
+plan tests => 17 * $iterations;
# ---------------------------------------------------------------------------
@@ -24,9 +24,12 @@ plan tests => 14 * $iterations;
q{ 1.0 META_HASHBL_EMAIL } => '',
q{ 1.0 META_HASHBL_BTC } => '',
q{ 1.0 META_HASHBL_URI } => '',
+ q{ 1.0 X_HASHBL_FUZZY_STRICT } => '',
+ q{ 1.0 META_HASHBL_FUZZY } => '',
);
%anti_patterns = (
q{ 1.0 X_HASHBL_SHA256 } => '',
+ q{ 1.0 X_HASHBL_FUZZY_LAX } => '',
q{ warn: } => '',
);
@@ -50,6 +53,10 @@ host.domain.com.hashbltest7.spamassassin
domain.com.hashbltest7.spamassassin.org
2qlyngefopecg66lt6pwfpegjaajbzasuxs5vzgii2vfbonj6rua.hashbltest8.spamassassin.org
11231234567.hashbltest9.spamassassin.org
+fz20d9.hashbltest10.spamassassin.org
+fz2142.hashbltest10.spamassassin.org
+fz220b.hashbltest10.spamassassin.org
+fz2337.hashbltest10.spamassassin.org
);
sub check_queries {
@@ -62,7 +69,7 @@ sub check_queries {
while (<WL>) {
my $line = $_;
print STDERR $line if $line =~ /warn:/;
- while ($line =~ m,([^\s/]+\.hashbltest\d\.spamassassin\.org)\b,g) {
+ while ($line =~ m,([^\s/]+\.hashbltest\d+\.spamassassin\.org)\b,g) {
my $query = $1;
if (!grep { $query eq $_ } @valid_queries) {
$invalid{$query}++;
@@ -150,9 +157,17 @@ tstlocalrules(q{
header X_HASHBL_ALIAS_NODOT eval:check_hashbl_emails('hashbltest8.spamassassin.org', 'sha256/nodot', 'body', '^127\.', 'domaincom')
tflags X_HASHBL_ALIAS_NODOT net
+ # check_hashbl_bodyfuzzy
+ body X_HASHBL_FUZZY_STRICT eval:check_hashbl_bodyfuzzy('hashbltest10.spamassassin.org', 'sim_threshold=95')
+ tflags X_HASHBL_FUZZY_STRICT net
+
+ body X_HASHBL_FUZZY_LAX eval:check_hashbl_bodyfuzzy('hashbltest10.spamassassin.org', 'sim_threshold=85')
+ tflags X_HASHBL_FUZZY_LAX net
+
# Bug 7897 - test that meta rules depending on net rules hit
meta META_HASHBL_EMAIL X_HASHBL_EMAIL
# It also needs to hit even if priority is lower than dnsbl (-100)
+ meta META_HASHBL_FUZZY X_HASHBL_FUZZY_STRICT
meta META_HASHBL_BTC X_HASHBL_BTC
priority META_HASHBL_BTC -500
# Or super high