Author: gbechis
Date: Wed Oct 22 07:22:26 2025
New Revision: 1929265
Log:
add the possibility to check for redirectors using Selenium
this catches redirects that uses Javascript and other tricks
Modified:
spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Redirectors.pm
spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
spamassassin/trunk/t/redirectors.t
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Redirectors.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Redirectors.pm Wed Oct 22 06:45:02 2025 (r1929264)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Plugin/Redirectors.pm Wed Oct 22 07:22:26 2025 (r1929265)
@@ -85,6 +85,7 @@ use vars qw(@ISA);
my $VERSION = 4.02;
use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; };
+use constant HAS_SELENIUM => eval { require Selenium::Remote::Driver; };
sub dbg { my $msg = shift; return Mail::SpamAssassin::Logger::dbg("Redirectors: $msg", @_); }
sub info { my $msg = shift; return Mail::SpamAssassin::Logger::info("Redirectors: $msg", @_); }
@@ -160,6 +161,60 @@ sub set_config {
=over 4
+=item url_redirector_use_selenium (default: 0)
+
+Use Selenium Chrome driver instead of LWP to access web pages.
+Due to how Selenium works, C<redir_url_code()> and C<redir_url_404> subs will not work.
+Selenium supports only GET requests, even if C<url_redirector> is configured
+to use HEAD requests, GET requests will be sent.
+
+=back
+
+=cut
+
+ push (@cmds, {
+ setting => 'url_redirector_use_selenium',
+ default => 0,
+ is_priv => 1,
+ type => $Mail::SpamAssassin::Conf::CONF_TYPE_BOOL
+ });
+
+=over 4
+
+=item url_redirector_selenium_host (default: 127.0.0.1)
+
+Set Selenium host to use.
+
+=back
+
+=cut
+
+ push (@cmds, {
+ setting => 'url_redirector_selenium_host',
+ default => '127.0.0.1',
+ is_priv => 1,
+ type => $Mail::SpamAssassin::Conf::CONF_TYPE_STRING
+ });
+
+=over 4
+
+=item url_redirector_selenium_port (default: 4444)
+
+Set Selenium port to use.
+
+=back
+
+=cut
+
+ push (@cmds, {
+ setting => 'url_redirector_selenium_port',
+ default => 4444,
+ is_priv => 1,
+ type => $Mail::SpamAssassin::Conf::CONF_TYPE_NUMERIC
+ });
+
+=over 4
+
=item clear_url_redirector [domain] [domain...]
Clear configured url_redirector domains, for example to
@@ -854,13 +909,50 @@ sub _check_redir {
# Initialize cache
$self->initialise_url_redirector_cache($conf);
+ if($conf->{url_redirector_use_selenium} and not HAS_SELENIUM) {
+ dbg("url_redirector_use_selenium setting enabled but Selenium::Remote::Driver Perl module not installed, LWP will be used instead");
+ }
+
+ my $ua;
+ if($conf->{url_redirector_use_selenium} and HAS_SELENIUM) {
+ $ua = Selenium::Remote::Driver->new('remote_server_addr' => $conf->{url_redirector_selenium_host},
+ 'port' => $conf->{url_redirector_selenium_port},
+ 'auto_close' => 0,
+ 'session_id' => $self->{selenium_session_id},
+ 'browser_name' =>'chrome',
+ 'extra_capabilities' => {
+ 'goog:chromeOptions' => {
+ 'args' => [
+ 'headless',
+ 'incognito',
+ 'user-agent=' . $conf->{url_redirector_user_agent}
+ ]
+ }
+ });
+ $ua->{ua}->{max_redirect} = $conf->{max_redir_url_redirections};
+ if(not defined $self->{selenium_session_id}) {
+ $self->{selenium_session_id} = $ua->{session_id};
+ dbg("Connecting to Selenium server with session id " . $self->{selenium_session_id});
+ } else {
+ $ua->session_id($self->{selenium_session_id});
+ dbg("Reusing Selenium session id " . $self->{selenium_session_id});
+ }
+ # Selenium might break when setting timeout
+ eval {
+ $ua->set_timeout('page load', $conf->{url_redirector_timeout} * 1000);
+ };
+ if($@) {
+ dbg("Error setting timeout to $conf->{url_redirector_timeout}");
+ }
+ } else {
# Initialize LWP
- my $ua = LWP::UserAgent->new(
- 'agent' => $conf->{url_redirector_user_agent},
- 'max_redirect' => 0,
- 'timeout' => $conf->{url_redirector_timeout},
- );
- $ua->env_proxy;
+ $ua = LWP::UserAgent->new(
+ 'agent' => $conf->{url_redirector_user_agent},
+ 'max_redirect' => 0,
+ 'timeout' => $conf->{url_redirector_timeout},
+ );
+ $ua->env_proxy;
+ }
# Launch HTTP requests
foreach my $uri (keys %redir_urls) {
@@ -917,46 +1009,83 @@ sub recursive_lookup {
dbg("URL $redir_url is not valid, skipping http check");
return;
}
- my $response = $ua->$method($redir_url);
- if (!$response->is_redirect) {
- dbg("URL is not a redirect: $redir_url = ".$response->status_line);
- my $rcode = $response->code;
- if ($rcode =~ /^\d{3}$/) {
- if($rcode eq 500) {
- if($response->headers->{'client-warning'} eq 'Internal response') {
- dbg("Connection timeout checking $redir_url");
- }
- } elsif($rcode eq 200) {
- if((defined $response->content) and ($response->content =~ /http-equiv=["']?refresh["']?.{1,64}?content=["']?(\d+);\s+url=((?:https?:\/\/)?[^"'\/\\]+)["']?/is)) {
- my $delay = $1;
- $location = $2;
- if($delay eq 0) {
- $rcode = 301;
- } elsif ($delay > 0) {
- $rcode = 302;
- }
- dbg("Found a meta http-equiv redirector, changing http response code from " . $response->code . " to $rcode");
- }
- } else {
- $pms->{"redir_url_$rcode"} = 1;
- # Update cache
- $self->cache_add($redir_url, $rcode);
- $pms->add_uri_detail_list($redir_url) if !$pms->{uri_detail_list}->{$redir_url};
+
+ if($conf->{url_redirector_use_selenium}) {
+ my $rcode;
+ my $newurl = '';
+ # Selenium doesn't support HEAD requests
+ if($method eq 'head') {
+ dbg("HEAD requests are not supported in Selenium, sending a GET request");
+ }
+ eval {
+ $ua->get($redir_url);
+ };
+ if($@) {
+ # Error in Selenium request
+ dbg("Error in Selenium request reading url $redir_url, error $@");
+ return;
+ } else {
+ $newurl = $ua->get_current_url();
+ if($newurl ne $redir_url) {
+ # url has changes, assume it's a redirect 301
+ $rcode = 301;
+ } else {
+ $rcode = 200;
+ dbg("URL is not a redirect: $redir_url = ".$rcode);
}
}
+ $location = $newurl;
+ $pms->{"redir_url_$rcode"} = 1;
+ # Update cache
+ $self->cache_add($redir_url, $rcode);
+ $pms->add_uri_detail_list($redir_url) if !$pms->{uri_detail_list}->{$redir_url};
if($rcode !~ /^30[12]/) {
+ # Calling quit prevents session_id from beeing reused
+ # $ua->quit();
return;
}
- }
+ } else {
+ my $response = $ua->$method($redir_url);
+ if (!$response->is_redirect) {
+ dbg("URL is not a redirect: $redir_url = ".$response->status_line);
+ my $rcode = $response->code;
+ if ($rcode =~ /^\d{3}$/) {
+ if($rcode eq 500) {
+ if($response->headers->{'client-warning'} eq 'Internal response') {
+ dbg("Connection timeout checking $redir_url");
+ }
+ } elsif($rcode eq 200) {
+ if((defined $response->content) and ($response->content =~ /http-equiv=["']?refresh["']?.{1,64}?content=["']?(\d+);\s+url=((?:https?:\/\/)?[^"'\/\\]+)["']?/is)) {
+ my $delay = $1;
+ $location = $2;
+ if($delay eq 0) {
+ $rcode = 301;
+ } elsif ($delay > 0) {
+ $rcode = 302;
+ }
+ dbg("Found a meta http-equiv redirector, changing http response code from " . $response->code . " to $rcode");
+ }
+ } else {
+ $pms->{"redir_url_$rcode"} = 1;
+ # Update cache
+ $self->cache_add($redir_url, $rcode);
+ $pms->add_uri_detail_list($redir_url) if !$pms->{uri_detail_list}->{$redir_url};
+ }
+ }
+ if($rcode !~ /^30[12]/) {
+ return;
+ }
+ }
- # if redirection has been done using http-equiv meta tag, location http header will not be available
- if(exists $response->headers->{location}) {
- $location = $response->headers->{location};
- if($redir_url ne $location) {
- if ($conf->{url_redirector_loginfo}) {
- info("found $redir_url => $location");
- } else {
- dbg("found $redir_url => $location");
+ # if redirection has been done using http-equiv meta tag, location http header will not be available
+ if(exists $response->headers->{location}) {
+ $location = $response->headers->{location};
+ if($redir_url ne $location) {
+ if ($conf->{url_redirector_loginfo}) {
+ info("found $redir_url => $location");
+ } else {
+ dbg("found $redir_url => $location");
+ }
}
}
}
@@ -1087,5 +1216,6 @@ sub has_redir_url_chained { 1 }
sub has_redir_url_chained_domain { 1 }
sub has_redir_url_maxchain { 1 }
sub has_redir_url_loop { 1 }
+sub has_selenium_support { 1 }
1;
Modified: spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm
==============================================================================
--- spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm Wed Oct 22 06:45:02 2025 (r1929264)
+++ spamassassin/trunk/lib/Mail/SpamAssassin/Util/DependencyInfo.pm Wed Oct 22 07:22:26 2025 (r1929265)
@@ -339,6 +339,11 @@ our @OPTIONAL_MODULES = (
desc => 'Mail::DMARC is used by the optional DMARC plugin.',
},
{
+ module => 'Selenium::Remote::Driver',
+ version => 0,
+ desc => 'Selenium::Remote::Driver can be used by the optional Redirectors plugin',
+},
+{
module => 'Devel::Cycle',
version => 0,
desc => 'Devel::Cycle is used in make test in tests that will be harmelessly
Modified: spamassassin/trunk/t/redirectors.t
==============================================================================
--- spamassassin/trunk/t/redirectors.t Wed Oct 22 06:45:02 2025 (r1929264)
+++ spamassassin/trunk/t/redirectors.t Wed Oct 22 07:22:26 2025 (r1929265)
@@ -8,6 +8,7 @@ use Test::More;
use constant HAS_LWP_USERAGENT => eval { require LWP::UserAgent; require LWP::Protocol::https; };
use constant HAS_DBI => eval { require DBI; };
use constant HAS_DBD_SQLITE => eval { require DBD::SQLite; DBD::SQLite->VERSION(1.59_01); };
+use constant HAS_SELENIUM => eval { require Selenium::Remote::Driver; };
use constant SQLITE => (HAS_DBI && HAS_DBD_SQLITE);
@@ -15,6 +16,7 @@ plan skip_all => "Net tests disabled"
plan skip_all => "LWP::Protocol::https required to run this test" unless HAS_LWP_USERAGENT;
my $tests = 4;
$tests += 4 if (SQLITE);
+$tests += 2 if (HAS_SELENIUM);
plan tests => $tests;
tstpre ("
@@ -84,3 +86,22 @@ isnt($row[0], 'https://spamassassin.apac
}
+if(HAS_SELENIUM) {
+tstprefs(q{
+dns_query_restriction allow google.com
+dns_query_restriction allow disq.us
+
+clear_url_redirector
+url_redirector_get google.com
+url_redirector_use_selenium 1
+
+body HAS_REDIR_URL eval:redir_url()
+uri URI_PAGE_LINK m,^http://spamassassin\.apache\.org/news\.html,
+});
+%patterns = (
+ q{ 1.0 HAS_REDIR_URL } => '',
+ q{ 1.0 URI_PAGE_LINK } => '',
+);
+sarun ("-t < data/spam/redirectors/base.eml", \&patterns_run_cb);
+ok_all_patterns();
+}
lmpx.com only provides a reader for public news (NNTP) servers. It is not
affiliated with the servers or forums shown here and is not responsible for
the content of articles, which is written by their respective authors.