svn commit: r1933166 - spamassassin/trunk/masses/rule-update-score-gen

Author: jhardin
Date: Mon Apr 20 01:06:54 2026
New Revision: 1933166

Log:
Bug #8389

RuleQA: Implement fallback to previously-published scores on masscheck corpus starvation that would previously have completely blocked rule publication. This allows rule bug fixes to be published even in the face of masscheck corpus problems.

Kill switch: Suppress all publication after checking corpus state by creating /usr/local/spamassassin/automc/DISABLE_PUBLISH on sa-vm - hard gate for disaster response.

Code generated with assistance of Claude Sonnet 4.6, adversarial review by Claude Opus 4.7

Added:
   spamassassin/trunk/masses/rule-update-score-gen/extract-scoreset   (contents, props changed)
Modified:
   spamassassin/trunk/masses/rule-update-score-gen/do-nightly-rescore-example.sh
   spamassassin/trunk/masses/rule-update-score-gen/merge-scoresets

Modified: spamassassin/trunk/masses/rule-update-score-gen/do-nightly-rescore-example.sh
==============================================================================
--- spamassassin/trunk/masses/rule-update-score-gen/do-nightly-rescore-example.sh	Sun Apr 19 18:12:03 2026	(r1933165)
+++ spamassassin/trunk/masses/rule-update-score-gen/do-nightly-rescore-example.sh	Mon Apr 20 01:06:54 2026	(r1933166)
@@ -36,20 +36,94 @@ set -e
 
 rm -rf scores scores-set0 scores-set1 scores-set2 scores-set3 stats-set0 stats-set1 stats-set2 stats-set3
 
+score_failure_reason() {
+  case $1 in
+    0) echo "ok" ;;
+    6) echo "insufficient ham contributors" ;;
+    7) echo "insufficient spam contributors" ;;
+    8) echo "insufficient ham message count" ;;
+    9) echo "insufficient spam message count" ;;
+    *) echo "exited with $1" ;;
+  esac
+}
+
+# Proactively check out existing scores from SVN so that:
+#   - per-set fallback scores are available if a scoring run fails
+#   - a fallback REVISION is available if the active scoreset fails
+# This checkout is updated and reused for the final commit.
+svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores trunk-rulesrc-scores
+
+for SET in 0 1 2 3; do
+  $PROGDIR/extract-scoreset trunk-rulesrc-scores/72_scores.cf $SET \
+    > scores-set${SET}-fallback
+done
+
+FALLBACK_REVISION=`svn info trunk-rulesrc-scores | awk '/^Last Changed Rev:/ {print $4}'`
+if [[ -z "$FALLBACK_REVISION" ]]; then
+  echo "Warning: could not determine fallback revision from SVN; fallback revision will be logged as <UNKNOWN>."
+  FALLBACK_REVISION="<UNKNOWN>"
+fi
+
+# Run both scoresets independently, capturing exit codes.
+# set -e is suspended for these calls so that a failure in one does not
+# prevent the other from running.
+set +e
+
 if [[ "$DOW" -eq 0 ]]; then
   echo 'Beginning of Week.  Running with 0 first.'
   $PROGDIR/generate-new-scores.sh 0 $1
+  RC0=$?
   $PROGDIR/generate-new-scores.sh 1 $1
+  RC1=$?
   SCORESET=1
-  REVISION=`grep "revision .*" scores-set$SCORESET | cut -d" " -f9`
 else
   echo 'Not Beginning of Week.  Running with 1 first.'
   $PROGDIR/generate-new-scores.sh 1 $1
+  RC1=$?
   $PROGDIR/generate-new-scores.sh 0 $1
+  RC0=$?
   SCORESET=0
-  REVISION=`grep "revision .*" scores-set$SCORESET | cut -d" " -f9`
 fi
 
+set -e
+
+REASON0=$(score_failure_reason $RC0)
+REASON1=$(score_failure_reason $RC1)
+
+if [[ $RC0 -ne 0 && $RC1 -ne 0 ]]; then
+  echo "Both scoresets failed (set0: $REASON0; set1: $REASON1), aborting."
+  exit 1
+fi
+
+# Emergency off switch: touch this file to block publication regardless of
+# scoring outcome. Placed after the status checks so the log still shows how
+# the scoring runs fared before the halt.
+SENTINEL="/usr/local/spamassassin/automc/DISABLE_PUBLISH"
+if [[ -f "$SENTINEL" ]]; then
+  echo "Publication blocked by $SENTINEL (set0: $REASON0; set1: $REASON1); aborting."
+  exit 1
+fi
+
+# Apply fallback scores for any failed set.
+PARTIAL_NOTE=""
+
+if [[ $RC1 -ne 0 ]]; then
+  echo "Set 1 scoring failed ($REASON1); carrying forward existing scores from SVN revision $FALLBACK_REVISION."
+  cp scores-set1-fallback scores-set1
+  cp scores-set3-fallback scores-set3
+  PARTIAL_NOTE="${PARTIAL_NOTE} [set1 carried forward r$FALLBACK_REVISION: $REASON1]"
+fi
+
+if [[ $RC0 -ne 0 ]]; then
+  echo "Set 0 scoring failed ($REASON0); carrying forward existing scores from SVN revision $FALLBACK_REVISION."
+  cp scores-set0-fallback scores-set0
+  cp scores-set2-fallback scores-set2
+  PARTIAL_NOTE="${PARTIAL_NOTE} [set0 carried forward r$FALLBACK_REVISION: $REASON0]"
+fi
+
+# Fallback files no longer needed; remove before any glob operations.
+rm -f scores-set[0-3]-fallback
+
 echo "Finished generating new scores"
 pwd
 
@@ -57,15 +131,21 @@ pwd
 sed -i -e 's/\b0\.000/0.001 # force non-zero/g' scores-set0
 sed -i -e 's/\b0\.000/0.001 # force non-zero/g' scores-set1
 
-cp scores-set0 scores-set2
-cp scores-set1 scores-set3
-trunk-new-rules-set$SCORESET/masses/rule-update-score-gen/merge-scoresets $SCORESET
+# Copy freshly-generated sets to their bayes counterparts.
+# Skipped if the fallback already populated the bayes set from SVN.
+[[ $RC0 -eq 0 ]] && cp scores-set0 scores-set2
+[[ $RC1 -eq 0 ]] && cp scores-set1 scores-set3
+
+$PROGDIR/merge-scoresets $SCORESET
 cat scores
 
-svn co https://svn.apache.org/repos/asf/spamassassin/trunk/rulesrc/scores trunk-rulesrc-scores
+REVISION=`grep "revision .*" scores-set$SCORESET | cut -d" " -f9`
+[[ -z "$REVISION" ]] && REVISION=$FALLBACK_REVISION
 
+# Update the checkout before committing in case changes landed during scoring.
+svn update trunk-rulesrc-scores
 cp scores trunk-rulesrc-scores/72_scores.cf
-cp scores-set* stats-set* trunk-rulesrc-scores/.
-
-svn ci trunk-rulesrc-scores/ -m "updated scores for revision $REVISION active rules added since last mass-check"
+cp scores-set[0-3] trunk-rulesrc-scores/.
+ls stats-set* >/dev/null 2>&1 && cp stats-set* trunk-rulesrc-scores/.
 
+svn ci trunk-rulesrc-scores/ -m "updated scores for revision $REVISION active rules added since last mass-check${PARTIAL_NOTE}"

Added: spamassassin/trunk/masses/rule-update-score-gen/extract-scoreset
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ spamassassin/trunk/masses/rule-update-score-gen/extract-scoreset	Mon Apr 20 01:06:54 2026	(r1933166)
@@ -0,0 +1,56 @@
+#!/usr/bin/perl
+
+# <@LICENSE>
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at:
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# </@LICENSE>
+
+=head1 NAME
+
+extract-scoreset - extract one scoreset column from 72_scores.cf
+
+=head1 SYNOPSIS
+
+  extract-scoreset 72_scores.cf <0|1|2|3>
+
+=head1 DESCRIPTION
+
+Reads a scores file in the format produced by merge-scoresets (four
+space-separated score values per rule) and emits C<score RULENAME VALUE>
+lines for the requested scoreset column.
+
+Used by do-nightly-rescore-example.sh to produce per-set fallback scores
+from the existing SVN scores when a scoring run fails.
+
+=cut
+
+use strict;
+use warnings;
+
+my ($file, $set) = @ARGV;
+die "usage: extract-scoreset <72_scores.cf> <0|1|2|3>\n"
+    unless defined $file && defined $set && $set =~ /^[0-3]$/;
+
+open(my $fh, '<', $file) or die "Cannot open $file: $!\n";
+while (<$fh>) {
+    next unless /^score\s+(\S+)\s+(.*)/;
+    my ($name, $rest) = ($1, $2);
+    $rest =~ s/\s*#.*//;
+    my @scores = split(' ', $rest);
+    die "$file line $.: expected 4 score values for '$name', got " . scalar(@scores) . ": $_"
+        unless @scores == 4;
+    printf "score %s %s\n", $name, $scores[$set];
+}
+close $fh;

Modified: spamassassin/trunk/masses/rule-update-score-gen/merge-scoresets
==============================================================================
--- spamassassin/trunk/masses/rule-update-score-gen/merge-scoresets	Sun Apr 19 18:12:03 2026	(r1933165)
+++ spamassassin/trunk/masses/rule-update-score-gen/merge-scoresets	Mon Apr 20 01:06:54 2026	(r1933166)
@@ -43,6 +43,9 @@ if (defined $active_rule_set) {
 
 for (my $i = 0; $i < 4; $i++) {
   next if (defined $active_rule_set && $i == $active_rule_set);
+  if (!-f "scores-set$i") {
+    die "scores-set$i not found and no fallback was generated; aborting merge.\n";
+  }
   open (SCORES, "scores-set$i") or die "Cannot open scores-set$i: $!";
   while(<SCORES>) {
     next unless /^score (\S+)\s+(-?[\d.]+)\b/;