Re: read articles w/o charset declaration

% $Hg: guess-charset.sl,v d491fa607a28 Sat Sep 16 01:25:31 2006 +0200 $

%%% Credits %%%
% Alain Bench for locale_charset script,
%   enlightening input about charsets,
%   discrete correction of blatant mistakes,
%   and a recipe to get loads of virtual EUROs
% Thomas Schultz for preference mechanism

% documentation {{{1
#iffalse
  Description:

Some clients do not bother to declare a charset even when the
message contains non-ascii chars. This script provides a hook
that tries to guess a fallback charset, relying on a configurable
list of charsets. It then replaces the article in question with a
converted version from a (hopefully) fitting fallback charset to
local charset.

  Requirements:

S-Lang version >= 2, with pcre plugin; Iconv.

A little script by Alain Bench to detect local charset:

|/* Prints the portable name for the current locale's charset.
| * Build with gcc -o locale_charset locale_charset.c -lcharset
| * to catch /usr/local/lib/libcharset.so.1
| * Or -liconv to catch /usr/local/lib/libiconv.so.2
| * Or -lintl  to catch /usr/local/lib/libintl.so.2
| */
|
|#include <stdio.h>
|#include <stdlib.h>
|#include <locale.h>
|#include <localcharset.h>
|
|int main ()
|{
|  setlocale(LC_ALL, "");
|  printf("%s\n", locale_charset());
|  exit(0);
|}

Compile and put a copy in your $PATH.

  Usage:

Check and set preferences (see below).

Put iconv_article_hook() in a read_article_hook, either by
removing the comments in the example at end of this file or:

public define read_article_hook ()
{
  % ...;
  GuessCharset->iconv_article_hook ();
}

Create eg. fallback_charset_hook() to have different fallback
charsets for varying purposes; small example:

static define fallback_charset_hook ()
{
  if (0 == strncmp (current_newsgroup (), "cz.", 3))
    GuessCharset->set_preference ("fallback_charsets", "utf-8,cp1250");
  else
    GuessCharset->set_preference ("fallback_charsets", "utf-8,cp1252");
}

Bind a key to call query_fallback_charsets() by uncommenting example
at end of this file, or in your slrnrc:

setkey article GuessCharset->query_fallback_charsets "^A"

  Preferences:

Use the set_preference() function to customize the behaviour of
the macro. Put calls to this function in a file and load it after
this file is loaded. Examples that show the default values:

GuessCharset->set_preference ("fallback_charsets", "utf-8,cp1252");

  Charsets to fallback on if no charset declared as comma or
  space (tolerantly) separated string.

GuessCharset->set_preference ("nntplocal", "/var/spool/news");

  Set "nntplocal" to location of your local news spool.  These
  scripts are a bit faster and less noisy if they can read
  directly from spool.

  Also raw_article_as_string() is less reliable and doesn't work
  if replace_article() has been called before on same article.
  
  If local spool is not readable by you (leafnode-2), it will be
  reset to the empty string "" automatically.
#endif
% }}}1

implements ("GuessCharset");

% variables {{{1
private variable
    Prefs                  = Assoc_Type [],
    fallback_charset_array = String_Type [],  % fallback charsets as list
    iconv_array            = String_Type [],  % list of charsets supported by iconv
    last_exit,      % in case every other charset fails
    iconv,          % full path to iconv executable
    locale_charset, % local charset
    tf_root;        % root of tempfile name

% set defaults
Prefs["fallback_charsets"] = "utf-8,cp1252";
Prefs["nntplocal"] = "/var/spool/news";

% system interaction {{{1
private define pipe_readline (syscall, err)
{
  variable out, fp = popen (syscall, "r");
  if (fp == NULL or -1 == fgets (&out, fp))
    throw RunTimeError, err;
  () = pclose (fp);
  return strtrim_end (out);
}

private define pipe_read (syscall)
{
  % returns stdout of system call `syscall'
  variable fp = popen (syscall, "r");
  if (fp == NULL)
    verror ("%s: failed", syscall);
  variable out = strjoin (fgetslines (fp), "");
  () = pclose (fp);
  return out;
}

private define tmpfile_write (s)
{
  % writes string `s' to mildly secure tempfile `tf'
  % and returns path `tf'
  variable
      tf = sprintf ("%s%i.%i", tf_root, _time (), getpid ()),
      fp = fopen (tf, "w");
  if (fp == NULL)
    verror ("could not write to: %s", tf);
  () = fputs (s, fp);
  () = fclose (fp);
  return tf;
}

% control charset preference {{{1
private define cycle_iconv_array (charset)
{
  % cycles thru list of supported charsets
  % looking for `charset'
  % returns `charset' on success, NULL otherwise
  variable cs;
  foreach cs (iconv_array)
  {
    if (strlow (cs) == strlow (charset))
      return charset;
  }
  return NULL;
}

private define set_fallback_charsets (charsets)
{
  % can be used to set $fallback_charsets eg. in hooks
  variable charset_array = strtok (charsets, "\\s,"), cs;
  foreach cs (charset_array)
  {
    if (NULL == cycle_iconv_array (cs))
      verror ("%s: not supported by %s", cs, iconv);
  }
  Prefs["fallback_charsets"] = strjoin (charset_array, ",");
  fallback_charset_array = @charset_array,
  last_exit = charset_array[-1];
}

% read spool if possible, otherwise unset $nntplocal on 1. run {{{1
private define path_to_article_as_file ()
{
  % returns path to article in news spool if spool is readable
  % otherwise write article to tempfile and return path
  if (Prefs["nntplocal"] == "")
    return tmpfile_write (raw_article_as_string());
  variable
      xref = extract_displayed_article_header ("xref");
  if (xref != "")
  {
    xref = strtok (xref);
    % -> ["host.name", "alt.lang.s-lang:42", "alt.astrology:839"]
    variable
        xref_len = length (xref), i, path, st;
    for (i = 1; i < xref_len; i++)
    {
      path = strcat (Prefs["nntplocal"], "/", strtrans (xref[i], ".:", "/"));
      % -> "/var/spool/news/alt/lang/s-lang/42"
      st = stat_file (path);
      if (st != NULL)
      {
        if (NULL != stat_is ("reg", st.st_mode))
          return path;
      }
    }
  }
  % silently unset $nntplocal if first run fails
  Prefs["nntplocal"] = "";
  return tmpfile_write (raw_article_as_string());
}

% init: variables, tools {{{1
% detect locale
locale_charset = pipe_readline ("locale_charset",
    "failed to detect local charset");

% initialize full path to iconv
iconv = pipe_readline ("which iconv", "iconv not found");

% root of temporary file name
tf_root = getenv ("TMPDIR");
if (tf_root == NULL)
  tf_root = "/tmp";
tf_root = strcat (tf_root, "/slrn_tmp.");

% get all charsets supported by iconv as list
iconv_array = strtok (pipe_read (sprintf ("%s -l", iconv)));

% initialize fallback_charset_array
set_fallback_charsets (Prefs["fallback_charsets"]);

% preference interface {{{1
static define set_preference (preference, value)
{
  !if (assoc_key_exists (Prefs, preference))
    verror ("no such preference: %s", preference);
  variable
      desired_type = typeof (Prefs[preference]), 
      val_type = typeof (value);
  if (val_type != desired_type)
    verror ("%s: wants %s, not %s", preference, desired_type, val_type);
  if (preference == "fallback_charsets")
    set_fallback_charsets (value);
  else
    Prefs[preference] = value;
}

static define query_fallback_charsets ()
{
  % sets $fallback_charsets + $fallback_charset_array interactively
  variable
      charsets = "", charset_array = String_Type [], cs;
  while (charsets == "")
  {
    charsets = read_mini ("Enter fallback charsets: ",
        Prefs["fallback_charsets"], "");
    charset_array = strtok (charsets, "\\s,");
    foreach cs (charset_array)
    {
      if (NULL == cycle_iconv_array (cs))
        charsets = "";
    }
  }
  Prefs["fallback_charsets"] = strjoin (charset_array, ",");
  fallback_charset_array = @charset_array,
  last_exit = charset_array[-1];
}

% fix missing charset {{{1
static define iconv_fix (verbose)
{
  % passes articles w/o charset declaration and containing 8bit chars
  % through iconv from $fallback_charset to local charset
  variable xcharset = extract_displayed_article_header ("content-type");
  if (xcharset != "")
    return ();
  if (0 == string_match (raw_article_as_string (), "[\x80-\xFF]", 1))
    return ();
  variable
      artpath = path_to_article_as_file (), charset, probe;
  foreach charset (fallback_charset_array)
  {
    % do not run iconv with last exit charset
    if (charset == last_exit)
      break;
    probe = sprintf ("%s -s -f %s -t ucs-4 '%s' >/dev/null 2>&1; echo $?",
        iconv, charset, artpath);
    if ("0" == pipe_readline (probe, "failure at iconv probe"))
      break;
  }
  if (strlow (charset) != strlow (locale_charset))
  {
    replace_article (pipe_read (sprintf ("%s -c -f %s '%s'",
        iconv, charset, artpath)));
    if (verbose)
      message (sprintf ("%s: %s -c -f %s",
          _function_name (), iconv, charset));
  }
  if (Prefs["nntplocal"] == "")
    () = remove (artpath);
}

% provide hook {{{1
static define iconv_article_hook ()
{
  iconv_fix (1);
}

% examples {{{1
%if (1 != register_hook ("read_article_hook",
%    "GuessCharset->iconv_article_hook"))
%  throw RunTimeError, "iconv_article_hook not registered";

%definekey ("GuessCharset->query_fallback_charsets", "^A", "article");

% EOF vim:fdm=marker