display undeclared iso-charset in UTF environment

Hi,

Here's a script I use to be able to read posts of certain clients
that don't declare Charset even though they use 8-bit characters.

Feel free to comment/improve.

Example Usage:

public define read_article_hook ()
{
  ArtFilter->iconv_fix (1)
}

#v+
implements ("ArtFilter");

%%% CAVEAT %%%
%% this file must be saved in iso-8859-15 encoding!
%% eg. in vim:
%% :w ++enc=iso-8859-15
%%%

if (_slang_version < 20000)
  error ("needs S-Lang version >= 2");

%%% user config %%%

%% fake vim modeline to signal encoding
% zim: set fenc=iso-8859-15 nomod: µµµµ€€€€

%% charset that will be fixed if not declared in UTF env
%% see also `hibits' below
private variable fix_charset = "ISO-8859-15";

%% if one of the following characters are found in an
%% article w/o Content-Type header in an UTF environment
%% iconv_fix() translates article from fix_charset to UTF
%% (instead of hardcoding this to latin9, one could
%% possibly read in several patterns from differently
%% encoded files that are set according
%% to current_newsgroup or LANG environment)
private variable hibits = "\\C[ßäöüáéíàèçôœëñåøµ©®¢£€]";

%% set this to location of your local news spool
%% these scripts are a bit faster and less noisy
%% if they can read from spool
%% also raw_article_as_string() is less reliable and
%% doesn't work if replace_article() has been called
%% before on same article
%% set to `""' if you read from remote server
%% if local spool is not readable by you (leafnode-2),
%% it will be reset to `""'
private variable nntplocal = "/var/spool/news";

%%% end user config %%%

% check whether we are in an UTF environment
private variable utf_env = getenv ("LANG");
if (NULL == utf_env)
  utf_env = getenv ("LC_CTYPE");
if (NULL != utf_env and 0 == string_match (utf_env, "\\Cutf-8", 1))
  utf_env = NULL;

% check whether spool is readable
private variable fp = fopen (strcat (nntplocal, "/active.read"), "r");
if (NULL == fp)
  nntplocal = "";

%%% end init %%%

static define get_article_filename ()
{
  % returns path to article in news spool
   variable xref = extract_displayed_article_header ("xref");
  if ("" == nntplocal or "" == xref)
    return NULL;

  variable i, path, st;
  xref = strchop (xref, ' ', 0);
  % -> ["my.host.name", "alt.lang.s-lang:6944", "alt.astrology:839"]
  for (i = 1; i < length (xref); i++)
  {
    % xref[i] -> "alt.lang.s-lang:6944"
    path = strcat (nntplocal, "/", strtrans (xref[i], ".:", "/"));
    % -> "/var/spool/news/alt/lang/s-lang/6944"
    st = stat_file (path);
    if (NULL != st and NULL != stat_is ("reg", st.st_mode))
      return path;
  }
  return NULL;
}

static define read_pipe (syscall)
{
  % returns stdout of system call `syscall'
  variable out = "";

  fp = popen (syscall, "r");
  if (NULL == fp)
    throw OpenError, strcat (_function_name (), ": error in pipeline");
  out = strjoin (fgetslines (fp), "");
  () = pclose (fp);
  return out;
}

static define tmpfile_write (s)
{
  % writes string `s' to mildly secure tempfile `tf'
  % and returns path `tf'
  variable tf = sprintf ("/tmp/slrn_tmp.%d", getpid ());
  fp = fopen (tf, "w");
  if (NULL == fp)
    throw OpenError, strcat (_function_name (), ": could not open tmpfile");
  () = fputs (s, fp);
  () = fclose (fp);
  return tf;
}

static define string_fifo (s, prog)
{
  % converts string s by writing it to temporary file
  % and reading output of prog operating on temp file
  variable tf = tmpfile_write (s);
  s = read_pipe (sprintf ("%s '%s'", prog, tf));
  () = system (sprintf ("rm -f '%s'", tf));
  return s;
}

%% caveat: filter must take filename as /last/ argument
%% display_filter ("sed -f myfilter.sed", 0);
%% or
%% display_filter ("procmail -pm my.procmailrc <", 1);
private define display_filter (filter, artstr, verbose)
{
  variable artpath = get_article_filename ();

  if (NULL != artpath) % operate on article in spool
    artstr = read_pipe (sprintf ("%s '%s'", filter, artpath));
  else
  {
    if ("" == artstr)
      artstr = raw_article_as_string ();
    artstr = string_fifo (artstr, filter);
  }

  replace_article (artstr);
  if (verbose == 1)
    message (sprintf ("%s: %s", _function_name (), filter));
}


static define iconv_fix (verbose)
{
  % in UTF environment passes articles w/o charset declaration
  % and containing 8bit chars through iconv from `fix_charset'
  % to UTF-8
  if (NULL == utf_env)
    return;

  variable xcharset = extract_displayed_article_header ("Content-Type"),
    rawart = raw_article_as_string,
    iconv = sprintf ("/sw/bin/iconv -c -f %s -t UTF-8", fix_charset);

  if ("" == xcharset and 0 != string_match (rawart, hibits, 1))
    display_filter (iconv, rawart, verbose);
}
#v-

BTW, anything new on my issue with slrn and leafnode-2? -- well,
the above script works better with leafnode-1 anyway.

Is there a way to force raw_article_as_string() to download the
article again?

c
-- 
_B A U S T E L L E N_ lesen! --->> <http://www.blacktrash.org/baustellen.html>

-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642