Re: display undeclared iso-charset in UTF environment

* Alain Bench on Friday, September 08, 2006 at 02:55:28 +0200:
> Should be configurable.

ok, last exit before the generic solution is in cvs.

#v+
% $Hg: display-filter.sl,v c4fc32aa7602 Sat Sep 09 12:03:32 2006 +0200 $

implements ("ArtFilter");

if (_slang_version < 20000)
  error ("S-Lang version >= 2 required");

%%% user config %%%

%% set this to location of your local news spool
%% these scripts are a bit faster and less noisy
%% if they can read from spool
%% also raw_article_as_string() is less reliable and
%% doesn't work if replace_article() has been called
%% before on same article
%% if local spool is not readable by you (leafnode-2),
%% it will be reset to `""'
private variable nntplocal = "/Volumes/spool/news";

%% fallback charset that will be fixed if not declared
%% can also be set interactively via
%% query_fallback_charset() or eg. in hooks via
%% set_fallback_charset() [see below]
private variable fallback_charset = "CP1252";

%% you may set this to the full path of your iconv
%% -- might make the script a tiny bit faster
private variable iconv = "iconv";

%%% end user config %%%

% check whether spool is readable
if (NULL == fopen (strcat (nntplocal, "/active.read"), "r"))
  nntplocal = "";

private variable hibits = sprintf ("[%s]", strjoin (["\d128", "\d129",
"\d130", "\d131", "\d132", "\d133", "\d134", "\d135", "\d136", "\d137", "\d138", "\d139",
"\d140", "\d141", "\d142", "\d143", "\d144", "\d145", "\d146", "\d147", "\d148", "\d149",
"\d150", "\d151", "\d152", "\d153", "\d154", "\d155", "\d156", "\d157", "\d158", "\d159",
"\d160", "\d161", "\d162", "\d163", "\d164", "\d165", "\d166", "\d167", "\d168", "\d169",
"\d170", "\d171", "\d172", "\d173", "\d174", "\d175", "\d176", "\d177", "\d178", "\d179",
"\d180", "\d181", "\d182", "\d183", "\d184", "\d185", "\d186", "\d187", "\d188", "\d189",
"\d190", "\d191", "\d192", "\d193", "\d194", "\d195", "\d196", "\d197", "\d198", "\d199",
"\d200", "\d201", "\d202", "\d203", "\d204", "\d205", "\d206", "\d207", "\d208", "\d209",
"\d210", "\d211", "\d212", "\d213", "\d214", "\d215", "\d216", "\d217", "\d218", "\d219",
"\d220", "\d221", "\d222", "\d223", "\d224", "\d225", "\d226", "\d227", "\d228", "\d229",
"\d230", "\d231", "\d232", "\d233", "\d234", "\d235", "\d236", "\d237", "\d238", "\d239",
"\d240", "\d241", "\d242", "\d243", "\d244", "\d245", "\d246", "\d247", "\d248", "\d249",
"\d250", "\d251", "\d252", "\d253", "\d254", "\d255"], ""));

%%% end init %%%

%%% helpers %%%

static define get_article_filename ()
{
  % returns path to article in news spool
  % or NULL if article not found/news spool not readable
  variable xref = extract_displayed_article_header ("xref");
  if (nntplocal != "" and xref != "")
  {
    xref = strchop (xref, ' ', 0);
    % -> ["host.name", "alt.lang.s-lang:42", "alt.astrology:839"]
    variable xref_len = length (xref), i, path, st;
    for (i = 1; i < xref_len; i++)
    {
      path = strcat (nntplocal, "/", strtrans (xref[i], ".:", "/"));
      % -> "/var/spool/news/alt/lang/s-lang/42"
      st = stat_file (path);
      if (st != NULL and NULL != stat_is ("reg", st.st_mode))
        return path;
    }
  }
  return NULL;
}

static define read_pipe (syscall)
{
  % returns stdout of system call `syscall'
  variable out, fp;
  fp = popen (syscall, "r");
  if (fp == NULL)
    throw OpenError, strcat (_function_name (), ": error in pipeline");
  out = strjoin (fgetslines (fp), "");
  () = pclose (fp);
  return out;
}

static define tmpfile_write (s)
{
  % writes string `s' to mildly secure tempfile `tf'
  % and returns path `tf'
  variable tf = sprintf ("/tmp/slrn_tmp.%d", getpid ()),
      fp = fopen (tf, "w");
  if (fp == NULL)
    throw OpenError, strcat (_function_name (), ": failed to open tmpfile");
  () = fputs (s, fp);
  () = fclose (fp);
  return tf;
}

static define string_fifo (s, prog)
{
  % converts string s by writing it to temporary file
  % and reading output of prog operating on temp file
  variable tf = tmpfile_write (s);
  s = read_pipe ("$prog '$tf'"$);
  () = system ("rm -f '$tf'"$);
  return s;
}

%% caveat: filter must take filename as /last/ argument
%% display_filter ("sed -f myfilter.sed", 0);
%% or
%% display_filter ("procmail -pm my.procmailrc <", 1);
private define display_filter (filter, artstr, verbose)
{
  variable artpath = get_article_filename ();
  if (artpath != NULL) % operate on article in spool
    artstr = read_pipe ("$filter '$artpath'"$);
  else
  {
    if (artstr == "")
      artstr = raw_article_as_string ();
    artstr = string_fifo (artstr, filter);
  }

  replace_article (artstr);
  if (verbose == 1)
    message (sprintf ("%s: %s", _function_name (), filter));
}

%%% configure fallback charset %%%

%% get all charsets supported by iconv
private variable iconv_list = strtok (read_pipe ("$iconv -l"$));

private define cycle_iconv_list (ch)
{
  % cycles thru list of supported charsets
  % looking for `ch'
  % returns `charset' on success, NULL otherwise
  variable charset;
  foreach charset (iconv_list)
  {
    if (charset == ch)
      return charset;
  }
  return NULL;
}

%%% public functions %%%

static define set_fallback_charset (ch)
{
  % can be used to set $fallback_charset eg. in hooks
  variable charset = cycle_iconv_list (ch);
  if (charset != NULL)
    fallback_charset = charset;
  else
    throw RunTimeError, "`$ch': not supported by $iconv"$;
}

static define query_fallback_charset ()
{
  % sets $fallback_charset interactively
  variable charset = NULL;
  while (charset == NULL)
  {
    charset = read_mini ("Enter value for fallback_charset: ",
        fallback_charset, "");
    charset = cycle_iconv_list (charset);
  }
  fallback_charset = charset;
}

static define iconv_fix (verbose)
{
  % passes articles w/o charset declaration and containing 8bit chars
  % through iconv from $fallback_charset to local charset
  variable xcharset = extract_displayed_article_header ("content-type"),
      rawart = raw_article_as_string ();
  if (xcharset == "" and 0 != string_match (rawart, hibits, 1))
    display_filter ("$iconv -c -f $fallback_charset"$, rawart, verbose);
}

%%% register hooks %%%

static define iconv_article_hook ()
{
  iconv_fix (1);
}

static define fallback_charset_hook ()
{
  % simple example that can be called from article_mode_hook
  if (0 == strncmp (current_newsgroup (), "cz.", 3))
    set_fallback_charset ("CP1250");
  else
    set_fallback_charset ("CP1252");
}

if (1 != register_hook ("read_article_hook", "ArtFilter->iconv_article_hook"))
  throw RunTimeError, "iconv_article_hook not registered";

if (1 != register_hook ("article_mode_hook",
    "ArtFilter->fallback_charset_hook"))
  throw RunTimeError, "fallback_charset_hook not registered";

definekey ("ArtFilter->query_fallback_charset", "^A", "article");
#v-

helpers may all be "private", I just hook into them from other
scripts. And using pcre plugin might be faster.

c
-- 
_B A U S T E L L E N_ lesen! --->> <http://www.blacktrash.org/baustellen.html>

-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642