Re: display undeclared iso-charset in UTF environment

Hi Alain,

* Alain Bench on Friday, September 08, 2006 at 02:55:28 +0200:
>  On Thursday, September 7, 2006 at 16:59:43 +0200, Christian Ebert wrote:
>> private variable hibits = "\\C[ßäöüáéíàèçôœëñåøµ©®¢£€]";
> 
>     I don't understand the interest of grepping specifically those
> chars. Why not just convert all unlabelled articles? Or grep for any
> char above 0x80?

Ok, I voted for decimal, easier to understand for me, and I loop
instead of grep.

It should work now for other 8bit charsets as well, if you adapt
fix_charset to your needs.

display-filter.sl:

#v+
implements ("ArtFilter");

if (_slang_version < 20000)
  error ("S-Lang version >= 2 required");

%%% user config %%%

%% assumed charset that will be fixed if not declared in UTF env
%% non-westerners, please adapt
private variable fix_charset = "CP1252";

%% set this to location of your local news spool
%% these scripts are a bit faster and less noisy
%% if they can read from spool
%% also raw_article_as_string() is less reliable and
%% doesn't work if replace_article() has been called
%% before on same article
%% if local spool is not readable by you (leafnode-2),
%% it will be reset to `""'
private variable nntplocal = "/var/spool/news";

%%% end user config %%%

% check whether we are in an UTF environment
private variable lc_env;
foreach lc_env (["LC_ALL", "LC_CTYPE", "LANG"])
{
  lc_env = getenv (lc_env);
  if (lc_env != NULL)
  {
    if (0 == string_match (lc_env, "\\Cutf-8", 1))
      lc_env = NULL;
    break;
  }
}

% check whether spool is readable
private variable fp = fopen (strcat (nntplocal, "/active.read"), "r");
if (fp == NULL)
  nntplocal = "";

private variable hibits = ["\d128", "\d129",
"\d130", "\d131", "\d132", "\d133", "\d134", "\d135", "\d136", "\d137", "\d138", "\d139",
"\d140", "\d141", "\d142", "\d143", "\d144", "\d145", "\d146", "\d147", "\d148", "\d149",
"\d150", "\d151", "\d152", "\d153", "\d154", "\d155", "\d156", "\d157", "\d158", "\d159",
"\d160", "\d161", "\d162", "\d163", "\d164", "\d165", "\d166", "\d167", "\d168", "\d169",
"\d170", "\d171", "\d172", "\d173", "\d174", "\d175", "\d176", "\d177", "\d178", "\d179",
"\d180", "\d181", "\d182", "\d183", "\d184", "\d185", "\d186", "\d187", "\d188", "\d189",
"\d190", "\d191", "\d192", "\d193", "\d194", "\d195", "\d196", "\d197", "\d198", "\d199",
"\d200", "\d201", "\d202", "\d203", "\d204", "\d205", "\d206", "\d207", "\d208", "\d209",
"\d210", "\d211", "\d212", "\d213", "\d214", "\d215", "\d216", "\d217", "\d218", "\d219",
"\d220", "\d221", "\d222", "\d223", "\d224", "\d225", "\d226", "\d227", "\d228", "\d229",
"\d230", "\d231", "\d232", "\d233", "\d234", "\d235", "\d236", "\d237", "\d238", "\d239",
"\d240", "\d241", "\d242", "\d243", "\d244", "\d245", "\d246", "\d247", "\d248", "\d249",
"\d250", "\d251", "\d252", "\d253", "\d254", "\d255"],
    iconv = sprintf ("iconv -c -f %s", fix_charset);

%%% end init %%%

%%% helpers %%%

static define get_article_filename ()
{
  % returns path to article in news spool
   variable xref = extract_displayed_article_header ("xref");
  if (nntplocal == "" or xref == "")
    return NULL;

  variable i, path, st;
  xref = strchop (xref, ' ', 0);
  for (i = 1; i < length (xref); i++)
  {
    path = strcat (nntplocal, "/", strtrans (xref[i], ".:", "/"));
    st = stat_file (path);
    if (st != NULL and NULL != stat_is ("reg", st.st_mode))
      return path;
  }
  return NULL;
}

static define read_pipe (syscall)
{
  % returns stdout of system call `syscall'
  variable out = "";

  fp = popen (syscall, "r");
  if (fp == NULL)
    throw OpenError, strcat (_function_name (), ": error in pipeline");
  out = strjoin (fgetslines (fp), "");
  () = pclose (fp);
  return out;
}

static define tmpfile_write (s)
{
  % writes string `s' to mildly secure tempfile `tf'
  % and returns path `tf'
  variable tf = sprintf ("/tmp/slrn_tmp.%d", getpid ());
  fp = fopen (tf, "w");
  if (fp == NULL)
    throw OpenError, strcat (_function_name (), ": could not open tmpfile");
  () = fputs (s, fp);
  () = fclose (fp);
  return tf;
}

static define string_fifo (s, prog)
{
  % converts string s by writing it to temporary file
  % and reading output of prog operating on temp file
  variable tf = tmpfile_write (s);
  s = read_pipe (sprintf ("%s '%s'", prog, tf));
  () = system (sprintf ("rm -f '%s'", tf));
  return s;
}

%% caveat: filter must take filename as /last/ argument
%% display_filter ("sed -f myfilter.sed", 0);
%% or
%% display_filter ("procmail -pm my.procmailrc <", 1);
private define display_filter (filter, artstr, verbose)
{
  variable artpath = get_article_filename ();

  if (artpath != NULL) % operate on article in spool
    artstr = read_pipe (sprintf ("%s '%s'", filter, artpath));
  else
  {
    if (artstr == "")
      artstr = raw_article_as_string ();
    artstr = string_fifo (artstr, filter);
  }

  replace_article (artstr);
  if (verbose == 1)
    message (sprintf ("%s: %s", _function_name (), filter));
}

static define iconv_fix (verbose)
{
  % in UTF environment passes articles w/o charset declaration
  % and containing 8bit chars through iconv
  if (lc_env == NULL)
    return;

  variable xcharset = extract_displayed_article_header ("Content-Type"),
    rawart = raw_article_as_string (), hibit;

  if (xcharset == "")
  {
    foreach hibit (hibits)
    {
      if (0 != is_substr (rawart, hibit))
      {
        display_filter (iconv, rawart, verbose);
        break;
      }
    }
  }
}

%%% register hook %%%

static define iconv_article_hook ()
{
  iconv_fix (1);
}

if (1 != register_hook ("read_article_hook", "ArtFilter->iconv_article_hook"))
  throw RunTimeError, "iconv_article_hook not registered";
#v-

c
-- 
_B A U S T E L L E N_ lesen! --->> <http://www.blacktrash.org/baustellen.html>

-------------------------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642