Re: UTF-8 and ispell

--aM3YZ0Iwxop3KEKx
Content-Type: text/plain; charset=iso-8859-1
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable

[Cc: to the pkg-jed-devel and dict-common-dev mailing lists]

Hi Paul,

Thanks for the patch, it took faster than I thought to adapt
dictionaries-common for the new ispell.sl with improved aspell and UTF-8
support.  I am attaching below the patch that I in fact applied to jed-ex=
tra
2.4. Notice that I have made aspell the default program, instead of ispel=
l.

I have changed the dictionaries-common package to generate the necessary
calls to aspell_add_dictionary in jed-ispell-dicts.sl file.  I am attachi=
ng
below this file as it is generated in my system.  Note that the appropria=
te
strings are wrapped around "if (_slang_utf8_ok) {...} else {...}".  I did
not implemented the \x{...} encoding as you suggested.  This would
complicate DictonariesCommon without necessity.

The changed dictionaries-common package is available in my private
apt-getable repository [1], just in case someone wants to try it.

Now, we have to make a plan to release this to Debian.  The first step wo=
uld
be to integrate the new ispell.sl in jed-extra, either by making a new
upstream release of jedmodes or simply by integrating the patch. The seco=
nd
step will be to release the new dictionaries-common package [Agust=EDn: p=
lease
review my patch to the dictionaries-common package attached below; it
applies against the CVS sources as two days ago].  It has to be done in t=
his
order, otherwise jed will fail when loading jed-ispell-dicts.sl.

--=20
Rafael

--aM3YZ0Iwxop3KEKx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="ispell-mode-aspell.patch"

--- jed-extra-2.4.orig/ispell/ispell_common.sl
+++ jed-extra-2.4/ispell/ispell_common.sl
@@ -36,27 +36,43 @@
 % and set the ispell_language settings
 private define make_ispell_command()
 {
-   variable options, ispell_options = "";
-   % this was added after a discussion on the JED mailing list about a
-   % security hole in modehook.sl
-   if (Ispell_Program_Name != "ispell" and Ispell_Program_Name != "aspell")
-     error ("spell program should be ispell or aspell");
-
-   ispell_letters = Ispell_Letters [ispell_current_dictionary];
-   ispell_otherchars =  Ispell_OtherChars [ispell_current_dictionary];
-   ispell_wordlist =  Ispell_Wordlist [ispell_current_dictionary];
-
-   if (ispell_current_dictionary != "default")
-     ispell_options += " -d " + Ispell_Hash_Name[ispell_current_dictionary];
-
-   if (Ispell_Extchar [ispell_current_dictionary] != "")
-     ispell_options += " -T " + Ispell_Extchar [ispell_current_dictionary];
-
-   % extra options come last
-   ispell_options += " " + Ispell_Options [ispell_current_dictionary];
-
-   ispell_wordchars = ispell_otherchars+ispell_letters;
-   ispell_non_letters = "^" + ispell_letters;
+   variable ispell_options = "";
+   if (Ispell_Program_Name == "ispell")
+     {
+	ispell_letters = Ispell_Letters [ispell_current_dictionary];
+	ispell_otherchars =  Ispell_OtherChars [ispell_current_dictionary];
+	ispell_wordlist =  Ispell_Wordlist [ispell_current_dictionary];
+	
+	if (ispell_current_dictionary != "default")
+	  ispell_options += " -d " + Ispell_Hash_Name[ispell_current_dictionary];
+	
+	if (Ispell_Extchar [ispell_current_dictionary] != "")
+	  ispell_options += " -T " + Ispell_Extchar [ispell_current_dictionary];
+	
+	% extra options come last
+	ispell_options += " " + Ispell_Options [ispell_current_dictionary];
+	
+	ispell_wordchars = ispell_otherchars+ispell_letters;
+	ispell_non_letters = "^" + ispell_letters;
+     }
+   else if (Ispell_Program_Name == "aspell")
+     {
+	ispell_letters = Aspell_Letters [ispell_current_dictionary];
+	ispell_otherchars =  Aspell_OtherChars [ispell_current_dictionary];
+	ispell_wordlist =  Aspell_Wordlist [ispell_current_dictionary];
+	
+	if (ispell_current_dictionary != "default")
+	  ispell_options += " -d " + Aspell_Hash_Name[ispell_current_dictionary];
+	
+	% extra options come last
+	ispell_options += " " + Aspell_Options [ispell_current_dictionary];
+	
+	ispell_wordchars = ispell_otherchars+ispell_letters;
+	ispell_non_letters = "^" + ispell_letters;
+	
+     }
+   else
+     throw RunTimeError, "spell program should be ispell or aspell";
    
    % we don't set a '-[thn]' option here because an ispell/flyspell
    % process may work on different buffers, and probably doesn't need it
@@ -85,6 +101,17 @@
      }
 }
 %}}}
+%{{{ get dictionaries
+private define dictionaries()
+{
+   if (Ispell_Program_Name == "aspell")
+     return assoc_get_keys(Aspell_Hash_Name);
+   else
+     return assoc_get_keys(Ispell_Hash_Name);
+}
+
+%}}}
+
 %{{{ change the global language
 
 %!%+
@@ -110,8 +137,8 @@
    else
      {
 	new_language = read_with_completion
-	  (strjoin(assoc_get_keys(Ispell_Hash_Name), ","),
-	   "new language", Ispell_Dictionary, "", 's');
+	  (strjoin(dictionaries(), ","),
+	      "new language", Ispell_Dictionary, "", 's');
      }
    Ispell_Dictionary = new_language;
    ispell_change_current_dictionary(new_language);
@@ -151,8 +178,8 @@
    else
      {
 	new_language = read_with_completion
-	  (strjoin(assoc_get_keys(Ispell_Hash_Name), ","),
-	   "new language", get_blocal_var("ispell_dictionary", Ispell_Dictionary), "", 's');
+	  (strjoin(dictionaries(), ","),
+	      "new language", get_blocal_var("ispell_dictionary", Ispell_Dictionary), "", 's');
      }
    define_blocal_var("ispell_dictionary", new_language);
    ispell_change_current_dictionary(new_language);
@@ -188,7 +215,7 @@
    menu_append_item (popup, "&Region", "ispell_region");
    menu_append_item (popup, "&Flyspell", "flyspell_mode");
    menu_append_separator(popup);
-   variable lang, languages = assoc_get_keys(Ispell_Hash_Name);
+   variable lang, languages = dictionaries();
    languages = languages[array_sort(languages)];
    menu_radio (popup, "dictionary", &Ispell_Dictionary, languages, ,
 	       &ispell_change_dictionary); 
--- jed-extra-2.4.orig/ispell/ispell_init.sl
+++ jed-extra-2.4/ispell/ispell_init.sl
@@ -35,7 +35,7 @@
 %{{{ custom variables
 
 % Your spell program.  This could be ispell or aspell.
-custom_variable("Ispell_Program_Name", "ispell");
+custom_variable("Ispell_Program_Name", "aspell");
 % your default dictionary. "default" means use system default
 custom_variable("Ispell_Dictionary", "default");
 
@@ -43,12 +43,12 @@
 %{{{ public variables
 
 
-public variable Ispell_Hash_Name = Assoc_Type [String_Type, "default"];
-public variable Ispell_Letters = Assoc_Type[String_Type, "A-Za-z"];
-public variable Ispell_OtherChars = Assoc_Type [String_Type, "'"];
-public variable Ispell_Extchar = Assoc_Type [String_Type, ""];
-public variable Ispell_Options = Assoc_Type [String_Type, ""];
-public variable Ispell_Wordlist = Assoc_Type [String_Type,"/usr/share/dict/words"];
+variable Ispell_Hash_Name = Assoc_Type [String_Type, "default"];
+variable Ispell_Letters = Assoc_Type[String_Type, "A-Za-z"];
+variable Ispell_OtherChars = Assoc_Type [String_Type, "'"];
+variable Ispell_Extchar = Assoc_Type [String_Type, ""];
+variable Ispell_Options = Assoc_Type [String_Type, ""];
+variable Ispell_Wordlist = Assoc_Type [String_Type,"/usr/share/dict/words"];
 
 public define ispell_add_dictionary() % (name, hash=name, letters = a-z etc.,
   % otherchars = "'", extchr = "", opts = "")
@@ -71,7 +71,32 @@
      Ispell_Options [name] = opts;
 }
 
-ispell_add_dictionary ("default");
+
+variable Aspell_Hash_Name = Assoc_Type [String_Type, "default"];
+variable Aspell_Letters = Assoc_Type[String_Type, "A-Za-z"];
+variable Aspell_OtherChars = Assoc_Type [String_Type, "'"];
+variable Aspell_Options = Assoc_Type [String_Type, ""];
+variable Aspell_Wordlist = Assoc_Type [String_Type,"/usr/share/dict/words"];
+
+public define aspell_add_dictionary() % (name, hash=name, letters = a-z etc.,
+  % otherchars = "'", opts = "")
+{
+   variable name, hash, letters, otherchars, opts;
+   (name, hash, letters, otherchars, opts)
+     = push_defaults ( , , , , , _NARGS);
+   if (hash != NULL)
+     Aspell_Hash_Name [name] = hash;
+   else
+     Aspell_Hash_Name [name] = name;
+   if (letters != NULL)
+     Aspell_Letters [name] = strcat ("a-zA-Z", letters);
+   if (otherchars != NULL)
+     Aspell_OtherChars [name] = otherchars;
+   if (opts != NULL)
+     Aspell_Options [name] = opts;
+}
+
+aspell_add_dictionary ("default");
 
 % This will set up the dictionaries on your system, if you are a Debian Unstable user.
 custom_variable("Ispell_Cache_File", "/var/cache/dictionaries-common/jed-ispell-dicts.sl");

--aM3YZ0Iwxop3KEKx
Content-Type: text/plain; charset=utf-8
Content-Disposition: attachment; filename="jed-ispell-dicts.sl"
Content-Transfer-Encoding: quoted-printable

%%% This file is part of the dictionaries-common package.
%%% It has been automatically generated.
%%% DO NOT EDIT!

ispell_add_dictionary (
  "brasileiro",
  "brasileiro",
  "??????????????????????????????????",
  "[---']",
  "",
  "");


ispell_add_dictionary (
  "francais",
  "francais",
  "??????????????????????????",
  "[-']",
  "~list",
  "");


ispell_add_dictionary (
  "american",
  "american",
  "",
  "[']",
  "",
  "-B -d american");


if (_slang_utf8_ok) {
aspell_add_dictionary (
  "portugues",
  "pt_PT",
  "=C3=A1=C3=A9=C3=AD=C3=B3=C3=BA=C3=A0=C3=A8=C3=AC=C3=B2=C3=B9=C3=A3=C3=B5=
=C3=A7=C3=A2=C3=AA=C3=B4=C3=81=C3=89=C3=8D=C3=93=C3=9A=C3=80=C3=88=C3=8C=C3=
=92=C3=99=C3=83=C3=95=C3=87=C3=82=C3=8A=C3=94",
  "'",
  "");
} else {
aspell_add_dictionary (
  "portugues",
  "pt_PT",
  "????????????????????????????????",
  "'",
  "");
}

if (_slang_utf8_ok) {
aspell_add_dictionary (
  "brasileiro",
  "pt_BR",
  "=C3=A1=C3=A9=C3=AD=C3=B3=C3=BA=C3=A0=C3=A8=C3=AC=C3=B2=C3=B9=C3=A3=C3=B5=
=C3=A7=C3=BC=C3=A2=C3=AA=C3=B4=C3=81=C3=89=C3=8D=C3=93=C3=9A=C3=80=C3=88=C3=
=8C=C3=92=C3=99=C3=83=C3=95=C3=87=C3=9C=C3=82=C3=8A=C3=94",
  "---'",
  "");
} else {
aspell_add_dictionary (
  "brasileiro",
  "pt_BR",
  "??????????????????????????????????",
  "---'",
  "");
}

if (_slang_utf8_ok) {
aspell_add_dictionary (
  "nederlands8",
  "dutch",
  "=C3=A0=C3=A2=C3=A4=C3=A7=C3=A8=C3=A9=C3=AA=C3=AB=C3=AE=C3=AF=C3=B1=C3=B6=
=C3=B3=C3=BB=C3=BC=C3=80=C3=82=C3=84=C3=87=C3=88=C3=89=C3=8A=C3=8B=C3=8E=C3=
=8F=C3=91=C3=96=C3=93=C3=9B=C3=9C",
  "'",
  "-C -d dutch");
} else {
aspell_add_dictionary (
  "nederlands8",
  "dutch",
  "??????????????????????????????",
  "'",
  "-C -d dutch");
}

--aM3YZ0Iwxop3KEKx
Content-Type: text/x-diff; charset=us-ascii
Content-Disposition: attachment; filename="dict-common-jed-aspell.patch"

Index: debian/changelog
===================================================================
RCS file: /cvsroot/dict-common/dictionaries-common/debian/changelog,v
retrieving revision 1.529
diff -u -u -w -r1.529 changelog
--- debian/changelog	12 Sep 2007 00:21:37 -0000	1.529
+++ debian/changelog	27 Sep 2007 08:23:13 -0000
@@ -1,3 +1,13 @@
+dictionaries-common (0.85.2.1) UNRELEASED; urgency=low
+
+  * scripts/Debian/DictionariesCommon.pm.in: Added UTF-8-aware support for
+    aspell in JED
+  * debian.control: Build-Depends on libtext-iconv-perl (needed by
+    build_jed_support in DictionariesCommon.pm)
+  * scripts/system/update-dictcommon.in: Call build_jed_support
+
+ -- Rafael Laboissiere <[email protected]>  Wed, 26 Sep 2007 12:21:23 +0200
+
 dictionaries-common (0.85.2) unstable; urgency=low
 
   * Improve independent handling of ispell and aspell
Index: debian/control
===================================================================
RCS file: /cvsroot/dict-common/dictionaries-common/debian/control,v
retrieving revision 1.86
diff -u -u -w -r1.86 control
--- debian/control	10 Sep 2007 16:51:45 -0000	1.86
+++ debian/control	27 Sep 2007 08:23:13 -0000
@@ -2,8 +2,10 @@
 Section: text
 Priority: standard
 Maintainer: Agustin Martin Domingo <[email protected]>
-Uploaders: Rafael Laboissiere <[email protected]>, Rene Engelhard <[email protected]>
-Build-Depends-Indep: docbook-xml, docbook-dsssl, jade, links, slice, autoconf
+Uploaders: Rafael Laboissiere <[email protected]>,
+ Rene Engelhard <[email protected]>
+Build-Depends-Indep: docbook-xml, docbook-dsssl, jade, links, slice,
+ autoconf, recode, libtext-iconv-perl
 Build-Depends: debhelper (>= 4.1.16), dpatch (>= 2.0.9)
 XS-Vcs-Browser: http://alioth.debian.org/plugins/scmcvs/cvsweb.php/?cvsroot=dict-common
 XS-Vcs-CVS: -d :pserver:anonymous-ncYOAQ89aKUz9KZJk/[email protected]:/cvsroot/dict-common
Index: scripts/Debian/DictionariesCommon.pm.in
===================================================================
RCS file: /cvsroot/dict-common/dictionaries-common/scripts/Debian/DictionariesCommon.pm.in,v
retrieving revision 1.36
diff -u -u -w -r1.36 DictionariesCommon.pm.in
--- scripts/Debian/DictionariesCommon.pm.in	12 Sep 2007 00:21:37 -0000	1.36
+++ scripts/Debian/DictionariesCommon.pm.in	27 Sep 2007 08:23:13 -0000
@@ -3,6 +3,7 @@
 package Debian::DictionariesCommon;
 
 use base qw(Exporter);
+use Text::Iconv;
 
 # List all exported symbols here.
 our @EXPORT_OK = qw(parseinfo updatedb loaddb
@@ -348,7 +349,7 @@
   my $dictionaries = loaddb ("ispell");
   my $slang = generate_comment ("%%% ");
 
-  foreach $k (keys %$dictionaries) {
+  foreach my $k (keys %$dictionaries) {
 
     my $lang = $dictionaries->{$k};
     next if (exists $lang->{'jed-display'} 
@@ -374,6 +375,53 @@
   "$otherchars",
   "$extendedcharmode",
   "$ispellargs");
+
+};
+
+  }
+
+  $dictionaries = loaddb ("aspell");
+
+  foreach my $k (keys %$dictionaries) {
+
+    my $lang = $dictionaries->{$k};
+    next if (exists $lang->{'jed-display'}
+	     && $lang->{'jed-display'} eq "no");
+
+    my $hashname = $lang->{"hash-name"};
+    my $additionalchars = exists $lang->{additionalchars} ?
+      $lang->{additionalchars} : "";
+    my $otherchars = exists $lang->{otherchars} ?
+      $lang->{otherchars} : "'";
+    $otherchars =~ s/^\[//;
+    $otherchars =~ s/\]$//;
+    my $emacsenname = exists $lang->{"emacsen-name"} ?
+      $lang->{"emacsen-name"} : $hashname;
+    my $ispellargs = exists $lang->{"ispell-args"} ?
+      $lang->{"ispell-args"} : "";
+    my $codingsystem = exists $lang->{"coding-system"} ?
+      $lang->{"coding-system"} : "l1";
+
+    my $converter = Text::Iconv->new ($codingsystem, "utf8");
+    my $additionalchars_utf = $converter->convert ($additionalchars);
+    my $otherchars_utf = $converter->convert ($otherchars);
+
+    $slang .= qq{
+if (_slang_utf8_ok) {
+aspell_add_dictionary (
+  "$emacsenname",
+  "$hashname",
+  "$additionalchars_utf",
+  "$otherchars_utf",
+  "$ispellargs");
+} else {
+aspell_add_dictionary (
+  "$emacsenname",
+  "$hashname",
+  "$additionalchars",
+  "$otherchars",
+  "$ispellargs");
+}
 };
   }
 
Index: scripts/system/update-dictcommon.in
===================================================================
RCS file: /cvsroot/dict-common/dictionaries-common/scripts/system/update-dictcommon.in,v
retrieving revision 1.6
diff -u -u -w -r1.6 update-dictcommon.in
--- scripts/system/update-dictcommon.in	7 Jul 2005 10:57:30 -0000	1.6
+++ scripts/system/update-dictcommon.in	27 Sep 2007 08:23:13 -0000
@@ -10,6 +10,7 @@
 
 updatedb ($class);
 build_emacsen_support ();
+build_jed_support ();
 
 system ("aspell-autobuildhash") == 0
     or die "Error running aspell-autobuildhash\n";

--aM3YZ0Iwxop3KEKx
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline

_______________________________________________
Dict-common-dev mailing list
Dict-common-dev-XbBxUvOt3X2LieD7tvxI8l/[email protected]
http://lists.alioth.debian.org/mailman/listinfo/dict-common-dev
--aM3YZ0Iwxop3KEKx--