Re: Tokenization of words containing hyphens

This is a multi-part message in MIME format.

------=_NextPart_000_000A_01CE7060.1C3E2850
Content-Type: text/plain;
	format=flowed;
	charset="iso-8859-1";
	reply-type=response
Content-Transfer-Encoding: 8bit

Thank you, Kevin.

I attach a file of differences from 0.60.5, covering the three procedures to 
which I made changes.

Regards,
Ciarán.

----- Original Message ----- 
From: "Kevin Atkinson" <[email protected]>
To: "Ciarán Ó Duibhín" <[email protected]>
Cc: <[email protected]>
Sent: Sunday, June 23, 2013 5:04 PM
Subject: Re: [aspell-devel] Tokenization of words containing hyphens


> Hi Ciarán,
>
> Thanks for looking into this.  I am very busy with other things, but will 
> try to more carefully read your post sometime in the next couple of weeks 
> and offer feedback.
>
> In order to do a full code review I need a more complete picture of the 
> changes you made.  The best way to do this is to post a patch (preferably 
> in unified diff format) of your changes to the list.
>
> Thanks Again,
> (Aspell Author) Kevin
> 

------=_NextPart_000_000A_01CE7060.1C3E2850
Content-Type: text/plain;
	format=flowed;
	name="changes.txt";
	reply-type=response
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="changes.txt"

--- C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5\modules\tokenizer\basic.cpp	=
2004-04-13 08:13:40 +0100
+++ =
C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5-msvc\modules\tokenizer\basic.cpp	=
2013-06-23 21:58:57 +0100
@@ -41,6 +41,7 @@
=20
     if (is_begin(*cur) && is_word(cur[1]))
     {
+      word.append(*cur);
       cur_pos +=3D cur->width;
       ++cur;
     }


--- =
C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5\modules\speller\default\language.cp=
p	2005-02-20 21:47:08 +0000
+++ =
C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5-msvc\modules\speller\default\langua=
ge.cpp	2013-06-23 22:01:18 +0100
@@ -441,7 +441,6 @@
       }
       while (*word && *inlist && *word =3D=3D *inlist) ++word, =
++inlist;
       if (*inlist) goto try_upper;
-      if (end && lang->special(*word).end) ++word;
       if (*word) goto try_upper;
       return true;
     try_upper:
@@ -449,7 +448,6 @@
       inlist =3D inlist0;
       while (*word && *inlist && *word =3D=3D lang->to_upper(*inlist)) =
++word, ++inlist;
       if (*inlist) goto fail;
-      if (end && lang->special(*word).end) ++word;
       if (*word) goto fail;
      =20
     } else { // case_insensitive
@@ -457,14 +455,12 @@
       while (*word && *inlist &&=20
              lang->to_upper(*word) =3D=3D lang->to_upper(*inlist)) =
++word, ++inlist;
       if (*inlist) goto fail;
-      if (end && lang->special(*word).end) ++word;
       if (*word) goto fail;
      =20
     }
     return true;
=20
   fail:
-    if (begin && lang->special(*word0).begin) {++word0; goto =
try_again;}
     return false;
   }
=20


--- =
C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5\modules\speller\default\speller_imp=
l.cpp	2005-01-17 10:19:16 +0000
+++ =
C:\de=E1nt=FAs\c=F3rais\aspell-0.60.5-msvc\modules\speller\default\spelle=
r_impl.cpp	2013-06-23 21:54:26 +0100
@@ -181,6 +181,39 @@
     clear_check_info(*ci);
     bool res =3D check2(word, try_uppercase, *ci, gi);
     if (res) return true;
+    unsigned i=3D0;
+    while (*(word+i)!=3D 0) {
+      if ((i > 0) && (i < word_end-word-1) && (*(word+i)=3D=3D'-')) {
+      	if (lang_->special('-').end) {  /* test up to hyphen as prefix, =
test remainder recursively as word */
+          char t =3D *(word+i+1);
+          *(word+i+1) =3D (char) 0;
+      		if (check2(word, try_uppercase, *ci, gi)) {
+      			*(word+i+1) =3D t;
+      			if (check(word+i+1, word_end, try_uppercase, =
run_together_limit, ci, gi))
+    	  			return true;
+    	  	}
+  		  	else
+  		      *(word+i+1) =3D t;
+        }
+        if (lang_->special('-').middle) {  /* test up to hyphen as =
word, test remainder recursively as word, then as suffix */
+        	*(word+i) =3D (char) 0;
+        	if (check2(word, try_uppercase, *ci, gi)) {
+        		*(word+i) =3D '-';
+        		if (check(word+i+1, word_end, try_uppercase, =
run_together_limit, ci, gi))
+        			return true;
+        		else {
+        			if (lang_->special('-').begin) {
+        				if (check(word+i, word_end, try_uppercase, =
run_together_limit, ci, gi))
+        				  return true;
+        			}
+      	  	}
+        	}
+        	else
+        		*(word+i) =3D '-';
+        }
+      }
+      ++i;
+    }
     if (run_together_limit <=3D 1) return false;
     enum {Yes, No, Unknown} is_title =3D try_uppercase ? Yes : Unknown;
     for (char * i =3D word + run_together_min_;=20

------=_NextPart_000_000A_01CE7060.1C3E2850
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Content-Disposition: inline

_______________________________________________
Aspell-devel mailing list
[email protected]
https://lists.gnu.org/mailman/listinfo/aspell-devel

------=_NextPart_000_000A_01CE7060.1C3E2850--