More patches for the C implementation of the English stemmer
Olly Betts <[email protected]>
| Newsgroups | gmane.comp.search.snowball |
|---|---|
| Message-ID | <20100728161743.GA19120@eisluft> |
Here are five more patches:
0001-Fix-bug-with-y-to-Y-marking.patch:
Fixes a bug - the algorithm description says that an initial 'y' should be
converted to 'Y', but the C implementation only does this if it is followed
by a vowel. For examples, this means that the C implementation stems "ygoe"
(an obsolete spelling of "ago") to "ygo", while the Snowball implementation
stems it to "ygoe".
0002-Use-const-qualifier-on-char-pointers-we-don-t-modify.patch:
Adding const here allows this to be compiled as C++, and fixes warnings with
some C compilers.
0003-Use-memcpy-in-setto-rather-than-memmove-since-we-kno.patch:
memcpy() is a little faster than memmove(), and safe here since the second
argument is always a string constant, so can't overlap with the first.
0004-Fix-indenting-and-add-missing-newline.patch:
Fixes a few code formatting inconsistencies.
0005-Specialise-setto-z-00-as-setto0-z.patch:
setto0(z, "\00") is used several times and the special case code is just
an assignment. This patch results in smaller compiled code which runs
faster.
Cheers,
Olly
_______________________________________________
Snowball-discuss mailing list
[email protected]
http://lists.tartarus.org/mailman/listinfo/snowball-discuss
0001-Fix-bug-with-y-to-Y-marking.patch
(text/x-diff, 823 B)
diff --git a/output.txt b/output.txt
index b6e4829..6a15930 100644
--- a/output.txt
+++ b/output.txt
@@ -29333,6 +29333,7 @@ yeso
yesterday
yet
yew
+ygoe
yield
yield
yield
diff --git a/stem_english.c b/stem_english.c
index ed1dd50..5db9998 100644
--- a/stem_english.c
+++ b/stem_english.c
@@ -459,7 +459,7 @@ extern int stem(struct stemmer * z, char * b, int k) {
for (i = 0; i < j; i++) {
int ch = b[i];
if (ch == 'y') {
- if (i == 0 && j > 1 && vowel(b[1]) || i > 0 && vowel(b[i - 1])) {
+ if (i == 0 || vowel(b[i - 1])) {
Y_found = TRUE; b[i] = ch = 'Y';
}
}
diff --git a/voc.txt b/voc.txt
index a5c0a36..0e8df01 100644
--- a/voc.txt
+++ b/voc.txt
@@ -29333,6 +29333,7 @@ yeso
yesterday
yet
yew
+ygoe
yield
yielded
yielding
0002-Use-const-qualifier-on-char-pointers-we-don-t-modify.patch
(text/x-diff, 1.9 KB)
diff --git a/stem_english.c b/stem_english.c
index 5db9998..2e3a9f7 100644
--- a/stem_english.c
+++ b/stem_english.c
@@ -103,7 +103,7 @@ static int valid_li_ending(struct stemmer * z) {
}
static int shortv(struct stemmer * z) {
- char * b = z->b;
+ const char * b = z->b;
int i = z->j;
int ch = b[i];
if vowel(ch) return FALSE;
@@ -113,9 +113,9 @@ static int shortv(struct stemmer * z) {
return FALSE;
}
-static int ends(struct stemmer * z, char * s) {
+static int ends(struct stemmer * z, const char * s) {
int length = s[0];
- char * b = z->b;
+ const char * b = z->b;
int k = z->k;
if (s[length] != b[k]) return FALSE; /* tiny speed-up */
if (length > k + 1) return FALSE;
@@ -124,14 +124,14 @@ static int ends(struct stemmer * z, char * s) {
return TRUE;
}
-static int equals(struct stemmer * z, char * s) {
+static int equals(struct stemmer * z, const char * s) {
if (s[0] != z->k + 1) return FALSE;
return ends(z, s);
}
-static int starts(struct stemmer * z, char * s) {
+static int starts(struct stemmer * z, const char * s) {
int length = s[0];
- char * b = z->b;
+ const char * b = z->b;
int k = z->k;
if (length > k + 1) return FALSE;
if (memcmp(b, s + 1, length) != 0) return FALSE;
@@ -139,7 +139,7 @@ static int starts(struct stemmer * z, char * s) {
return TRUE;
}
-static void setto(struct stemmer * z, char * s) {
+static void setto(struct stemmer * z, const char * s) {
int length = s[0];
int j = z->j;
memmove(z->b + j + 1, s + 1, length);
@@ -149,7 +149,7 @@ static void setto(struct stemmer * z, char * s) {
#define R1(z) (z->j + 1 >= z->p1)
#define R2(z) (z->j + 1 >= z->p2)
-static void r(struct stemmer * z, char * s) { if (R1(z)) setto(z, s); }
+static void r(struct stemmer * z, const char * s) { if (R1(z)) setto(z, s); }
static void step0(struct stemmer * z) {
char * b = z->b;
0003-Use-memcpy-in-setto-rather-than-memmove-since-we-kno.patch
(text/x-diff, 405 B)
diff --git a/stem_english.c b/stem_english.c
index 2e3a9f7..e41fb95 100644
--- a/stem_english.c
+++ b/stem_english.c
@@ -142,7 +142,7 @@ static int starts(struct stemmer * z, const char * s) {
static void setto(struct stemmer * z, const char * s) {
int length = s[0];
int j = z->j;
- memmove(z->b + j + 1, s + 1, length);
+ memcpy(z->b + j + 1, s + 1, length);
z->k = j+length;
}
0004-Fix-indenting-and-add-missing-newline.patch
(text/x-diff, 1.3 KB)
diff --git a/stem_english.c b/stem_english.c
index e41fb95..f01dc7e 100644
--- a/stem_english.c
+++ b/stem_english.c
@@ -68,8 +68,8 @@ static int cons(int ch) {
switch (ch) {
case 'a': case 'e': case 'i': case 'o': case 'u': case 'y':
return FALSE;
- }
- return TRUE;
+ }
+ return TRUE;
}
#define vowel(ch) (!cons(ch))
@@ -267,7 +267,7 @@ static void step2(struct stemmer * z) {
break;
case 'g':
if (ends(z, "\03" "ogi") && z->b[z->j] == 'l') { r(z, "\02" "og"); break; }
- }
+ }
}
static void step3(struct stemmer * z) {
@@ -316,7 +316,8 @@ static void step4(struct stemmer * z) {
return;
case 's':
if (ends(z, "\03" "ism")) break; return;
- case 't': if (ends(z, "\03" "ate")) break;
+ case 't':
+ if (ends(z, "\03" "ate")) break;
if (ends(z, "\03" "iti")) break; return;
case 'u':
if (ends(z, "\03" "ous")) break; return;
@@ -324,10 +325,10 @@ static void step4(struct stemmer * z) {
if (ends(z, "\03" "ive")) break; return;
case 'z':
if (ends(z, "\03" "ize")) break; return;
- default:
+ default:
return;
- }
- if (R2(z)) z->k = z->j;
+ }
+ if (R2(z)) z->k = z->j;
}
static void step5(struct stemmer * z) {
0005-Specialise-setto-z-00-as-setto0-z.patch
(text/x-diff, 1.5 KB)
diff --git a/stem_english.c b/stem_english.c
index f01dc7e..022dd93 100644
--- a/stem_english.c
+++ b/stem_english.c
@@ -146,6 +146,10 @@ static void setto(struct stemmer * z, const char * s) {
z->k = j+length;
}
+static void setto0(struct stemmer * z) {
+ z->k = z->j;
+}
+
#define R1(z) (z->j + 1 >= z->p1)
#define R2(z) (z->j + 1 >= z->p2)
@@ -209,7 +213,7 @@ static void step1b(struct stemmer * z) {
return;
ed_test:
if (vowelinstem(z)) {
- setto(z, "\00");
+ setto0(z);
if (ends(z, "\02" "at")) setto(z, "\03" "ate"); else
if (ends(z, "\02" "bl")) setto(z, "\03" "ble"); else
if (ends(z, "\02" "iz")) setto(z, "\03" "ize"); else
@@ -274,7 +278,7 @@ static void step3(struct stemmer * z) {
switch (z->b[z->k]) {
case 'e':
if (ends(z, "\05" "icate")) { r(z, "\02" "ic"); break; }
- if (ends(z, "\05" "ative")) { if (R2(z)) setto(z, "\00"); break; }
+ if (ends(z, "\05" "ative")) { if (R2(z)) setto0(z); break; }
if (ends(z, "\05" "alize")) { r(z, "\02" "al"); break; }
break;
case 'i':
@@ -333,11 +337,11 @@ static void step4(struct stemmer * z) {
static void step5(struct stemmer * z) {
if (ends(z, "\01" "e")) {
- if (R2(z) || R1(z) && !shortv(z)) setto(z, "\00");
+ if (R2(z) || R1(z) && !shortv(z)) setto0(z);
return;
}
if (ends(z, "\01" "l")) {
- if (R2(z) && z->b[z->j] == 'l') setto(z, "\00");
+ if (R2(z) && z->b[z->j] == 'l') setto0(z);
}
}
signature.asc
(application/pgp-signature, 835 B)
-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.9 (GNU/Linux) iQIcBAEBCAAGBQJMUFgnAAoJEBgUewc7rSsH4pYQAL1FXGzfpZtXoWsqJ8nL41mI QhscdgqOW+hHytSMZv5w75fW6i+XZ62eNarmntKQTOAv7Gd/qvScGNaQeCBLFiLs InfAX2DoXg0JuapVEtZg3mExckmjNl3u0ZzSfkMuDapeV7QR1PJE6Kg1loBuFLgH CLqQSfnm39pbAlOmlG57j9S74h5Rxq2HiezyKy9s2t4RvxheTI2nhunBb0K0pj3m 1cjS4HOT9adoHYqkx6XiQHEA24S2p7kyKQs8bGn79aq7zjzBg80Kc4phr3NHGLPV su3L4FYxfWbF5VHlM7iBWDQ0XH18vFOCO6xHdH3LlFD4yB1MtPuKzZtTcBIKvR7V 0vr/Dd1t2rDHJiZfevuGA51Th8Wu2SmyV9y4dXCZpGGinO4E4hnpaZokyndrcJbv GmbLatSZhrJfYCJV7scp3U/Ikft23ZATqNl9svVH/L0/Z61hX3nabaCzyBKX/QhP 7OMH3/E7UItzcH2kd4jifjPpxSph2nQpAU2sshVjbcDAgAKj+l5j17OKzu6a98GY SKBXZ0XzmiWfoOCJGimihZmaiL4s5pK/vtjtflryThjLPcCyeWqXvjLmDkR9N68A gK8XJbMzQ/0P8x1J9VcGk6ZULi2hWTkrZjfROO+Qp7f97TnJ9hj18aA7KEbNzZgL lYIxLkg1MCoWxiq4ALfu =hlzK -----END PGP SIGNATURE-----