Simple UTF-8 support patch
Andreas Jobs <[email protected]> Wed, 27 Apr 2005 23:50:47 +0200
| Newsgroups | gmane.comp.web.htdig.devel |
|---|---|
| Message-ID | <[email protected]> |
Hi, I recently had the following problem: Due to the use of a CMS some of our pages are now UTF-8 encoded. Since we are a german university our pages may contain german umlauts ;-) I use ht://Dig to index all servers on the campus. The problem is/was, that we cannot find words with umlauts on those UTF-8 pages. First workaround: add accept-charset="ISO-8859-1" to the ht://Dig search form. Now we can find words with umlauts on old (non UTF-8)pages but not one the new (UTF-8) pages. Attached you'll find a patch, that does a simple UTF-8 to 8bit ASCII conversion. All non-convertable characters are are mapped to a questionmark(?). ReadBody may not be the best place to add this code (and it should be added to ReadChunkedBody as well), but it was the easiest way to achieve my goal. One may give me a hint for a better place :-) Comments welcome .... Andreas -- ! Andreas Jobs Network Operating Center ! ! Ruhr-Universitaet Bochum ! ! The only way to clean a compromised system is to flatten and rebuild. !
htdig-3.2.0b6-UTF8.patch
(text/plain, 2.3 KB)
diff -ur htdig-3.2.0b6.orig/htnet/HtHTTP.cc htdig-3.2.0b6/htnet/HtHTTP.cc
--- htdig-3.2.0b6.orig/htnet/HtHTTP.cc 2004-05-28 15:15:23.000000000 +0200
+++ htdig-3.2.0b6/htnet/HtHTTP.cc 2005-04-27 23:26:16.000000000 +0200
@@ -643,6 +643,8 @@
String line = 0;
int inHeader = 1;
+ _needUTF8Convert = 0;
+
if (_response._modification_time)
{
delete _response._modification_time;
@@ -731,7 +733,15 @@
token = strtok(token, "\n\t");
if (token && *token)
+ {
_response._content_type = token;
+ if ((_response._content_type.indexOf("text/html") != -1) && (_response._content_type.indexOf("UTF-8") != -1))
+ {
+ if ( debug > 4 )
+ cout << "needUTF8Convert flagged" << endl;
+ _needUTF8Convert = 1;
+ }
+ }
}
else if( ! mystrncasecmp((char*)line, "content-length:", 15))
@@ -970,6 +980,31 @@
}
+ if ( _needUTF8Convert )
+ {
+ if ( debug > 4 )
+ cout << "Converting UTF-8 characters" << endl;
+
+ char *srcPtr, *dstPtr;
+ srcPtr = dstPtr = _response._contents.get();
+ while ( *srcPtr )
+ {
+ if ( ( *srcPtr & 0x80 ) == 0 )
+ *dstPtr++ = *srcPtr++;
+ else if ( ( *srcPtr & 0xE0 ) == 0xC0 ) {
+ *dstPtr++ = (((*srcPtr & 0x03) << 6) | (*(srcPtr+1) & 0x3F) ) & 0xFF;
+ srcPtr += 2;
+ } else if ( ( *srcPtr & 0xF0 ) == 0xE0 ) {
+ *dstPtr++ = '?';
+ srcPtr += 3;
+ } else {
+ *dstPtr++ = '?';
+ srcPtr += 4;
+ }
+ }
+ *dstPtr = 0;
+ }
+
// Set document length
_response._document_length = _response._contents.length();
diff -ur htdig-3.2.0b6.orig/htnet/HtHTTP.h htdig-3.2.0b6/htnet/HtHTTP.h
--- htdig-3.2.0b6.orig/htnet/HtHTTP.h 2004-05-28 15:15:23.000000000 +0200
+++ htdig-3.2.0b6/htnet/HtHTTP.h 2005-04-27 23:25:43.000000000 +0200
@@ -316,6 +316,7 @@
int _bytes_read; // Bytes read
URL _url; // URL to retrieve
URL _referer; // Referring URL
+ int _needUTF8Convert; // Flag for simple UTF-8 convert
String _accept_language; // accept-language directive
signature.asc
(application/pgp-signature, 185 B)
-----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.0 (SunOS) iD8DBQFCcAk3mYk7fpbvie4RAn8RAJ4rZoZEysTFtwhE7GUjlpyR8s6YHQCfei1d krvo9tFAlyfUQpw7v23pZsQ= =Z3N2 -----END PGP SIGNATURE-----