Re: [RFC] wml->wmlc default character mangling
Paul P Komkoff Jr <[email protected]>
| Newsgroups | gmane.comp.mobile.kannel.devel |
|---|---|
| Organization | Department of Fish & Wildlife |
| Message-ID | <[email protected]> |
Replying to Stipe Tolj: > Hi list, > > ok, following the thread frrom Rune Saetre [Msg-Id: > <[email protected]>] I come up with > following status-quo conclusion while reviewing the code: (HEAD in CVS): I was staring at the code for hour or two. This all needs heavy cleanup, concentrating all conversion stuff in wap-appl and removing it from wml-compile entirely. Including stripping of preamble in case of successful recoding. But I wrote on it here at least 2 times already. See attachemnt for how I am doing it now (skip gwlib/ part if you don't like it). > * entering gw/wap-appl.c:return_reply() we receive the HTTP response > (headers and body) > * if status was OK (HTTP 200) we perform some character convert to utf-8 > for various content-types, see line 842. > * in fact we check via find_charset_encoding() if there has been an > encoding in the XML preamble. Hence in the body of the response. If there > is not, we assume utf-8 as character encoding. (BTW, no implication from > the content-type HTTP header yet). > * if XML preamble gives a non-utf-8 encoding, but the device indicates > that it supports utf-8, we do convert, line 866. > * we do the same as a "backup" for iso-8859-1 in case utf-8 is not > supported by the device. > * then we run into the conver_content() routine that does the lookup in > the converter array, bringing us up to convert_wml_to_wmlc() and hence to > wml_compile(). > * so far so good > * wml_compile() seems to do the mangling "once again" ?! ^^^^^ double-encoding bug is triggered by passing already converted xml with preamble left intact. > * gw/wml_compiler.c:351: if passed charset var is not utf-8 then call > gw/xml_shared.c:set_charset(), which converts to utf-8. > * now we obey the rules: first: pick XML preamble encoding value, second: > pick given HTTP header value, third: fallback to utf-8. > * and we come up to the libxml2 parsing calls. > > ok, so where are the things in here to be changed (from policy site of > view) if any? -- Paul P 'Stingray' Komkoff Jr // http://stingr.net/key <- my pgp key This message represents the official view of the voices in my head
charset
(text/plain, 5.5 KB)
Index: gateway.C7/configure.in
===================================================================
--- gateway.C7.orig/configure.in 2005-03-17 17:43:20.155547649 +0300
+++ gateway.C7/configure.in 2005-03-17 17:43:22.168247831 +0300
@@ -213,6 +213,18 @@
]
)
+AC_MSG_CHECKING([whether to do all wapbox xml processing in utf-8])
+AC_ARG_ENABLE(scharset,
+[ --enable-scharset do all wapbox xml processing in utf-8],
+[
+ if test "$enableval" != yes; then
+ AC_MSG_RESULT(no)
+ else
+ AC_MSG_RESULT(yes)
+ AC_DEFINE(NEW_CHARSETS, 1, [Simplify wapbox charset processing])
+ fi
+])
+
dnl Extra feature checks
dnl GW_HAVE_TYPE_FROM(HDRNAME, TYPE, HAVENAME, DESCRIPTION)
Index: gateway.C7/gw/wap-appl.c
===================================================================
--- gateway.C7.orig/gw/wap-appl.c 2005-03-17 17:43:20.158547203 +0300
+++ gateway.C7/gw/wap-appl.c 2005-03-17 17:43:22.170247533 +0300
@@ -523,6 +523,10 @@
* to handle those charsets for all content types, just WML/XHTML. */
static void add_charset_headers(List *headers)
{
+#ifdef NEW_CHARSETS
+ if (!http_charset_accepted(headers, "utf-8"))
+ http_header_add(headers, "Accept-Charset", "utf-8");
+#else
long i, len;
gw_assert(charsets != NULL);
@@ -532,6 +536,7 @@
if (!http_charset_accepted(headers, charset))
http_header_add(headers, "Accept-Charset", charset);
}
+#endif
}
@@ -720,6 +725,23 @@
}
+static void strip_preamble(Octstr *document) {
+ long gt = 0, enc = 0;
+ Octstr *text = NULL, *encoding = NULL;
+
+ encoding = octstr_imm(" encoding");
+ enc = octstr_search(document, encoding, 0);
+ gt = octstr_search_char(document, '>', 0);
+
+ if (enc > 0 && gt > enc) {
+ gt++;
+ text = octstr_copy(document, gt, octstr_len(document) - gt);
+ octstr_truncate(document, 0);
+ octstr_append_data(document, octstr_get_cstr(text), octstr_len(text));
+ octstr_destroy(text);
+ }
+}
+
/*
* Return an HTTP reply back to the phone.
*/
@@ -865,12 +887,30 @@
/* get charset used in content body, default to utf-8 if not present */
if ((charset = find_charset_encoding(content.body)) == NULL)
+#ifdef NEW_CHARSETS
+ if (octstr_len(content.charset) > 0) {
+ charset = octstr_duplicate(content.charset);
+ } else {
+ charset = octstr_imm("UTF-8");
+ }
+#else
charset = octstr_imm("UTF-8");
+#endif
/* convert to utf-8 if original charset is not utf-8
* and device supports it */
- if (octstr_case_compare(charset, octstr_imm("UTF-8")) < 0 &&
+#if 0
+ if (octstr_case_compare(charset, octstr_imm("UTF-8")) != 0) {
+ debug("wsp",0,"Converting wml/xhtml from charset <%s> to UTF-8",
+ octstr_get_cstr(charset));
+ if (charset_convert(content.body, octstr_get_cstr(charset), "UTF-8") >= 0) {
+ octstr_destroy(content.charset);
+ content.charset = octstr_create("UTF-8");
+ }
+ }
+#else
+ if (octstr_case_compare(charset, octstr_imm("UTF-8")) != 0 &&
!http_charset_accepted(device_headers, octstr_get_cstr(charset))) {
if (!http_charset_accepted(device_headers, "UTF-8")) {
warning(0, "WSP: Device doesn't support charset <%s> neither UTF-8",
@@ -883,6 +923,7 @@
octstr_get_cstr(charset), "UTF-8") >= 0) {
octstr_destroy(content.charset);
content.charset = octstr_create("UTF-8");
+ strip_preamble(content.body);
/* XXX it might be good idea to change <?xml...encoding?> */
}
}
@@ -890,7 +931,7 @@
/* convert to iso-8859-1 if original charset is not iso
* and device supports it */
- else if (octstr_case_compare(charset, octstr_imm("ISO-8859-1")) < 0 &&
+ else if (octstr_case_compare(charset, octstr_imm("ISO-8859-1")) != 0 &&
!http_charset_accepted(device_headers, octstr_get_cstr(charset))) {
if (!http_charset_accepted(device_headers, "ISO-8859-1")) {
warning(0, "WSP: Device doesn't support charset <%s> neither ISO-8859-1",
@@ -903,10 +944,12 @@
octstr_get_cstr(charset), "ISO-8859-1") >= 0) {
octstr_destroy(content.charset);
content.charset = octstr_create("ISO-8859-1");
+ strip_preamble(content.body);
/* XXX it might be good idea to change <?xml...encoding?> */
}
}
}
+#endif
octstr_destroy(charset);
}
Index: gateway.C7/gwlib/http.c
===================================================================
--- gateway.C7.orig/gwlib/http.c 2005-03-17 17:42:58.958705197 +0300
+++ gateway.C7/gwlib/http.c 2005-03-17 17:44:19.827658706 +0300
@@ -3023,9 +3023,11 @@
* to return charset 'iso-8859-1' in case of no given encoding and
* content-type is a 'text' subtype.
*/
+ /* REMOVE THIS for now
if (octstr_len(*charset) == 0 &&
octstr_ncompare(*type, octstr_imm("text"), 4) == 0)
octstr_append_cstr(*charset, "ISO-8859-1");
+ */
}
}