RE: charset stuff
Pogrebnoy Alexander <[email protected]>
| Newsgroups | gmane.comp.mobile.kannel.devel |
|---|---|
| Message-ID | <D97C208245E2D411A9430002A5095488016FCA71@lk-server.office.gt.com.ua> |
> -----Original Message----- > From: Paul P Komkoff Jr [mailto:[email protected]] > Sent: Sunday, February 22, 2004 11:28 PM > To: [email protected] > Subject: charset stuff > > > I was jumping around like mad because some sites aren't working. > Some digging shows that it is because we got wrong charset body > without correct charset header. > More deeper digging involved Russian Apache default config (which says > that mobile User-Agents are braindead and all and any ;charset= header > portion should be killed regardless of result body encoding). I had the same troubles when try to update to newest version of kannel. Previously used - kannel-1.2.1 + SAR patched. In my case many content providers use any of UTF-8, ISO-8859-5, WINDOWS-1251, KOI8-r cyrillic charsets, and indicate it either in the Content-Type HTTP reply header or xml preamble. All of it works with ver-1.2.1, because 1.2.1 has not code blocks that adapt content body's charset to device in wap_appl.c and wml_compiler.c, and always return binary wmlc data in UTF-8. Considering adapt content feature in newest version, content with this charsets seems must to be work ok, but don't. Allow me to express one's thoughts about why it may happens... . first of all... device_headers == NULL for S_MethodInvoke_Ind in return_reply. for S_MethodInvoke_Ind device_headers must be = orig_event->u.S_MethodInvoke_Ind.session_headers, not .request_headers like in S_Unit_MethodInvoke_Ind. . error in comparison (octstr_case_compare(charset, octstr_imm("UTF-8")) < 0) in return_reply and wml_compile. It must be !=0, because for any of WINDOWS-125X octstr_case_compare return 1. . in return_reply content charset adoption must be based on both charset indication (http header and xml preamble), and in 2-nd case will delete it indication. I propose to examine following attached patch that imho corrects above. PS: Don't forget update libxml for security reason according to http://cve.mitre.org/cgi-bin/cvename.cgi?name=CAN-2004-0110
charset.patch
(application/octet-stream, 5.2 KB)
diff -Nru ../kannel-snapshot-cvs-20040226/gw/wap-appl.c gw/wap-appl.c
--- ../kannel-snapshot-cvs-20040226/gw/wap-appl.c 2004-01-22 16:08:24.000000000 +0200
+++ gw/wap-appl.c 2004-03-04 20:26:44.000000000 +0200
@@ -732,7 +732,7 @@
/* ensure we pass only the orginal headers to the convertion routine */
device_headers = (orig_event->type == S_MethodInvoke_Ind) ?
- orig_event->u.S_MethodInvoke_Ind.request_headers :
+ orig_event->u.S_MethodInvoke_Ind.session_headers :
orig_event->u.S_Unit_MethodInvoke_Ind.request_headers;
if (device_headers == NULL)
device_headers = list_create();
@@ -844,14 +844,18 @@
octstr_search(content.type, octstr_imm("application/vnd.wap.xhtml+xml"), 0) >= 0) {
Octstr *charset;
- /* get charset used in content body, default to utf-8 if not present */
+ /* get charset used in content body or http header, default to utf-8 if not present */
if ((charset = find_charset_encoding(content.body)) == NULL)
- charset = octstr_imm("UTF-8");
+ if (octstr_len(content.charset) > 0) {
+ charset = octstr_duplicate(content.charset);
+ } else {
+ charset = octstr_create("UTF-8");
+ }
/* convert to utf-8 if original charset is not utf-8
* and device supports it */
- if (octstr_case_compare(charset, octstr_imm("UTF-8")) < 0 &&
+ if (octstr_case_compare(charset, octstr_imm("UTF-8")) != 0 &&
!http_charset_accepted(device_headers, octstr_get_cstr(charset))) {
if (!http_charset_accepted(device_headers, "UTF-8")) {
warning(0, "WSP: Device doesn't support charset <%s> neither UTF-8",
@@ -865,13 +869,16 @@
octstr_destroy(content.charset);
content.charset = octstr_create("UTF-8");
/* XXX it might be good idea to change <?xml...encoding?> */
+ if (delete_charset_encoding(content.body))
+ debug("wsp",0,"Delete encoding=\"%s\" from <?xml ... ?> ",
+ octstr_get_cstr(charset));
}
}
}
/* convert to iso-8859-1 if original charset is not iso
* and device supports it */
- else if (octstr_case_compare(charset, octstr_imm("ISO-8859-1")) < 0 &&
+ else if (octstr_case_compare(charset, octstr_imm("ISO-8859-1")) != 0 &&
!http_charset_accepted(device_headers, octstr_get_cstr(charset))) {
if (!http_charset_accepted(device_headers, "ISO-8859-1")) {
warning(0, "WSP: Device doesn't support charset <%s> neither ISO-8859-1",
@@ -885,6 +892,9 @@
octstr_destroy(content.charset);
content.charset = octstr_create("ISO-8859-1");
/* XXX it might be good idea to change <?xml...encoding?> */
+ if (delete_charset_encoding(content.body))
+ debug("wsp",0,"Delete encoding=\"%s\" from <?xml ... ?> ",
+ octstr_get_cstr(charset));
}
}
}
diff -Nru ../kannel-snapshot-cvs-20040226/gw/wml_compiler.c gw/wml_compiler.c
--- ../kannel-snapshot-cvs-20040226/gw/wml_compiler.c 2004-02-16 20:55:22.000000000 +0200
+++ gw/wml_compiler.c 2004-03-04 19:11:48.000000000 +0200
@@ -348,7 +348,7 @@
/* transcode from charset to UTF-8 */
if (charset && octstr_len(charset) &&
- octstr_case_compare(charset, octstr_imm("UTF-8")) == -1) {
+ octstr_case_compare(charset, octstr_imm("UTF-8")) != 0) {
debug("wml_compile", 0, "WML compiler: Transcoding from <%s> to UTF-8",
octstr_get_cstr(charset));
set_charset(wml_text, charset);
diff -Nru ../kannel-snapshot-cvs-20040226/gw/xml_shared.c gw/xml_shared.c
--- ../kannel-snapshot-cvs-20040226/gw/xml_shared.c 2004-01-22 16:08:24.000000000 +0200
+++ gw/xml_shared.c 2004-03-04 20:44:11.000000000 +0200
@@ -153,6 +153,25 @@
return encoding;
}
+/*
+ * delete_charset_encoding -- parses for a encoding argument within
+ * the xml preabmle, ie. <?xml verion="xxx" encoding="ISO-8859-1"?>
+ * and delete it.
+ */
+int delete_charset_encoding(Octstr *document)
+{
+ long gt = 0, enc = 0;
+
+ enc = octstr_search(document, octstr_imm(" encoding="), 0);
+ gt = octstr_search(document, octstr_imm("?>"), 0);
+
+ if (enc < 0 || enc + 10 > gt)
+ return 0;
+
+ octstr_delete(document, enc, gt - enc );
+ return 1;
+}
+
/*
* only_blanks - checks if a text node contains only white space, when it can
diff -Nru ../kannel-snapshot-cvs-20040226/gw/xml_shared.h gw/xml_shared.h
--- ../kannel-snapshot-cvs-20040226/gw/xml_shared.h 2004-01-22 16:08:24.000000000 +0200
+++ gw/xml_shared.h 2004-03-04 20:45:03.000000000 +0200
@@ -104,6 +104,13 @@
Octstr *find_charset_encoding(Octstr *document);
/*
+ * delete_charset_encoding -- parses for a encoding argument within
+ * the xml preabmle, ie. <?xml verion="xxx" encoding="ISO-8859-1"?>
+ * and delete it.
+ */
+int delete_charset_encoding(Octstr *document);
+
+/*
* element_check_content - a helper function for checking if an element has
* content or attributes. Returns status bit for attributes (0x80) and another
* for content (0x40) added into one octet.