charset changes in Publisher

--Apple-Mail-5--531733632
Content-Transfer-Encoding: 7bit
Content-Type: text/plain;
	charset=US-ASCII;
	format=flowed

Pay attention to charset parameters in requests.
Add support for changing the default charset.


--Apple-Mail-5--531733632
Content-Transfer-Encoding: 7bit
Content-Type: text/plain;
	x-unix-mode=0664;
	name="charset.diff.txt"
Content-Disposition: attachment;
	filename=charset.diff.txt

Index: publish.py
===================================================================
--- publish.py	(revision 26332)
+++ publish.py	(revision 26338)
@@ -4,7 +4,7 @@
 Logic for publishing modules and objects on the Web.
 """
 
-import sys, traceback, cStringIO
+import sys, traceback, StringIO
 import time
 import urlparse
 import cgitb
@@ -211,7 +211,7 @@
 
     def _generate_plaintext_error(self, request, original_response,
                                   exc_type, exc_value, tb):
-        error_file = cStringIO.StringIO()
+        error_file = StringIO.StringIO()
 
         # format the traceback
         traceback.print_exception(exc_type, exc_value, tb, file=error_file)
@@ -226,7 +226,7 @@
 
     def _generate_cgitb_error(self, request, original_response,
                               exc_type, exc_value, tb):
-        error_file = cStringIO.StringIO()
+        error_file = StringIO.StringIO()
         hook = cgitb.Hook(file=error_file)
         hook(exc_type, exc_value, tb)
         error_file.write('<h2>Original Request</h2>')
Index: http_request.py
===================================================================
--- http_request.py	(revision 26332)
+++ http_request.py	(revision 26338)
@@ -47,6 +47,16 @@
     else:
         return None
 
+def _decode_string(s, charset):
+    if charset == 'iso-8859-1':
+        return s
+    try:
+        return s.decode(charset)
+    except LookupError:
+        raise RequestError('unknown charset %r' % charset)
+    except UnicodeDecodeError:
+        raise RequestError('invalid %r encoded string' % charset)
+
 def parse_header(line):
     """Parse a Content-type like header.
 
@@ -74,7 +84,7 @@
                            'with a "name" parameter: got %r' % full_cdisp)
     return (name, cdisp_params.get('filename'))
 
-def parse_query(qs):
+def parse_query(qs, charset):
     """(qs: string) -> {key:string, string|[string]}
 
     Parse a query given as a string argument and return a dictionary.
@@ -88,6 +98,8 @@
             name, value = chunk.split('=', 1)
         name = urllib.unquote(name.replace('+', ' '))
         value = urllib.unquote(value.replace('+', ' '))
+        name = _decode_string(name, charset)
+        value = _decode_string(value, charset)
         _add_field_value(fields, name, value)
     return fields
 
@@ -127,6 +139,8 @@
     when handling an exception.
     """
 
+    DEFAULT_CHARSET = 'iso-8859-1'
+
     def __init__(self, stdin, environ):
         self.stdin = stdin
         self.environ = environ
@@ -165,43 +179,42 @@
     def process_inputs(self):
         query = self.get_query()
         if query:
-            self.form.update(parse_query(query))
+            self.form.update(parse_query(query, self.DEFAULT_CHARSET))
         length = self.environ.get('CONTENT_LENGTH') or "0"
         try:
             length = int(length)
         except ValueError:
             raise RequestError('invalid content-length header')
-        content_type = self.environ.get("CONTENT_TYPE")
-        if content_type:
-            content_type = content_type.split(';')[0]
-        if content_type == 'application/x-www-form-urlencoded':
-            self._process_urlencoded(length)
-        elif content_type == 'multipart/form-data':
-            self._process_multipart(length)
+        ctype = self.environ.get("CONTENT_TYPE")
+        if ctype:
+            ctype, ctype_params = parse_header(ctype)
+            if ctype == 'application/x-www-form-urlencoded':
+                self._process_urlencoded(length, ctype_params)
+            elif ctype == 'multipart/form-data':
+                self._process_multipart(length, ctype_params)
 
-    def _process_urlencoded(self, length):
+    def _process_urlencoded(self, length, params):
         query = self.stdin.read(length)
         if len(query) != length:
             raise RequestError('unexpected end of request body')
-        self.form.update(parse_query(query))
+        charset = params.get('charset', self.DEFAULT_CHARSET)
+        self.form.update(parse_query(query, charset))
 
-    def _process_multipart(self, length):
-        full_ctype = self.environ.get('CONTENT_TYPE')
-        (ctype, ctype_params) = parse_header(full_ctype)
-        boundary = ctype_params.get('boundary')
-        assert ctype == 'multipart/form-data'
+    def _process_multipart(self, length, params):
+        boundary = params.get('boundary')
         if not boundary:
             raise RequestError('multipart/form-data missing boundary')
+        charset = params.get('charset')
         mimeinput = MIMEInput(self.stdin, boundary, length)
         try:
             for line in mimeinput.readpart():
                 pass # discard lines up to first boundary
             while mimeinput.moreparts():
-                self._process_multipart_body(mimeinput)
+                self._process_multipart_body(mimeinput, charset)
         except EOFError:
             raise RequestError('unexpected end of multipart/form-data')
 
-    def _process_multipart_body(self, mimeinput):
+    def _process_multipart_body(self, mimeinput, charset):
         headers = StringIO()
         lines = mimeinput.readpart()
         for line in lines:
@@ -210,25 +223,30 @@
                 break
         headers.seek(0)
         headers = rfc822.Message(headers)
-        full_cdisp = headers.get('content-disposition')
-        if not full_cdisp:
+        ctype, ctype_params = parse_header(headers.get('content-type', ''))
+        if ctype and 'charset' in ctype_params:
+            charset = ctype_params['charset']
+        cdisp, cdisp_params = parse_header(headers.get('content-disposition',
+                                                       ''))
+        if not cdisp:
             raise RequestError('expected Content-Disposition header')
-        (cdisp, cdisp_params) = parse_header(full_cdisp)
         name = cdisp_params.get('name')
         filename = cdisp_params.get('filename')
         if not (cdisp == 'form-data' and name):
             raise RequestError('expected Content-Disposition: form-data'
-                               'with a "name" parameter: got %r' % full_cdisp)
+                               'with a "name" parameter: got %r' %
+                               headers.get('content-disposition', ''))
         # FIXME: should really to handle Content-Transfer-Encoding and other
         # MIME complexity here.  See RFC2048 for the full horror story.
         if filename:
             # it might be large file upload so use a temporary file
-            content_type = headers.get('content-type')
-            upload = Upload(filename, content_type)
+            upload = Upload(filename, ctype, charset)
             upload.receive(lines)
             _add_field_value(self.form, name, upload)
         else:
-            _add_field_value(self.form, name, ''.join(lines))
+            value = _decode_string(''.join(lines),
+                                   charset or self.DEFAULT_CHARSET)
+            _add_field_value(self.form, name, value)
 
     def get_header(self, name, default=None):
         """get_header(name : string, default : string = None) -> string
@@ -599,9 +617,11 @@
       content_type
         the content type provided by the user-agent in the request
         that uploaded this file.
+      charset
+        the charset provide by the user-agent
     """
 
-    def __init__(self, orig_filename, content_type=None):
+    def __init__(self, orig_filename, content_type=None, charset=None):
         if orig_filename:
             self.orig_filename = orig_filename
             bspos = orig_filename.rfind("\\")
@@ -621,6 +641,7 @@
             self.orig_filename = None
             self.base_filename = None
         self.content_type = content_type
+        self.charset = charset
         self.fp = None
 
     def receive(self, lines):

--Apple-Mail-5--531733632--