Unicode improvements

diff -rN -u old-quixote/__init__.py new-quixote/__init__.py
--- old-quixote/__init__.py	2005-04-11 09:15:30.000000000 -0600
+++ new-quixote/__init__.py	2005-08-29 18:04:31.000000000 -0600
@@ -13,6 +13,10 @@
      get_session, get_session_manager, get_user, get_field, get_cookie
 
 
+# This is the default charset used by the HTTPRequest, HTTPResponse,
+# DefaultLogger, and sendmail components.
+DEFAULT_CHARSET = 'iso-8859-1'
+
 def enable_ptl():
     """
     Installs the import hooks needed to import PTL modules.  This must
@@ -23,4 +27,4 @@
     that, if you use ZODB, you must import ZODB before calling this
     function.
     """
-    import quixote.ptl.install 
+    import quixote.ptl.install
diff -rN -u old-quixote/http_request.py new-quixote/http_request.py
--- old-quixote/http_request.py	2005-05-18 17:42:59.000000000 -0600
+++ new-quixote/http_request.py	2005-08-29 17:46:01.000000000 -0600
@@ -13,6 +13,7 @@
 import rfc822
 from cStringIO import StringIO
 
+import quixote
 from quixote.http_response import HTTPResponse
 from quixote.errors import RequestError
 
@@ -48,7 +49,9 @@
         return None
 
 def _decode_string(s, charset):
-    if charset == 'iso-8859-1':
+    if charset == 'iso-8859-1' == quixote.DEFAULT_CHARSET:
+        # To avoid breaking applications that are not Unicode-safe, return
+        # a str instance in this case.
         return s
     try:
         return s.decode(charset)
@@ -139,13 +142,14 @@
     when handling an exception.
     """
 
-    DEFAULT_CHARSET = 'iso-8859-1'
+    DEFAULT_CHARSET = None # defaults to quixote.DEFAULT_CHARSET
 
     def __init__(self, stdin, environ):
         self.stdin = stdin
         self.environ = environ
         self.form = {}
         self.session = None
+        self.charset = self.DEFAULT_CHARSET or quixote.DEFAULT_CHARSET
         self.response = HTTPResponse()
 
         # The strange treatment of SERVER_PORT_SECURE is because IIS
@@ -179,7 +183,7 @@
     def process_inputs(self):
         query = self.get_query()
         if query:
-            self.form.update(parse_query(query, self.DEFAULT_CHARSET))
+            self.form.update(parse_query(query, self.charset))
         length = self.environ.get('CONTENT_LENGTH') or "0"
         try:
             length = int(length)
@@ -197,7 +201,9 @@
         query = self.stdin.read(length)
         if len(query) != length:
             raise RequestError('unexpected end of request body')
-        charset = params.get('charset', self.DEFAULT_CHARSET)
+        # Use the declared charset if it's provided (most browser's don't
+        # provide it to avoid breaking old HTTP servers).
+        charset = params.get('charset', self.charset)
         self.form.update(parse_query(query, charset))
 
     def _process_multipart(self, length, params):
@@ -244,8 +250,7 @@
             upload.receive(lines)
             _add_field_value(self.form, name, upload)
         else:
-            value = _decode_string(''.join(lines),
-                                   charset or self.DEFAULT_CHARSET)
+            value = _decode_string(''.join(lines), charset or self.charset)
             _add_field_value(self.form, name, value)
 
     def get_header(self, name, default=None):
diff -rN -u old-quixote/http_response.py new-quixote/http_response.py
--- old-quixote/http_response.py	2005-05-18 17:42:44.000000000 -0600
+++ new-quixote/http_response.py	2005-08-29 20:13:17.000000000 -0600
@@ -13,6 +13,7 @@
     pass
 import struct
 from rfc822 import formatdate
+import quixote
 from quixote.html import stringify
 
 status_reasons = {
@@ -95,8 +96,9 @@
       content_type : string
         the MIME content type of the response (does not include extra params
         like charset)
-      charset : string
-        the character encoding of the the response
+      charset : string | None
+        the character encoding of the the response.  If none, the 'charset'
+        parameter of the Context-Type header will not be included.
       status_code : int
         HTTP response status code (integer between 100 and 599)
       reason_phrase : string
@@ -134,14 +136,17 @@
     """
 
     DEFAULT_CONTENT_TYPE = 'text/html'
-    DEFAULT_CHARSET = 'iso-8859-1'
+    DEFAULT_CHARSET = None # defaults to quixote.DEFAULT_CHARSET
+
     
     def __init__(self, status=200, body=None, content_type=None, charset=None):
         """
         Creates a new HTTP response.
         """
         self.content_type = content_type or self.DEFAULT_CONTENT_TYPE
-        self.charset = charset or self.DEFAULT_CHARSET
+        self.charset = (charset or
+                        self.DEFAULT_CHARSET or
+                        quixote.DEFAULT_CHARSET)
         self.set_status(status)
         self.headers = {}
 
@@ -155,17 +160,25 @@
         self.buffered = True
         self.javascript_code = None
 
-    def set_content_type(self, content_type, charset='iso-8859-1'):
-        """(content_type : string, charset : string = 'iso-8859-1')
+    def set_content_type(self, content_type, charset=None):
+        """(content_type : string, charset : string = None)
 
         Set the content type of the response to the MIME type specified by
-        'content_type'.  Also sets the charset, defaulting to 'iso-8859-1'.
-        """
-        self.charset = charset
+        'content_type'.  If 'charset' is not provided and the content_type is
+        text/* then the charset attribute remains unchanged, otherwise the
+        charset attribute is set to None and the charset parameter will not
+        be included as part of the Content-Type header.
+        """
+        content_type = content_type.lower()
+        if charset is not None or not content_type.startswith('text/'):
+            self.charset = charset
         self.content_type = content_type
 
     def set_charset(self, charset):
-        self.charset = str(charset).lower()
+        if not charset:
+            self.charset = None
+        else:
+            self.charset = str(charset).lower()
 
     def set_status(self, status, reason=None):
         """set_status(status : int, reason : string = None)
@@ -220,10 +233,18 @@
     def _encode_chunk(self, chunk):
         """(chunk : str | unicode) -> str
         """
-        if self.charset == 'iso-8859-1' and isinstance(chunk, str):
-            return chunk # non-ASCII chars are okay
+        if isinstance(chunk, unicode):
+            if self.charset is None:
+                # iso-8859-1 is the default for the HTTP protocol if charset
+                # parameter of content-type header is not provided
+                chunk = chunk.encode('iso-8859-1')
+            else:
+                chunk = chunk.encode(self.charset)
         else:
-            return chunk.encode(self.charset)
+            # we assume that the str is in the correct encoding or does
+            # not contain character data
+            pass
+        return chunk
 
     def _compress_body(self, body):
         """(body: str) -> str
@@ -401,9 +422,11 @@
 
         # Content-type
         if "content-type" not in self.headers:
-            headers.append(('Content-Type',
-                            '%s; charset=%s' % (self.content_type,
-                                                self.charset)))
+            if self.charset is not None:
+                value = '%s; charset=%s' % (self.content_type, self.charset)
+            else:
+                value = '%s' % self.content_type
+            headers.append(('Content-Type', value))
 
         # Content-Length
         if "content-length" not in self.headers:
diff -rN -u old-quixote/logger.py new-quixote/logger.py
--- old-quixote/logger.py	2005-03-17 14:12:10.000000000 -0700
+++ new-quixote/logger.py	2005-08-29 19:28:02.000000000 -0600
@@ -3,8 +3,10 @@
 """
 import sys
 import os
+import codecs
 import time
 import socket
+import quixote
 from quixote.sendmail import sendmail
 
 class DefaultLogger:
@@ -26,25 +28,37 @@
         if set then internal server errors will cause messages to be sent to
         this address
     """
+
+    DEFAULT_CHARSET = None # defaults to quixote.DEFAULT_CHARSET
+
     def __init__(self, access_log=None, error_log=None, error_email=None):
         if access_log:
-            self.access_log = open(access_log, 'a', 1)
+            self.access_log = self._open_log(access_log)
         else:
             self.access_log = None
         if error_log is None:
             self.error_log = sys.stderr
         else:
-            self.error_log = open(error_log, 'a', 1)
+            self.error_log = self._open_log(error_log)
         self.error_email = error_email
         sys.stdout = self.error_log # print is handy for debugging
 
+    def _open_log(self, filename):
+        charset = self.DEFAULT_CHARSET or quixote.DEFAULT_CHARSET
+        if charset == 'iso-8859-1':
+            return open(filename, 'ab', 1)
+        else:
+            return codecs.open(filename, 'ab',
+                               encoding=charset,
+                               buffering=1)
+
     def log(self, msg):
         """
         Write an message to the error log with a time stamp.
         """
         timestamp = time.strftime("%Y-%m-%d %H:%M:%S",
                                   time.localtime(time.time()))
-        self.error_log.write("[%s] %s\n" % (timestamp, msg))
+        self.error_log.write("[%s] %s%s" % (timestamp, msg, os.linesep))
 
     def log_internal_error(self, error_summary, error_msg):
         """(error_summary: str, error_msg: str)
@@ -78,7 +92,7 @@
         if query:
             request_uri += "?" + query
         proto = request.get_environ('SERVER_PROTOCOL')
-        self.access_log.write('%s %s %s %d "%s %s %s" %s %r %0.2fsec\n' %
+        self.access_log.write('%s %s %s %d "%s %s %s" %s %r %0.2fsec%s' %
                                (request.get_environ('REMOTE_ADDR'),
                                 user,
                                 timestamp,
@@ -88,5 +102,6 @@
                                 proto,
                                 request.response.status_code,
                                 request.get_environ('HTTP_USER_AGENT', ''),
-                                seconds
+                                seconds,
+                                os.linesep,
                                ))
diff -rN -u old-quixote/sendmail.py new-quixote/sendmail.py
--- old-quixote/sendmail.py	2005-05-11 14:57:56.000000000 -0600
+++ new-quixote/sendmail.py	2005-08-29 18:03:02.000000000 -0600
@@ -8,6 +8,7 @@
 import re
 from types import ListType, TupleType, StringType
 from smtplib import SMTP
+import quixote
 
 rfc822_specials_re = re.compile(r'[\(\)\<\>\@\,\;\:\\\"\.\[\]]')
 
@@ -220,7 +221,9 @@
     headers = ["From: %s" % from_addr.format(),
                "Subject: %s" % subject]
     _add_recip_headers(headers, "To", to_addrs)
-
+    if quixote.DEFAULT_CHARSET != 'iso-8859-1':
+        headers.append('Content-Type: text/plain; charset=%s' %
+                       quixote.DEFAULT_CHARSET)
     if cc_addrs:
         _add_recip_headers(headers, "Cc", cc_addrs)
 
@@ -255,6 +258,8 @@
                            for recip in smtp_recipients]
 
     message = "\n".join(headers) + "\n\n" + msg_body
+    if quixote.DEFAULT_CHARSET != 'iso-8859-1':
+        message = message.encode(quixote.DEFAULT_CHARSET)
 
     # Sanity checks
     assert type(smtp_sender) is StringType, \