spambayes/spambayes storage.py,1.50,1.51

Update of /cvsroot/spambayes/spambayes/spambayes
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv5654/spambayes

Modified Files:
	storage.py 
Log Message:
Fix [ 1187208 ] import into CDB chokes on 8-bit chars

Index: storage.py
===================================================================
RCS file: /cvsroot/spambayes/spambayes/spambayes/storage.py,v
retrieving revision 1.50
retrieving revision 1.51
diff -C2 -d -r1.50 -r1.51
*** storage.py	21 Apr 2005 07:16:48 -0000	1.50
--- storage.py	22 Apr 2005 04:08:25 -0000	1.51
***************
*** 65,68 ****
--- 65,69 ----
  import os
  import sys
+ import types
  from spambayes import classifier
  from spambayes.Options import options, get_pathname_option
***************
*** 620,623 ****
--- 621,636 ----
          return wi
  
+     # Stolen from sb_dbexpimp.py
+     # Heaven only knows what encoding non-ASCII stuff will be in
+     # Try a few common western encodings and punt if they all fail
+     def uunquote(self, s):
+         for encoding in ("utf-8", "cp1252", "iso-8859-1"):
+             try:
+                 return unicode(s, encoding)
+             except UnicodeDecodeError:
+                 pass
+         # punt
+         return s
+ 
      def load(self):
          if os.path.exists(self.db_name):
***************
*** 627,631 ****
              self.nham, self.nspam = [int(i) for i in \
                                       data[self.statekey].split(',')]
!             self.wordinfo = dict([(k, self._WordInfoFactory(v)) \
                                    for k, v in data.iteritems() \
                                        if k != self.statekey])
--- 640,645 ----
              self.nham, self.nspam = [int(i) for i in \
                                       data[self.statekey].split(',')]
!             self.wordinfo = dict([(self.uunquote(k),
!                                    self._WordInfoFactory(v)) \
                                    for k, v in data.iteritems() \
                                        if k != self.statekey])
***************
*** 645,648 ****
--- 659,664 ----
          items = [(self.statekey, "%d,%d" % (self.nham, self.nspam))]
          for word, wi in self.wordinfo.iteritems():
+             if isinstance(word, types.UnicodeType):
+                 word = word.encode("utf-8")
              items.append((word, "%d,%d" % (wi.hamcount, wi.spamcount)))
          db = open(self.db_name, "wb")