rev 347 - in offlineimap/head: debian offlineimap/folder offlineimap/ui
[email protected] Wed, 8 Jan 2003 14:16:10 -0600 (CST)
| Newsgroups | gmane.mail.imap.offlineimap.subversion |
|---|---|
| Message-ID | <[email protected]> |
Author: jgoerzen Date: 2003-01-08 14:16:07 -0600 (Wed, 08 Jan 2003) New Revision: 347 Modified: offlineimap/head/debian/changelog offlineimap/head/debian/control offlineimap/head/offlineimap/folder/Base.py offlineimap/head/offlineimap/folder/IMAP.py offlineimap/head/offlineimap/folder/Maildir.py offlineimap/head/offlineimap/ui/Blinkenlights.py offlineimap/head/offlineimap/ui/UIBase.py Log: Preparing for 3.99.7 Modified: offlineimap/head/debian/control ============================================================================== --- offlineimap/head/debian/control (original) +++ offlineimap/head/debian/control Wed Jan 8 14:16:10 2003 @@ -2,7 +2,7 @@ Section: mail Priority: optional Maintainer: John Goerzen <[email protected]> -Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff +Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff, docbook-utils Standards-Version: 3.5.2 Package: offlineimap Modified: offlineimap/head/debian/changelog ============================================================================== --- offlineimap/head/debian/changelog (original) +++ offlineimap/head/debian/changelog Wed Jan 8 14:16:10 2003 @@ -1,3 +1,33 @@ +offlineimap (3.99.7) unstable; urgency=low + + * This is a 4.0 TRACK release, and may be unstable or in flux! + * Converted entire manual to DocBook SGML so it will be easier to + expand in the future. The HTML manual, also, looks far nicer now + than it did before. + * Fixed the Tk.VerboseUI -- small, silly error introduced in 3.99.6. + * Multiple performance and reliability enhancements to syncing + algorithms, as described below. + * The process of uploading new messages from local folders to the IMAP + server was not internally multi-threaded previously. Now it is. + This means that if you have a single folder with lots of new messages + locally, the processing time should be dramatically sped up. Moreover, + the process should be more reliable because we do not risk connections + going dead. + * The process of synchronizing flags has been overhauled and optimized. + Previously, for each message where a flag (seen, replied, etc.) was + changed, we'd issue a separate command to the IMAP server to adjust + things. Now, we issue one command for each flag. In other words, + instead of seing 45 messages saying something like "Adding flag S to + message 1421", you now see one message saying "Adding flag S to 45 + messages" -- and the interaction with the IMAP server may well be + almost 45 times faster on this. We will now issue at most four + commands per flag operation (add or remove) per folder, where before, + we may have issued as many as two per message. Should be a + large speedup in most cases, but a small slowdown in a few. + * Potentially increased the reliability of writing files to the Maildir. + + -- John Goerzen <[email protected]> Wed, 8 Jan 2003 07:05:01 -0600 + offlineimap (3.99.6) unstable; urgency=low * This a 4.0 TRACK release, and may be unstable or in flux! Modified: offlineimap/head/offlineimap/folder/Base.py ============================================================================== --- offlineimap/head/offlineimap/folder/Base.py (original) +++ offlineimap/head/offlineimap/folder/Base.py Wed Jan 8 14:16:10 2003 @@ -131,7 +131,7 @@ def addmessagesflags(self, uidlist, flags): for uid in uidlist: - self.addmessageflags(uid) + self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given @@ -143,6 +143,10 @@ newflags.sort() self.savemessageflags(uid, newflags) + def deletemessagesflags(self, uidlist, flags): + for uid in uidlist: + self.deletemessageflags(uid, flags) + def deletemessage(self, uid): raise NotImplementedException @@ -150,6 +154,35 @@ for uid in uidlist: self.deletemessage(uid) + def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): + if register: + UIBase.getglobalui().registerthread(self.getaccountname()) + UIBase.getglobalui().copyingmessage(uid, self, applyto) + successobject = None + successuid = None + message = self.getmessage(uid) + flags = self.getmessageflags(uid) + for tryappend in applyto: + successuid = tryappend.savemessage(uid, message, flags) + if successuid >= 0: + successobject = tryappend + break + # Did we succeed? + if successobject != None: + if successuid: # Only if IMAP actually assigned a UID + # Copy the message to the other remote servers. + for appendserver in \ + [x for x in applyto if x != successobject]: + appendserver.savemessage(successuid, message, flags) + # Copy to its new name on the local server and delete + # the one without a UID. + self.savemessage(successuid, message, flags) + self.deletemessage(uid) # It'll be re-downloaded. + else: + # Did not find any server to take this message. Ignore. + pass + + def syncmessagesto_neguid(self, dest, applyto): """Pass 1 of folder synchronization. @@ -158,33 +191,28 @@ and once that succeeds, get the UID, add it to the others for real, add it to local for real, and delete the fake one.""" - for uid in self.getmessagelist().keys(): - if uid >= 0: - continue - UIBase.getglobalui().copyingmessage(uid, self, applyto) - successobject = None - successuid = None - message = self.getmessage(uid) - flags = self.getmessageflags(uid) - for tryappend in applyto: - successuid = tryappend.savemessage(uid, message, flags) - if successuid >= 0: - successobject = tryappend - break - # Did we succeed? - if successobject != None: - if successuid: # Only if IMAP actually assigned a UID - # Copy the message to the other remote servers. - for appendserver in \ - [x for x in applyto if x != successobject]: - appendserver.savemessage(successuid, message, flags) - # Copy to its new name on the local server and delete - # the one without a UID. - self.savemessage(successuid, message, flags) - self.deletemessage(uid) # It'll be re-downloaded. + uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0] + threads = [] + + usethread = None + if applyto != None: + usethread = applyto[0] + + for uid in uidlist: + if usethread: + usethread.waitforthread() + thread = InstanceLimitedThread(\ + usethread.getcopyinstancelimit(), + target = self.syncmessagesto_neguid_msg, + name = "New msg sync from %s" % self.getvisiblename(), + args = (uid, dest, applyto)) + thread.setDaemon(1) + thread.start() + threads.append(thread) else: - # Did not find any server to take this message. Ignore. - pass + self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0) + for thread in threads: + thread.join() def copymessageto(self, uid, applyto, register = 1): # Sometimes, it could be the case that if a sync takes awhile, @@ -260,6 +288,17 @@ Look for any flag matching issues -- set dest message to have the same flags that we have.""" + + # As an optimization over previous versions, we store up which flags + # are being used for an add or a delete. For each flag, we store + # a list of uids to which it should be added. Then, we can call + # addmessagesflags() to apply them in bulk, rather than one + # call per message as before. This should result in some significant + # performance improvements. + + addflaglist = {} + delflaglist = {} + for uid in self.getmessagelist().keys(): if uid < 0: # Ignore messages missed by pass 1 continue @@ -267,17 +306,26 @@ destflags = dest.getmessageflags(uid) addflags = [x for x in selfflags if x not in destflags] - if len(addflags): - UIBase.getglobalui().addingflags(uid, addflags, applyto) - for object in applyto: - object.addmessageflags(uid, addflags) + + for flag in addflags: + if not flag in addflaglist: + addflaglist[flag] = [] + addflaglist[flag].append(uid) delflags = [x for x in destflags if x not in selfflags] - if len(delflags): - UIBase.getglobalui().deletingflags(uid, delflags, applyto) - for object in applyto: - object.deletemessageflags(uid, delflags) + for flag in delflags: + if not flag in delflaglist: + delflaglist[flag] = [] + delflaglist[flag].append(uid) + for object in applyto: + for flag in addflaglist.keys(): + UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object]) + object.addmessagesflags(addflaglist[flag], [flag]) + for flag in delflaglist.keys(): + UIBase.getglobalui().deletingflags(addflaglist[flag], flag, [object]) + object.deletemessagesflags(delflaglist[flag], [flag]) + def syncmessagesto(self, dest, applyto = None): """Syncs messages in this folder to the destination. If applyto is specified, it should be a list of folders (don't forget Modified: offlineimap/head/offlineimap/folder/Maildir.py ============================================================================== --- offlineimap/head/offlineimap/folder/Maildir.py (original) +++ offlineimap/head/offlineimap/folder/Maildir.py Wed Jan 8 14:16:10 2003 @@ -170,12 +170,13 @@ attempts += 1 else: break - file = open(os.path.join(tmpdir, messagename), "wt") + tmpmessagename = messagename.split(',')[0] + file = open(os.path.join(tmpdir, tmpmessagename), "wt") file.write(content) file.close() - os.link(os.path.join(tmpdir, messagename), + os.link(os.path.join(tmpdir, tmpmessagename), os.path.join(newdir, messagename)) - os.unlink(os.path.join(tmpdir, messagename)) + os.unlink(os.path.join(tmpdir, tmpmessagename)) self.messagelist[uid] = {'uid': uid, 'flags': [], 'filename': os.path.join(newdir, messagename)} self.savemessageflags(uid, flags) Modified: offlineimap/head/offlineimap/folder/IMAP.py ============================================================================== --- offlineimap/head/offlineimap/folder/IMAP.py (original) +++ offlineimap/head/offlineimap/folder/IMAP.py Wed Jan 8 14:16:10 2003 @@ -198,6 +198,15 @@ self.addmessagesflags([uid], flags) def addmessagesflags(self, uidlist, flags): + self.processmessagesflags('+', uidlist, flags) + + def deletemessageflags(self, uid, flags): + self.deletemessagesflags([uid], flags) + + def deletemessagesflags(self, uidlist, flags): + self.processmessagesflags('-', uidlist, flags) + + def processmessagesflags(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: @@ -207,7 +216,7 @@ return r = imapobj.uid('store', imaputil.listjoin(uidlist), - '+FLAGS', + operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + r[1] r = r[1] @@ -234,10 +243,15 @@ except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() + if operation == '+': + for flag in flags: + if not flag in self.messagelist[uid]['flags']: + self.messagelist[uid]['flags'].append(flag) + self.messagelist[uid]['flags'].sort() + elif operation == '-': + for flag in flags: + if flag in self.messagelist[uid]['flags']: + self.messagelist[uid]['flags'].remove(flag) def deletemessage(self, uid): self.deletemessages([uid]) Modified: offlineimap/head/offlineimap/ui/UIBase.py ============================================================================== --- offlineimap/head/offlineimap/ui/UIBase.py (original) +++ offlineimap/head/offlineimap/ui/UIBase.py Wed Jan 8 14:16:10 2003 @@ -221,17 +221,17 @@ ", ".join([str(u) for u in uidlist]), ds)) - def addingflags(s, uid, flags, destlist): + def addingflags(s, uidlist, flags, destlist): if s.verbose >= 0: ds = s.folderlist(destlist) - s._msg("Adding flags %s to message %d on %s" % \ - (", ".join(flags), uid, ds)) + s._msg("Adding flags %s to %d messages on %s" % \ + (", ".join(flags), len(uidlist), ds)) - def deletingflags(s, uid, flags, destlist): + def deletingflags(s, uidlist, flags, destlist): if s.verbose >= 0: ds = s.folderlist(destlist) - s._msg("Deleting flags %s to message %d on %s" % \ - (", ".join(flags), uid, ds)) + s._msg("Deleting flags %s to %d messages on %s" % \ + (", ".join(flags), len(uidlist), ds)) ################################################## Threads Modified: offlineimap/head/offlineimap/ui/Blinkenlights.py ============================================================================== --- offlineimap/head/offlineimap/ui/Blinkenlights.py (original) +++ offlineimap/head/offlineimap/ui/Blinkenlights.py Wed Jan 8 14:16:10 2003 @@ -63,13 +63,13 @@ s.gettf().setcolor('red') s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - def addingflags(s, uid, flags, destlist): + def addingflags(s, uidlist, flags, destlist): s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uid, flags, destlist) + s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist) - def deletingflags(s, uid, flags, destlist): + def deletingflags(s, uidlist, flags, destlist): s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uid, flags, destlist) + s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist) def init_banner(s): s.availablethreadframes = {}