SVN: ZODB/branches/3.8/ New Feature:
Jim Fulton <[email protected]>
| Newsgroups | gmane.comp.python.zope.zodb.cvs |
|---|---|
| Message-ID | <[email protected]> |
Log message for revision 102792:
New Feature:
- There's a new utility script, strip_versions that strips version
data from storages. This is needed to prepare databases containing
version records for using ZODB 3.9, which no-longer supports
versions.
Changed:
U ZODB/branches/3.8/NEWS.txt
U ZODB/branches/3.8/setup.py
A ZODB/branches/3.8/src/ZODB/scripts/strip_versions.py
A ZODB/branches/3.8/src/ZODB/scripts/strip_versions.test
U ZODB/branches/3.8/src/ZODB/scripts/tests.py
-=-
Modified: ZODB/branches/3.8/NEWS.txt
===================================================================
--- ZODB/branches/3.8/NEWS.txt 2009-08-14 20:53:21 UTC (rev 102791)
+++ ZODB/branches/3.8/NEWS.txt 2009-08-14 21:03:37 UTC (rev 102792)
@@ -1,6 +1,13 @@
-Next Release
-===========
+Whats new in ZODB 3.8.2
+=======================
+New Feature:
+
+- There's a new utility script, strip_versions that strips version
+ data from storages. This is needed to prepare databases containing
+ version records for using ZODB 3.9, which no-longer supports
+ versions.
+
Bugs Fixed:
- Fixed ``NameError`` in cases where a directory cannot be created,
Modified: ZODB/branches/3.8/setup.py
===================================================================
--- ZODB/branches/3.8/setup.py 2009-08-14 20:53:21 UTC (rev 102791)
+++ ZODB/branches/3.8/setup.py 2009-08-14 21:03:37 UTC (rev 102792)
@@ -56,6 +56,7 @@
"src/ZODB/scripts/fstail.py",
"src/ZODB/scripts/fstest.py",
"src/ZODB/scripts/repozo.py",
+ "src/ZODB/scripts/strip_versions.py",
"src/ZEO/scripts/zeopack.py",
"src/ZEO/scripts/runzeo.py",
"src/ZEO/scripts/zeopasswd.py",
@@ -71,6 +72,7 @@
fsrefs = ZODB.scripts.fsrefs:main
fstail = ZODB.scripts.fstail:Main
repozo = ZODB.scripts.repozo:main
+ strip_versions = ZODB.scripts.strip_versions:main
zeopack = ZEO.scripts.zeopack:main
runzeo = ZEO.runzeo:main
zeopasswd = ZEO.zeopasswd:main
Added: ZODB/branches/3.8/src/ZODB/scripts/strip_versions.py
===================================================================
--- ZODB/branches/3.8/src/ZODB/scripts/strip_versions.py (rev 0)
+++ ZODB/branches/3.8/src/ZODB/scripts/strip_versions.py 2009-08-14 21:03:37 UTC (rev 102792)
@@ -0,0 +1,102 @@
+##############################################################################
+#
+# Copyright Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+import optparse
+import sys
+import ZODB.BaseStorage
+import ZODB.config
+import ZODB.FileStorage
+
+parser = optparse.OptionParser("""%prog input output
+
+Remove version records by copying input to output, stripping any
+version records seen in input.
+
+The input and output arguments are file-storage file names. If the -c
+option is used, then the input and output arguments must be storage
+configuration files.
+""")
+parser.add_option(
+ "-c", "--config", dest="config", action="store_true",
+ help=
+ 'Treat the input and output names as configuration file names,'
+ ' rather than file-storage file names')
+
+class Record:
+
+ def __init__(self, transaction, it):
+ self.__transaction = transaction
+ self.__it = it
+
+ def __getattr__(self, name):
+ return getattr(self.__transaction, name)
+
+ def __iter__(self):
+ it = self.__it
+ for record in self.__transaction:
+ if record.version:
+ it.versioned += 1
+ continue # Strip version records
+ it.copied += 1
+ if hasattr(record, 'data_txn'):
+ record.data_txn = None
+ yield record
+
+class Iterator:
+
+ versioned = copied = 0
+
+ def __init__(self, iterator):
+ self.close = iterator.close
+ def it():
+ for transaction in iterator:
+ yield Record(transaction, self)
+ self.__it = it()
+
+ def next(self):
+ return self.__it.next()
+
+ def __iter__(self):
+ return self
+ iterator = __iter__
+
+def main(args=None):
+ if args is None:
+ args = sys.argv[1:]
+
+ (options, args) = parser.parse_args(args)
+
+ if len(args) != 2:
+ parser.parse_args(['-h'])
+
+ input, output = args
+ toclose = []
+ if options.config:
+ input_storage = ZODB.config.storageFromFile(open(input))
+ it = input_storage.iterator()
+ toclose.append(input_storage)
+ output_storage = ZODB.config.storageFromFile(open(output))
+ else:
+ it = ZODB.FileStorage.FileIterator(input)
+ output_storage = ZODB.FileStorage.FileStorage(output)
+ toclose.append(output_storage)
+
+ it = Iterator(it)
+ ZODB.BaseStorage.copy(it, output_storage)
+
+ for s in toclose:
+ s.close()
+
+ print 'Copied', it.copied, 'records.'
+ print 'Removed', it.versioned, 'versioned records.'
Property changes on: ZODB/branches/3.8/src/ZODB/scripts/strip_versions.py
___________________________________________________________________
Added: svn:keywords
+ Id
Added: svn:eol-style
+ native
Added: ZODB/branches/3.8/src/ZODB/scripts/strip_versions.test
===================================================================
--- ZODB/branches/3.8/src/ZODB/scripts/strip_versions.test (rev 0)
+++ ZODB/branches/3.8/src/ZODB/scripts/strip_versions.test 2009-08-14 21:03:37 UTC (rev 102792)
@@ -0,0 +1,119 @@
+Strip version records from a FileStorage database.
+==================================================
+
+Older releases of ZODB supported versions. ZODB versions 3.9 and
+later do not. The strip_versions script strips version records from
+old data files.
+
+We have a demo data file that has some version data in it:
+
+>>> open('old.fs', 'wb').write("""\
+... RlMyMQOAAADPZRciAAAAAAAAAJwgAAAAGQAAaW5pdGlhbCBkYXRhYmFzZSBjcmVhdGlvbgAAAAAA
+... AAAAA4AAAM9lFyIAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAQihjcGVyc2lzdGVudC5tYXBwaW5n
+... ClBlcnNpc3RlbnRNYXBwaW5nCnEBTnQufXECVQpfY29udGFpbmVycQN9cQRzLgAAAAAAAACcA4AA
+... AM9mXgAAAAAAAAAA/iAAAAAAAAAAAAAAAAAAAAOAAADPZl4AAAAAAAAAADQAAAAAAAAAqAAAAAAA
+... AAAAAFRjcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1xAihVAWEoVQgA
+... AAAAAAAAAXEDaAF0UVUKX2NvbnRhaW5lcnEEfXEFdS4AAAAAAAAAAQOAAADPZl4AAAAAAAAAAAAA
+... AAAAAAAAqAAAAAAAAAAAAD9jcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEB
+... Ln1xAlUKX2NvbnRhaW5lcnEDfXEEcy4AAAAAAAAA/gOAAADPZ30zAAAAAAAAARIgAAAAAAAAAAAA
+... AAAAAAADgAAAz2d9MwAAAAAAAAC/AAAAAAAAAa4AAAAAAAAAAABoY3BlcnNpc3RlbnQubWFwcGlu
+... ZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQIoVQFhKFUIAAAAAAAAAAFxA2gBdFFVCl9jb250YWlu
+... ZXJxBH1xBVUBYihVCAAAAAAAAAACcQZoAXRRdS4AAAAAAAAAAgOAAADPZ30zAAAAAAAAAAAAAAAA
+... AAABrgAAAAAAAAAAAD9jcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1x
+... AlUKX2NvbnRhaW5lcnEDfXEEcy4AAAAAAAABEgOAAADPaWndAAAAAAAAAJggAAAAAAAAAAAAAAAA
+... AAEDgAAAz2lp3QAAAAAAAAE9AAAAAAAAAsgAAgAAAAAAAABFAAAAAAAAAT0AAAAAAAAAAHYxY3Bl
+... cnNpc3RlbnQubWFwcGluZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31x
+... BFUBMUsBc3MuAAAAAAAAAJgDgAAAz2p4zAAAAAAAAAEoIAAAAAAAAAAAAAAAAAABA4AAAM9qeMwA
+... AAAAAAAC3wAAAAAAAANoAAIAAAAAAAAAWgAAAAAAAAE9AAAAAAAAAt92MWNwZXJzaXN0ZW50Lm1h
+... cHBpbmcKUGVyc2lzdGVudE1hcHBpbmcKcQEufXECKFUBYShVCAAAAAAAAAADcQNoAXRRVQpfY29u
+... dGFpbmVycQR9cQVVATFLAXN1LgAAAAAAAAADA4AAAM9qeMwAAAAAAAAAAAAAAAAAAANoAAIAAAAA
+... AAAAPwAAAAAAAAAAAAAAAAAAA392MWNwZXJzaXN0ZW50Lm1hcHBpbmcKUGVyc2lzdGVudE1hcHBp
+... bmcKcQEufXECVQpfY29udGFpbmVycQN9cQRzLgAAAAAAAAEoA4AAAM9sJXcAAAAAAAAAmCAAAAAA
+... AAAAAAAAAAAAAgOAAADPbCV3AAAAAAAAAlcAAAAAAAAEmAACAAAAAAAAAEUAAAAAAAACVwAAAAAA
+... AAAAdjJjcGVyc2lzdGVudC5tYXBwaW5nClBlcnNpc3RlbnRNYXBwaW5nCnEBLn1xAlUKX2NvbnRh
+... aW5lcnEDfXEEVQExSwFzcy4AAAAAAAAAmAOAAADPbTXMAAAAAAAAASggAAAAAAAAAAAAAAAAAAID
+... gAAAz201zAAAAAAAAASvAAAAAAAABTgAAgAAAAAAAABaAAAAAAAAAlcAAAAAAAAEr3YyY3BlcnNp
+... c3RlbnQubWFwcGluZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQIoVQpfY29udGFpbmVycQN9cQRV
+... ATFLAXNVAWIoVQgAAAAAAAAABHEFaAF0UXUuAAAAAAAAAAQDgAAAz201zAAAAAAAAAAAAAAAAAAA
+... BTgAAgAAAAAAAAA/AAAAAAAAAAAAAAAAAAAFT3YyY3BlcnNpc3RlbnQubWFwcGluZwpQZXJzaXN0
+... ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31xBHMuAAAAAAAAASgDgAAAz26rZgAAAAAA
+... AAB7IAAAAAAAAAAAAAAAAAAEA4AAAM9uq2YAAAAAAAAF5QAAAAAAAAZoAAAAAAAAAAAAAAAAAAAA
+... AAXlAAAAAAAAAAIDgAAAz26rZgAAAAAAAAVPAAAAAAAABmgAAAAAAAAAAAAAAAAAAAAABU8AAAAA
+... AAAAewOAAADPcGhEAAAAAAAAAK0gAAAAAAAAAAAAAAAAAAIDgAAAz3BoRAAAAAAAAAaxAAAAAAAA
+... BusAAgAAAAAAAABaAAAAAAAABrEAAAAAAAAAAHYzY3BlcnNpc3RlbnQubWFwcGluZwpQZXJzaXN0
+... ZW50TWFwcGluZwpxAS59cQIoVQpfY29udGFpbmVycQN9cQRVATFLAnNVAWIoVQgAAAAAAAAABHEF
+... aAF0UXUuAAAAAAAAAK0DgAAAz3GSVQAAAAAAAAE8IAAAAAAAAAAAAAAAAAACA4AAAM9xklUAAAAA
+... AAAHAgAAAAAAAAegAAIAAAAAAAAAbgAAAAAAAAaxAAAAAAAABwJ2M2NwZXJzaXN0ZW50Lm1hcHBp
+... bmcKUGVyc2lzdGVudE1hcHBpbmcKcQEufXECKFUBYyhVCAAAAAAAAAAFcQNoAXRRVQFiKFUIAAAA
+... AAAAAARxBGgBdFFVCl9jb250YWluZXJxBX1xBlUBMUsCc3UuAAAAAAAAAAUDgAAAz3GSVQAAAAAA
+... AAAAAAAAAAAAB6AAAgAAAAAAAAA/AAAAAAAAAAAAAAAAAAAHt3YzY3BlcnNpc3RlbnQubWFwcGlu
+... ZwpQZXJzaXN0ZW50TWFwcGluZwpxAS59cQJVCl9jb250YWluZXJxA31xBHMuAAAAAAAAATwDgAAA
+... z3L6MwAAAAAAAAB7IAAAAAAAAAAAAAAAAAAFA4AAAM9y+jMAAAAAAAAIYQAAAAAAAAjkAAAAAAAA
+... AAAAAAAAAAAAAAAAAAAAAAAAAAIDgAAAz3L6MwAAAAAAAAe3AAAAAAAACOQAAAAAAAAAAAAAAAAA
+... AAAABrEAAAAAAAAAew==""".decode('base64'))
+
+Let's strip the versions:
+
+ >>> import ZODB.scripts.strip_versions
+ >>> ZODB.scripts.strip_versions.main('old.fs new.fs'.split())
+ Copied 9 records.
+ Removed 9 versioned records.
+
+ >>> import ZODB.FileStorage
+ >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('new.fs'))
+
+ >>> conn = db.open()
+ >>> oids = sorted(db._storage._index)
+ >>> import ZODB.utils
+ >>> for oid in oids:
+ ... try:
+ ... ob = conn.get(oid)
+ ... except ZODB.POSException.POSKeyError:
+ ... continue
+ ... print ZODB.utils.u64(oid), sorted(ob.items())
+ 0 []
+ 1 []
+ 2 [('1', 1)]
+ 4 []
+
+ >>> db.close()
+
+The example above assumed the old and new databases were both file
+storages. If we want to use some other storage formats, we need to
+specify the old and new databases using storage configuration files.
+If the -c (--config) option is used, then the arguments are treated as
+configuration file names.
+
+ >>> import os
+ >>> open('old.conf', 'w').write("""
+ ... <filestorage>
+ ... path old.fs
+ ... read-only true
+ ... </filestorage>
+ ... """)
+ >>> open('new.conf', 'w').write("""
+ ... <filestorage>
+ ... path new2.fs
+ ... </filestorage>
+ ... """)
+
+ >>> ZODB.scripts.strip_versions.main('-c old.conf new.conf'.split())
+ Copied 9 records.
+ Removed 9 versioned records.
+
+ >>> db = ZODB.DB(ZODB.FileStorage.FileStorage('new.fs'))
+ >>> conn = db.open()
+ >>> oids = sorted(db._storage._index)
+ >>> import ZODB.utils
+ >>> for oid in oids:
+ ... try:
+ ... ob = conn.get(oid)
+ ... except ZODB.POSException.POSKeyError:
+ ... continue
+ ... print ZODB.utils.u64(oid), sorted(ob.items())
+ 0 []
+ 1 []
+ 2 [('1', 1)]
+ 4 []
+
+ >>> db.close()
Property changes on: ZODB/branches/3.8/src/ZODB/scripts/strip_versions.test
___________________________________________________________________
Added: svn:eol-style
+ native
Modified: ZODB/branches/3.8/src/ZODB/scripts/tests.py
===================================================================
--- ZODB/branches/3.8/src/ZODB/scripts/tests.py 2009-08-14 20:53:21 UTC (rev 102791)
+++ ZODB/branches/3.8/src/ZODB/scripts/tests.py 2009-08-14 21:03:37 UTC (rev 102792)
@@ -17,9 +17,12 @@
"""
import unittest
from zope.testing import doctest
+import ZODB.tests.util
def test_suite():
return unittest.TestSuite((
- doctest.DocFileSuite('referrers.txt'),
+ doctest.DocFileSuite('referrers.txt', 'strip_versions.test',
+ setUp=ZODB.tests.util.setUp, tearDown=ZODB.tests.util.tearDown,
+ ),
))