r13383 - in Products.Archetypes/trunk: . Products/Archetypes Products/Archetypes/examples Products/Archetypes/tests
"Hanno Schlichting" <[email protected]>
| Newsgroups | gmane.comp.web.zope.plone.archetypes.cvs |
|---|---|
| Message-ID | <[email protected]> |
Author: hannosch
Date: Sat Dec 25 20:41:18 2010
New Revision: 13383
Added:
Products.Archetypes/trunk/Products/Archetypes/tests/test_reference_optimizations.py (contents, props changed)
Modified:
Products.Archetypes/trunk/CHANGES.txt
Products.Archetypes/trunk/Products/Archetypes/ReferenceEngine.py
Products.Archetypes/trunk/Products/Archetypes/Referenceable.py
Products.Archetypes/trunk/Products/Archetypes/examples/DDocument.py
Products.Archetypes/trunk/Products/Archetypes/tests/test_referenceCatalog.py
Log:
Merged optimizations from the `experimental.atrefspeedup` distribution into the reference engine. We avoid intersections with the large relationship index and loading the persistent reference objects where possible.
Modified: Products.Archetypes/trunk/CHANGES.txt
==============================================================================
--- Products.Archetypes/trunk/CHANGES.txt (original)
+++ Products.Archetypes/trunk/CHANGES.txt Sat Dec 25 20:41:18 2010
@@ -4,6 +4,11 @@
1.7 - Unreleased
----------------
+- Merged optimizations from the `experimental.atrefspeedup` distribution into
+ the reference engine. We avoid intersections with the large relationship
+ index and loading the persistent reference objects where possible.
+ [hannosch]
+
- Fix Referenceable, UIDCatalog to support references to non-Archetypes-based
content.
[toutpt]
Modified: Products.Archetypes/trunk/Products/Archetypes/ReferenceEngine.py
==============================================================================
--- Products.Archetypes/trunk/Products/Archetypes/ReferenceEngine.py (original)
+++ Products.Archetypes/trunk/Products/Archetypes/ReferenceEngine.py Sat Dec 25 20:41:18 2010
@@ -6,10 +6,9 @@
from Products.CMFCore.utils import getToolByName
from Products.Archetypes.interfaces.referenceable import IReferenceable
+from Products.Archetypes.interfaces import IContentReference
from Products.Archetypes.interfaces import IReference
from Products.Archetypes.interfaces import IReferenceCatalog
-from Products.Archetypes.interfaces.referenceengine import \
- IContentReference, IReference as Z2IReference
from Products.Archetypes.utils import make_uuid, getRelURL, shasattr
from Products.Archetypes.config import (
@@ -29,6 +28,7 @@
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.ZCatalog.ZCatalog import ZCatalog
from Products.ZCatalog.Catalog import Catalog
+from Products.ZCatalog.Lazy import LazyMap
from Products import CMFCore
from plone.uuid.interfaces import IUUID
@@ -87,28 +87,16 @@
"""the uid method for compat"""
return IUUID(self, None)
- ###
# Convenience methods
+
def getSourceObject(self):
- tool = getToolByName(self, UID_CATALOG, None)
- if tool is None: return ''
- brains = tool(UID=self.sourceUID)
- for brain in brains:
- obj = brain.getObject()
- if obj is not None:
- return obj
+ return self._optimizedGetObject(self.sourceUID)
def getTargetObject(self):
- tool = getToolByName(self, UID_CATALOG, None)
- if tool is None: return ''
- brains = tool(UID=self.targetUID)
- for brain in brains:
- obj = brain.getObject()
- if obj is not None:
- return obj
+ return self._optimizedGetObject(self.targetUID)
- ###
# Catalog support
+
def targetId(self):
target = self.getTargetObject()
if target is not None:
@@ -368,28 +356,77 @@
for b in self.getBackReferences(object, relationship):
self._deleteReference(b)
- def getReferences(self, object, relationship=None, targetObject=None):
+ def getReferences(self, object, relationship=None, targetObject=None,
+ objects=True):
"""return a collection of reference objects"""
- sID, sobj = self._uidFor(object)
- if targetObject:
- tID, tobj = self._uidFor(targetObject)
- else:
- tID, tobj = None,None
+ return self._optimizedReferences(object, relationship=relationship,
+ targetObject=targetObject, objects=objects, attribute='sourceUID')
- brains = self._queryFor(sid=sID, relationship=relationship, tid=tID)
- return self._resolveBrains(brains)
-
- def getBackReferences(self, object, relationship=None, targetObject=None):
+ def getBackReferences(self, object, relationship=None, targetObject=None,
+ objects=True):
"""return a collection of reference objects"""
# Back refs would be anything that target this object
+ return self._optimizedReferences(object, relationship=relationship,
+ targetObject=targetObject, objects=objects, attribute='targetUID')
+
+ def _optimizedReferences(self, object, relationship=None,
+ targetObject=None, objects=True, attribute='sourceUID'):
+
sID, sobj = self._uidFor(object)
if targetObject:
tID, tobj = self._uidFor(targetObject)
+ if attribute == 'sourceUID':
+ brains = self._queryFor(sID, tID, relationship)
+ else:
+ brains = self._queryFor(tID, sID, relationship)
else:
- tID, tobj = None,None
+ brains = self._optimizedQuery(sID, attribute, relationship)
+
+ if objects:
+ return self._resolveBrains(brains)
+ return brains
+
+ def _optimizedQuery(self, uid, indexname, relationship):
+ """query reference catalog for object matching the info we are
+ given, returns brains
+ """
+ if not uid: # pragma: no cover
+ return []
- brains = self._queryFor(tid=sID, relationship=relationship, sid=tID)
- return self._resolveBrains(brains)
+ _catalog = self._catalog
+ indexes = _catalog.indexes
+
+ # First get one or multiple record ids for the source/target uid index
+ rids = indexes[indexname]._index.get(uid, None)
+ if rids is None:
+ return []
+ elif isinstance(rids, int):
+ rids = [rids]
+ else:
+ rids = list(rids)
+
+ # As a second step make sure we only get references of the right type
+ # The unindex holds data of the type: [(-311870037, 'relatesTo')]
+ # The index holds data like: [('relatesTo', -311870037)]
+ if relationship is None:
+ result_rids = rids
+ else:
+ rel_unindex_get = indexes['relationship']._unindex.get
+ result_rids = set()
+ if isinstance(relationship, str):
+ relationship = set([relationship])
+ for r in rids:
+ rels = rel_unindex_get(r, None)
+ if rels is None:
+ rels = set()
+ elif isinstance(rels, str):
+ rels = set([rels])
+ if not rels.isdisjoint(relationship):
+ result_rids.add(r)
+
+ # Create brains
+ return LazyMap(_catalog.__getitem__,
+ list(result_rids), len(result_rids))
def hasRelationshipTo(self, source, target, relationship):
sID, sobj = self._uidFor(source)
Modified: Products.Archetypes/trunk/Products/Archetypes/Referenceable.py
==============================================================================
--- Products.Archetypes/trunk/Products/Archetypes/Referenceable.py (original)
+++ Products.Archetypes/trunk/Products/Archetypes/Referenceable.py Sat Dec 25 20:41:18 2010
@@ -11,7 +11,6 @@
from OFS.ObjectManager import BeforeDeleteException
from Products.CMFCore.utils import getToolByName
-from Products.CMFCore.permissions import View
from OFS.CopySupport import CopySource
from OFS.Folder import Folder
from utils import getRelURL
@@ -75,10 +74,11 @@
def getRefs(self, relationship=None, targetObject=None):
"""get all the referenced objects for this object"""
- tool = getToolByName(self, config.REFERENCE_CATALOG)
- refs = tool.getReferences(self, relationship, targetObject=targetObject)
- if refs:
- return [ref.getTargetObject() for ref in refs]
+ tool = getToolByName(self, 'reference_catalog')
+ brains = tool.getReferences(self, relationship, targetObject=targetObject,
+ objects=False)
+ if brains:
+ return [self._optimizedGetObject(b.targetUID) for b in brains]
return []
def _getURL(self):
@@ -87,10 +87,11 @@
def getBRefs(self, relationship=None, targetObject=None):
"""get all the back referenced objects for this object"""
- tool = getToolByName(self, config.REFERENCE_CATALOG)
- refs = tool.getBackReferences(self, relationship, targetObject=targetObject)
- if refs:
- return [ref.getSourceObject() for ref in refs]
+ tool = getToolByName(self, 'reference_catalog')
+ brains = tool.getBackReferences(self, relationship,
+ targetObject=targetObject, objects=False)
+ if brains:
+ return [self._optimizedGetObject(b.sourceUID) for b in brains]
return []
#aliases
@@ -113,6 +114,24 @@
return refs
return []
+ def _optimizedGetObject(self, uid):
+ tool = getToolByName(self, 'uid_catalog', None)
+ if tool is None: # pragma: no cover
+ return ''
+ tool = aq_inner(tool)
+ traverse = aq_parent(tool).unrestrictedTraverse
+
+ _catalog = tool._catalog
+ rids = _catalog.indexes['UID']._index.get(uid, ())
+ if isinstance(rids, int):
+ rids = (rids, )
+
+ for rid in rids:
+ path = _catalog.paths[rid]
+ obj = traverse(path, default=None)
+ if obj is not None:
+ return obj
+
def _register(self, reference_manager=None):
"""register with the archetype tool for a unique id"""
if IUUID(self, None) is not None:
Modified: Products.Archetypes/trunk/Products/Archetypes/examples/DDocument.py
==============================================================================
--- Products.Archetypes/trunk/Products/Archetypes/examples/DDocument.py (original)
+++ Products.Archetypes/trunk/Products/Archetypes/examples/DDocument.py Sat Dec 25 20:41:18 2010
@@ -40,7 +40,13 @@
ReferenceField('related',
relationship = 'related',
- multiValued = False,
+ multiValued = True,
+ widget=ReferenceWidget(),
+ keepReferencesOnCopy = True),
+
+ ReferenceField('rel2',
+ relationship = 'rel2',
+ multiValued = True,
widget=ReferenceWidget(),
keepReferencesOnCopy = True),
),
Modified: Products.Archetypes/trunk/Products/Archetypes/tests/test_referenceCatalog.py
==============================================================================
--- Products.Archetypes/trunk/Products/Archetypes/tests/test_referenceCatalog.py (original)
+++ Products.Archetypes/trunk/Products/Archetypes/tests/test_referenceCatalog.py Sat Dec 25 20:41:18 2010
@@ -343,21 +343,19 @@
self.failUnless(results[0].Title==str(dext.Title()))
def test_reference_non_archetypes_content(self):
- #create a archetype based content instance
+ # create a archetype based content instance
ob = makeContent(self.folder, portal_type='DDocument',id='mydocument')
uc = getattr(self.portal, config.UID_CATALOG)
uc.catalog_object(ob, '/'.join(ob.getPhysicalPath()))
- #create a non archetype based content
+ # create a non archetype based content
dext = DexterityLike()
dext.path = list(self.folder.getPhysicalPath())
self.folder[dext.id] = dext
notify(ObjectCreatedEvent(dext)) #it supposed to add uuid attribute
uc.catalog_object(dext, '/'.join(dext.getPhysicalPath()))
- #TODO: create the relation between those
+ # create the relation between those
ob.setRelated(dext)
- related = ob.getRelated()
-
- self.failUnless(related==dext)
+ self.assertEqual(ob.getRelated()[0], dext)
def test_suite():
from unittest import TestSuite, makeSuite
Added: Products.Archetypes/trunk/Products/Archetypes/tests/test_reference_optimizations.py
==============================================================================
--- (empty file)
+++ Products.Archetypes/trunk/Products/Archetypes/tests/test_reference_optimizations.py Sat Dec 25 20:41:18 2010
@@ -0,0 +1,250 @@
+from Products.CMFCore.utils import getToolByName
+
+from Products.Archetypes.tests.atsitetestcase import ATSiteTestCase
+from Products.Archetypes.tests.utils import makeContent
+
+
+class ATRefSpeedupTestCase(ATSiteTestCase):
+
+ def afterSetUp(self):
+ self.rc = getToolByName(self.portal, 'reference_catalog')
+ self.loginAsPortalOwner()
+ makeContent(self.portal, portal_type='DDocument', id='doc1')
+ makeContent(self.portal, portal_type='DDocument', id='doc2')
+ makeContent(self.portal, portal_type='DDocument', id='doc3')
+
+
+class TestGetReferences(ATRefSpeedupTestCase):
+
+ def test_none(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ self.assertEquals(self.rc.getReferences(doc1), [])
+ self.assertEquals(self.rc.getReferences(doc1, 'related'), [])
+ self.assertEquals(self.rc.getReferences(doc1, 'related', doc2), [])
+
+ result = self.rc.getReferences(doc1, ['related', 'rel2'])
+ self.assertEquals(result, [])
+ result = self.rc.getReferences(doc1, ['related', 'rel2'], doc2)
+ self.assertEquals(result, [])
+
+ self.assertEquals(doc1.getReferences(), [])
+
+ def test_single(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ doc1.setRelated([doc2.UID()])
+ result = self.rc.getReferences(doc1)
+ self.assertEquals(result[0].getTargetObject(), doc2)
+ result = self.rc.getReferences(doc1, 'related')
+ self.assertEquals(result[0].getTargetObject(), doc2)
+ result = self.rc.getReferences(doc1, 'related', doc2)
+ self.assertEquals(result[0].getTargetObject(), doc2)
+ result = self.rc.getReferences(doc1, 'related', doc3)
+ self.assertEquals(result, [])
+
+ self.assertEquals(doc1.getReferences()[0], doc2)
+
+ doc1.setRel2([doc3.UID()])
+ result = self.rc.getReferences(doc1, ['related', 'rel2'])
+ result = [r.getTargetObject() for r in result]
+ self.assertEquals(set(result), set([doc2, doc3]))
+
+ result = self.rc.getReferences(doc1, ['related', 'rel2'], doc2)
+ result = [r.getTargetObject() for r in result]
+ self.assertEquals(set(result), set([doc2]))
+
+ def test_many(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ uids = [doc2.UID(), doc3.UID()]
+ doc1.setRelated(uids)
+ result = [r.getTargetObject() for r in self.rc.getReferences(doc1)]
+ self.assertEquals(set(result), set([doc2, doc3]))
+
+ self.assertEquals(set(doc1.getReferences()), set([doc2, doc3]))
+
+ doc1.setRel2([doc2.UID()])
+ result = self.rc.getReferences(doc1, ['related', 'rel2'])
+ result = [r.getTargetObject() for r in result]
+ self.assertEquals(set(result), set([doc2, doc3]))
+
+ def test_bidi(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ doc1.setRelated([doc2.UID()])
+ doc2.setRelated([doc1.UID()])
+ result = [r.getTargetObject() for r in self.rc.getReferences(doc1)]
+ self.assertEquals(result, [doc2])
+ result = [r.getTargetObject() for r in self.rc.getReferences(doc2)]
+ self.assertEquals(result, [doc1])
+
+ doc1.setRel2([doc3.UID()])
+ result = self.rc.getReferences(doc1, ['related', 'rel2'])
+ result = [r.getTargetObject() for r in result]
+ self.assertEquals(set(result), set([doc2, doc3]))
+
+ def test_missing_uid_catalog_entry(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc1.setRelated([doc2.UID()])
+
+ result = [r.getTargetObject() for r in self.rc.getReferences(doc1)]
+ self.assertEquals(result, [doc2])
+
+ # Forcefully remove the target object from the uid catalog
+ uc = getToolByName(self.portal, 'uid_catalog')
+ uc.uncatalog_object(doc2._getURL())
+
+ references = self.rc.getReferences(doc1)
+ self.assertEquals(len(references), 1)
+ self.assertEquals(references[0].getTargetObject(), None)
+
+
+class TestGetBackReferences(ATRefSpeedupTestCase):
+
+ def test_none(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ self.assertEquals(self.rc.getBackReferences(doc1), [])
+ self.assertEquals(self.rc.getBackReferences(doc1, 'related'), [])
+ self.assertEquals(self.rc.getBackReferences(doc1, 'related', doc2),
+ [])
+
+ result = self.rc.getBackReferences(doc1, ['related', 'rel2'])
+ self.assertEquals(result, [])
+ result = self.rc.getBackReferences(doc1, ['related', 'rel2'], doc2)
+ self.assertEquals(result, [])
+
+ self.assertEquals(doc1.getBackReferences(), [])
+
+ def test_single(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ doc1.setRelated([doc2.UID()])
+
+ result = self.rc.getBackReferences(doc2)
+ self.assertEquals(result[0].getSourceObject(), doc1)
+ result = self.rc.getBackReferences(doc2, 'related')
+ self.assertEquals(result[0].getSourceObject(), doc1)
+ result = self.rc.getBackReferences(doc2, 'related', doc1)
+ self.assertEquals(result[0].getSourceObject(), doc1)
+ result = self.rc.getBackReferences(doc2, 'related', doc3)
+ self.assertEquals(result, [])
+
+ self.assertEquals(doc2.getBackReferences('related')[0], doc1)
+
+ doc1.setRel2([doc3.UID()])
+ result = self.rc.getBackReferences(doc2, ['related', 'rel2'])
+ self.assertEquals(result[0].getSourceObject(), doc1)
+ result = self.rc.getBackReferences(doc2, ['related', 'rel2'], doc1)
+ self.assertEquals(result[0].getSourceObject(), doc1)
+
+ def test_many(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ uids = [doc2.UID(), doc3.UID()]
+ doc1.setRelated(uids)
+
+ result = [r.getSourceObject() for r in self.rc.getBackReferences(doc2)]
+ self.assertEquals(set(result), set([doc1]))
+ result = [r.getSourceObject() for r in self.rc.getBackReferences(doc3)]
+ self.assertEquals(set(result), set([doc1]))
+
+ self.assertEquals(set(doc2.getBackReferences()), set([doc1]))
+
+ uids2 = [doc1.UID(), doc2.UID()]
+ doc3.setRel2(uids2)
+ result = self.rc.getBackReferences(doc2, ['related', 'rel2'])
+ result = [r.getSourceObject() for r in result]
+ self.assertEquals(set(result), set([doc1, doc3]))
+
+ def test_bidi(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc1.setRelated([doc2.UID()])
+ doc2.setRelated([doc1.UID()])
+
+ result = [r.getSourceObject() for r in self.rc.getBackReferences(doc1)]
+ self.assertEquals(result, [doc2])
+ result = [r.getSourceObject() for r in self.rc.getBackReferences(doc2)]
+ self.assertEquals(result, [doc1])
+
+ doc2.setRel2([doc1.UID()])
+ result = self.rc.getBackReferences(doc2, ['related', 'rel2'])
+ result = [r.getSourceObject() for r in result]
+ self.assertEquals(set(result), set([doc1]))
+
+ def test_missing_uid_catalog_entry(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc2.setRelated([doc1.UID()])
+
+ result = [r.getSourceObject() for r in self.rc.getBackReferences(doc1)]
+ self.assertEquals(result, [doc2])
+
+ # Forcefully remove the target object from the uid catalog
+ uc = getToolByName(self.portal, 'uid_catalog')
+ uc.uncatalog_object(doc2._getURL())
+
+ references = self.rc.getBackReferences(doc1)
+ self.assertEquals(len(references), 1)
+ self.assertEquals(references[0].getSourceObject(), None)
+
+
+class TestReferenceable(ATRefSpeedupTestCase):
+
+ def test_no_references(self):
+ doc1 = self.portal.doc1
+ self.assertEquals(doc1.getRelated(), [])
+ self.assertEquals(doc1.getRawRelated(), [])
+ self.assertEquals(doc1.getReferences(), [])
+ self.assertEquals(doc1.getRelationships(), [])
+
+ def test_single_reference(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc1.setRelated([doc2.UID()])
+ self.assertEquals(doc1.getRelated(), [doc2])
+ self.assertEquals(doc1.getRawRelated(), [doc2.UID()])
+ self.assertEquals(doc1.getReferences(), [doc2])
+ self.assertEquals(doc1.getRelationships(), ['related'])
+
+ def test_many_references(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc3 = self.portal.doc3
+ uids = [doc2.UID(), doc3.UID()]
+ doc1.setRelated(uids)
+ self.assertEquals(set(doc1.getRelated()), set([doc2, doc3]))
+ self.assertEquals(set(doc1.getRawRelated()), set(uids))
+ self.assertEquals(set(doc1.getReferences()), set([doc2, doc3]))
+ self.assertEquals(doc1.getRelationships(), ['related'])
+
+ def test_bidi_references(self):
+ doc1 = self.portal.doc1
+ doc2 = self.portal.doc2
+ doc1.setRelated([doc2.UID()])
+ doc2.setRelated([doc1.UID()])
+ self.assertEquals(doc1.getRelated(), [doc2])
+ self.assertEquals(doc1.getRawRelated(), [doc2.UID()])
+ self.assertEquals(doc1.getReferences(), [doc2])
+ self.assertEquals(doc1.getRelationships(), ['related'])
+ self.assertEquals(doc2.getRelated(), [doc1])
+ self.assertEquals(doc2.getRawRelated(), [doc1.UID()])
+ self.assertEquals(doc2.getReferences(), [doc1])
+ self.assertEquals(doc2.getRelationships(), ['related'])
+
+
+def test_suite():
+ from unittest import TestSuite, makeSuite
+ suite = TestSuite()
+ suite.addTest(makeSuite(TestGetReferences))
+ suite.addTest(makeSuite(TestGetBackReferences))
+ suite.addTest(makeSuite(TestReferenceable))
+ return suite
------------------------------------------------------------------------------
Learn how Oracle Real Application Clusters (RAC) One Node allows customers
to consolidate database storage, standardize their database environment, and,
should the need arise, upgrade to a full multi-node Oracle RAC database
without downtime or disruption
http://p.sf.net/sfu/oracle-sfdevnl