r47298 - Merge flatten-optimization-8300: Optimize template flattening.
mithrandi-TA+aISz0psMTMxyoc4vAAJOcrHinNvQL0E9HWUfgJXw@public.gmane.org Tue, 19 Apr 2016 03:14:27 -0600 (MDT)
| Newsgroups | gmane.comp.python.twisted.commits |
|---|---|
| Message-ID | <[email protected]> |
Author: mithrandi
Date: Tue Apr 19 03:14:20 2016
New Revision: 47298
Added:
trunk/twisted/web/topfiles/8300.misc
Modified:
trunk/twisted/web/_flatten.py
Log:
Merge flatten-optimization-8300: Optimize template flattening.
Author: mithrandi
Reviewer: glyph
Fixes: #8300
Optimize template flattening by passing the write callable down into the
flattener and calling it directly, instead of yielding strings (often
very small strings) back up to the driver.
This yields a significant speed increase on PyPy, and a moderate one on
CPython.
Modified: trunk/twisted/web/_flatten.py
==============================================================================
--- trunk/twisted/web/_flatten.py (original)
+++ trunk/twisted/web/_flatten.py Tue Apr 19 03:14:20 2016
@@ -51,7 +51,7 @@
"""
Escape some character or UTF-8 byte data for inclusion in the top level of
an attribute. L{attributeEscapingDoneOutside} actually passes the data
- through unchanged, because L{flattenWithAttributeEscaping} handles the
+ through unchanged, because L{writeWithAttributeEscaping} handles the
quoting of the text within attributes outside the generator returned by
L{_flattenElement}; this is used as the C{dataEscaper} argument to that
L{_flattenElement} call so that that generator does not redundantly escape
@@ -69,10 +69,10 @@
-def flattenWithAttributeEscaping(root):
+def writeWithAttributeEscaping(write):
"""
- Decorate the generator returned by L{_flattenElement} so that its output is
- properly quoted for inclusion within an XML attribute value.
+ Decorate a C{write} callable so that all output written is properly quoted
+ for inclusion within an XML attribute value.
If a L{Tag <twisted.web.template.Tag>} C{x} is flattened within the context
of the contents of another L{Tag <twisted.web.template.Tag>} C{y}, the
@@ -94,38 +94,22 @@
comments and CDATA, so if you were to serialize a L{comment
<twisted.web.template.Comment>} in an attribute you should get C{<y
attr="<-- comment -->" />}. Therefore in order to capture these
- meta-characters, the attribute generator from L{_flattenElement} context is
- wrapped with an L{flattenWithAttributeEscaping}.
-
- Because I{all} characters serialized in the context of an attribute are
- quoted before they are yielded by the generator returned by
- L{flattenWithAttributeEscaping}, on the "outside" of the L{_flattenElement}
- call, the L{_flattenElement} generator therefore no longer needs to quote
- text that appears directly within the attribute itself.
+ meta-characters, flattening is done with C{write} callable that is wrapped
+ with L{writeWithAttributeEscaping}.
The final case, and hopefully the much more common one as compared to
serializing L{Tag <twisted.web.template.Tag>} and arbitrary L{IRenderable}
objects within an attribute, is to serialize a simple string, and those
- should be passed through for L{flattenWithAttributeEscaping} to quote
+ should be passed through for L{writeWithAttributeEscaping} to quote
without applying a second, redundant level of quoting.
- @param root: A value that may be yielded by L{_flattenElement}; either an
- iterable yielding L{bytes} (or more iterables), or bytes itself.
- @type root: L{bytes} or C{iterable}
-
- @return: The same type as L{_flattenElement} returns, with all the bytes
- encoded for representation within an attribute.
- @rtype: the same type as the C{subFlatten} argument
- """
- if isinstance(root, bytes):
- root = escapeForContent(root)
- root = root.replace(b'"', b'"')
- yield root
- elif isinstance(root, Deferred):
- yield root.addCallback(flattenWithAttributeEscaping)
- else:
- for subroot in root:
- yield flattenWithAttributeEscaping(subroot)
+ @param write: A callable which will be invoked with the escaped L{bytes}.
+
+ @return: A callable that writes data with escaping.
+ """
+ def _write(data):
+ write(escapeForContent(data).replace(b'"', b'"'))
+ return _write
@@ -180,7 +164,8 @@
-def _flattenElement(request, root, slotData, renderFactory, dataEscaper):
+def _flattenElement(request, root, write, slotData, renderFactory,
+ dataEscaper):
"""
Make C{root} slightly more flat by yielding all its immediate contents as
strings, deferreds or generators that are recursive calls to itself.
@@ -193,6 +178,9 @@
L{GeneratorType}, L{Deferred}, or an object that implements
L{IRenderable}.
+ @param write: A callable which will be invoked with each L{bytes} produced
+ by flattening C{root}.
+
@param slotData: A C{list} of C{dict} mapping C{str} slot names to data
with which those slots will be replaced.
@@ -204,7 +192,7 @@
rendering context. This is really only one of two values:
L{attributeEscapingDoneOutside} or L{escapeForContent}, depending on
whether the rendering context is within an attribute or not. See the
- explanation in L{flattenWithAttributeEscaping}.
+ explanation in L{writeWithAttributeEscaping}.
@return: An iterator that eventually yields L{bytes} that should be written
to the output. However it may also yield other iterators or
@@ -217,22 +205,22 @@
of the same type.
"""
def keepGoing(newRoot, dataEscaper=dataEscaper,
- renderFactory=renderFactory):
- return _flattenElement(request, newRoot, slotData, renderFactory,
- dataEscaper)
+ renderFactory=renderFactory, write=write):
+ return _flattenElement(request, newRoot, write, slotData,
+ renderFactory, dataEscaper)
if isinstance(root, (bytes, unicode)):
- yield dataEscaper(root)
+ write(dataEscaper(root))
elif isinstance(root, slot):
slotValue = _getSlotValue(root.name, slotData, root.default)
yield keepGoing(slotValue)
elif isinstance(root, CDATA):
- yield b'<![CDATA['
- yield escapedCDATA(root.data)
- yield b']]>'
+ write(b'<![CDATA[')
+ write(escapedCDATA(root.data))
+ write(b']]>')
elif isinstance(root, Comment):
- yield b'<!--'
- yield escapedComment(root.data)
- yield b'-->'
+ write(b'<!--')
+ write(escapedComment(root.data))
+ write(b'-->')
elif isinstance(root, Tag):
slotData.append(root.slotData)
if root.render is not None:
@@ -249,23 +237,25 @@
yield keepGoing(root.children)
return
- yield b'<'
+ write(b'<')
if isinstance(root.tagName, unicode):
tagName = root.tagName.encode('ascii')
else:
tagName = root.tagName
- yield tagName
+ write(tagName)
for k, v in iteritems(root.attributes):
if isinstance(k, unicode):
k = k.encode('ascii')
- yield b' ' + k + b'="'
+ write(b' ' + k + b'="')
# Serialize the contents of the attribute, wrapping the results of
# that serialization so that _everything_ is quoted.
- attribute = keepGoing(v, attributeEscapingDoneOutside)
- yield flattenWithAttributeEscaping(attribute)
- yield b'"'
+ yield keepGoing(
+ v,
+ attributeEscapingDoneOutside,
+ write=writeWithAttributeEscaping(write))
+ write(b'"')
if root.children or nativeString(tagName) not in voidElements:
- yield b'>'
+ write(b'>')
# Regardless of whether we're in an attribute or not, switch back
# to the escapeForContent dataEscaper. The contents of a tag must
# be quoted no matter what; in the top-level document, just so
@@ -274,16 +264,16 @@
# parse the tag within the attribute, all the quoting is still
# correct.
yield keepGoing(root.children, escapeForContent)
- yield b'</' + tagName + b'>'
+ write(b'</' + tagName + b'>')
else:
- yield b' />'
+ write(b' />')
elif isinstance(root, (tuple, list, GeneratorType)):
for element in root:
yield keepGoing(element)
elif isinstance(root, CharRef):
escaped = '&#%d;' % (root.ordinal,)
- yield escaped.encode('ascii')
+ write(escaped.encode('ascii'))
elif isinstance(root, Deferred):
yield root.addCallback(lambda result: (result, keepGoing(result)))
elif IRenderable.providedBy(root):
@@ -294,7 +284,7 @@
-def _flattenTree(request, root):
+def _flattenTree(request, root, write):
"""
Make C{root} into an iterable of L{bytes} and L{Deferred} by doing a depth
first traversal of the tree.
@@ -307,12 +297,15 @@
L{list}, L{GeneratorType}, L{Deferred}, or something providing
L{IRenderable}.
+ @param write: A callable which will be invoked with each L{bytes} produced
+ by flattening C{root}.
+
@return: An iterator which yields objects of type L{bytes} and L{Deferred}.
A L{Deferred} is only yielded when one is encountered in the process of
flattening C{root}. The returned iterator must not be iterated again
until the L{Deferred} is called back.
"""
- stack = [_flattenElement(request, root, [], None, escapeForContent)]
+ stack = [_flattenElement(request, root, write, [], None, escapeForContent)]
while stack:
try:
frame = stack[-1].gi_frame
@@ -327,9 +320,7 @@
roots.append(frame.f_locals['root'])
raise FlattenerError(e, roots, extract_tb(exc_info()[2]))
else:
- if type(element) is bytes:
- yield element
- elif isinstance(element, Deferred):
+ if isinstance(element, Deferred):
def cbx(originalAndToFlatten):
original, toFlatten = originalAndToFlatten
stack.append(toFlatten)
@@ -365,14 +356,10 @@
except:
result.errback()
else:
- if type(element) is bytes:
- write(element)
- continue
- else:
- def cby(original):
- _writeFlattenedData(state, write, result)
- return original
- element.addCallbacks(cby, result.errback)
+ def cby(original):
+ _writeFlattenedData(state, write, result)
+ return original
+ element.addCallbacks(cby, result.errback)
break
@@ -401,7 +388,7 @@
unexpected exception occurs.
"""
result = Deferred()
- state = _flattenTree(request, root)
+ state = _flattenTree(request, root, write)
_writeFlattenedData(state, write, result)
return result