Re: Wrong line numbers in tracebacks
Patrik Simons <[email protected]> Thu, 1 Aug 2013 15:56:38 +0300
| Newsgroups | gmane.comp.web.quixote.user |
|---|---|
| Message-ID | <[email protected]> |
This is a multi-part message in MIME format. --Multipart=_Thu__1_Aug_2013_15_56_38_+0300_DUS.VO_F6I198Dlt Content-Type: text/plain; charset=US-ASCII Content-Transfer-Encoding: 7bit On Mon, 15 Jul 2013 14:07:29 +0300 Patrik Simons <[email protected]> wrote: > I've had some problems with wrong line numbers in tracebacks from ptl > files. Setting lineno for the wrapped function code fixes it: > It turns out that the fix is not enough, compiler.pyassem loses line number information when it reorders blocks. As the compiler package is no more in python 3, I don't think it is worth fixing. I've made a version of ptl_compile that uses the ast module from python 2.6 and up instead. The file is attached, patch below. It is only lightly tested. What do you think? --- ptl_compile.py.orig 2013-08-01 10:23:12.421634339 +0300 +++ ptl_compile.py 2013-08-01 15:30:50.558713374 +0300 @@ -8,34 +8,24 @@ the compiler package. """ +import ast import sys import os import stat -import symbol -import token import re import imp -import stat import marshal import struct -assert sys.hexversion >= 0x20300b1, 'PTL requires Python 2.3 or newer' - -from compiler import pycodegen, transformer -from compiler import ast -from compiler.consts import OP_ASSIGN -from compiler import misc, syntax - HTML_TEMPLATE_PREFIX = "_q_html_template_" PLAIN_TEMPLATE_PREFIX = "_q_plain_template_" -class TemplateTransformer(transformer.Transformer): - +class TemplateTransformer(ast.NodeTransformer): def __init__(self, *args, **kwargs): - transformer.Transformer.__init__(self, *args, **kwargs) + ast.NodeTransformer.__init__(self, *args, **kwargs) # __template_type is a stack whose values are # "html", "plain", or None - self.__template_type = [] + self.__template_type = [None] def _get_template_type(self): """Return the type of the function being compiled ( @@ -46,129 +36,93 @@ else: return None - def file_input(self, nodelist): - doc = None # self.get_docstring(nodelist, symbol.file_input) - if sys.hexversion >= 0x02050000: - html_imp = ast.From( - 'quixote.html', - [('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')], - 0) - vars_imp = ast.From("__builtin__", [("vars", "_q_vars")], 0) - else: - html_imp = ast.From( - 'quixote.html', - [('TemplateIO', '_q_TemplateIO'), ('htmltext', '_q_htmltext')]) - vars_imp = ast.From("__builtin__", [("vars", "_q_vars")]) - - ptl_imports = [ vars_imp, html_imp ] - stmts = [] - for node in nodelist: - if node[0] != token.ENDMARKER and node[0] != token.NEWLINE: - self.com_append_stmt(stmts, node) + def visit_Module(self, node): + html_imp = ast.ImportFrom(module='quixote.html', + names=[ast.alias(name='TemplateIO', + asname='_q_TemplateIO'), + ast.alias(name='htmltext', + asname='_q_htmltext')], + level=0) + ast.fix_missing_locations(html_imp) + vars_imp = ast.ImportFrom(module='__builtin__', + names=[ast.alias(name='vars', + asname='_q_vars')], level=0) + ast.fix_missing_locations(vars_imp) + ptl_imports = [vars_imp, html_imp] # count __future__ statements i = 0 - for stmt in stmts: - if isinstance(stmt, ast.From) and stmt.modname == '__future__': + for stmt in node.body: + if isinstance(stmt, ast.ImportFrom) and stmt.module == '__future__': i += 1 else: break - stmts[i:i] = ptl_imports - return ast.Module(doc, ast.Stmt(stmts)) - - def funcdef(self, nodelist): - if len(nodelist) == 6: - assert nodelist[0][0] == symbol.decorators - decorators = self.decorators(nodelist[0][1:]) - else: - assert len(nodelist) == 5 - decorators = None - - lineno = nodelist[-4][2] - name = nodelist[-4][1] - args = nodelist[-3][2] + node.body[i:i] = ptl_imports + return self.generic_visit(node) + def visit_FunctionDef(self, node): + name = node.name if not re.match('_q_(html|plain)_(dollar_)?template_', name): - # just a normal function, let base class handle it + # just a normal function self.__template_type.append(None) - n = transformer.Transformer.funcdef(self, nodelist) + node = self.generic_visit(node) else: if name.startswith(PLAIN_TEMPLATE_PREFIX): - name = name[len(PLAIN_TEMPLATE_PREFIX):] + node.name = name[len(PLAIN_TEMPLATE_PREFIX):] template_type = "plain" elif name.startswith(HTML_TEMPLATE_PREFIX): - name = name[len(HTML_TEMPLATE_PREFIX):] + node.name = name[len(HTML_TEMPLATE_PREFIX):] template_type = "html" else: raise RuntimeError, 'unknown prefix on %s' % name self.__template_type.append(template_type) - - if args[0] == symbol.varargslist: - names, defaults, flags = self.com_arglist(args[1:]) - else: - names = defaults = () - flags = 0 - doc = None # self.get_docstring(nodelist[-1]) - - # code for function - code = self.com_node(nodelist[-1]) + node = self.generic_visit(node) # _q_output = _q_TemplateIO() - klass = ast.Name('_q_TemplateIO') - args = [ast.Const(template_type == "html")] - instance = ast.CallFunc(klass, args) - assign_name = ast.AssName('_q_output', OP_ASSIGN) - assign = ast.Assign([assign_name], instance) + klass = ast.Name(id='_q_TemplateIO', ctx=ast.Load()) + arg = ast.Num(n=template_type == "html") + instance = ast.Call(func=klass, args=[arg], keywords=[], + starargs=None, kwargs=None) + assign_name = ast.Name(id='_q_output', ctx=ast.Store()) + assign = ast.Assign(targets=[assign_name], value=instance) + ast.copy_location(assign, node) + ast.fix_missing_locations(assign) + node.body.insert(0, assign) # return _q_output.getvalue() - func = ast.Getattr(ast.Name('_q_output'), "getvalue") - ret = ast.Return(ast.CallFunc(func, [])) - - # wrap original function code - code = ast.Stmt([assign, code, ret]) - code.lineno = lineno - - if sys.hexversion >= 0x20400a2: - n = ast.Function(decorators, name, names, defaults, flags, doc, - code) - else: - n = ast.Function(name, names, defaults, flags, doc, code) - n.lineno = lineno + n = ast.Name(id='_q_output', ctx=ast.Load()) + n = ast.Attribute(value=n, attr='getvalue', ctx=ast.Load()) + n = ast.Call(func=n, args=[], keywords=[], starargs=None, + kwargs=None) + ret = ast.Return(value=n) + ast.copy_location(ret, node.body[-1]) + ast.fix_missing_locations(ret) + node.body.append(ret) self.__template_type.pop() - return n + return node - def expr_stmt(self, nodelist): - if self._get_template_type() is None: - return transformer.Transformer.expr_stmt(self, nodelist) - - # Instead of discarding objects on the stack, call - # "_q_output += obj". - exprNode = self.com_node(nodelist[-1]) - if len(nodelist) == 1: - lval = ast.Name('_q_output') - n = ast.AugAssign(lval, '+=', exprNode) - if hasattr(exprNode, 'lineno'): - n.lineno = exprNode.lineno - elif nodelist[1][0] == token.EQUAL: - nodes = [ ] - for i in range(0, len(nodelist) - 2, 2): - nodes.append(self.com_assign(nodelist[i], OP_ASSIGN)) - n = ast.Assign(nodes, exprNode) - n.lineno = nodelist[1][2] + def visit_Expr(self, node): + if "html" == self._get_template_type(): + node = self.generic_visit(node) + # Instead of discarding objects on the stack, call + # "_q_output += obj". + lval = ast.Name(id='_q_output', ctx=ast.Store()) + ast.copy_location(lval, node) + aug = ast.AugAssign(target=lval, op=ast.Add(), value=node.value) + return ast.copy_location(aug, node) else: - lval = self.com_augassign(nodelist[0]) - op = self.com_augassign_op(nodelist[1]) - n = ast.AugAssign(lval, op[1], exprNode) - n.lineno = op[2] - return n + return node - def atom_string(self, nodelist): - const_node = transformer.Transformer.atom_string(self, nodelist) + def visit_Str(self, node): if "html" == self._get_template_type(): - return ast.CallFunc(ast.Name('_q_htmltext'), [const_node]) + n = ast.Name(id='_q_htmltext', ctx=ast.Load()) + ast.copy_location(n, node) + n = ast.Call(func=n, args=[node], keywords=[], starargs=None, + kwargs=None) + return ast.copy_location(n, node) else: - return const_node + return node _template_re = re.compile( r"^(?P<indent>[ \t]*) def (?:[ \t]+)" @@ -200,31 +154,26 @@ def parse(buf, filename='<string>'): buf = translate_tokens(buf) try: - return TemplateTransformer().parsesuite(buf) + node = ast.parse(buf, filename) except SyntaxError, e: # set the filename attribute raise SyntaxError(str(e), (filename, e.lineno, e.offset, e.text)) + t = TemplateTransformer() + return t.visit(node) PTL_EXT = ".ptl" -class Template(pycodegen.Module): - - def _get_tree(self): - tree = parse(self.source, self.filename) - misc.set_filename(self.filename, tree) - syntax.check(tree) - return tree - - def dump(self, fp): - mtime = os.stat(self.filename)[stat.ST_MTIME] - fp.write('\0\0\0\0') - fp.write(struct.pack('<I', mtime)) - marshal.dump(self.code, fp) - fp.flush() - fp.seek(0) - fp.write(imp.get_magic()) +def dump(code, filename, fp): + mtime = os.stat(filename)[stat.ST_MTIME] + fp.write('\0\0\0\0') + fp.write(struct.pack('<I', mtime)) + marshal.dump(code, fp) + fp.flush() + fp.seek(0) + fp.write(imp.get_magic()) +_compile = compile def compile_template(input, filename, output=None): """(input, filename, output=None) -> code @@ -233,12 +182,11 @@ If output is not None then the code is written to output. The code object is returned. """ - buf = input.read() - template = Template(buf, filename) - template.compile() + node = parse(input.read(), filename) + code = _compile(node, filename, 'exec') if output is not None: - template.dump(output) - return template.code + dump(code, filename, output) + return code def compile(inputname, outputname): """(inputname, outputname) --Multipart=_Thu__1_Aug_2013_15_56_38_+0300_DUS.VO_F6I198Dlt Content-Type: text/x-python; name="ptl_import.py" Content-Disposition: attachment; filename="ptl_import.py" Content-Transfer-Encoding: 7bit """Import hooks; when installed, these hooks allow importing .ptl files as if they were Python modules. Note: there's some unpleasant incompatibility between ZODB's import trickery and the import hooks here. Bottom line: if you're using ZODB, import it *before* installing the PTL import hooks. """ import sys import os.path import imp, ihooks, new import struct import marshal import __builtin__ # Check for a deficient ihooks module. Python 2.6 was released without # ihooks.py being updated to support relative imports. Any library that uses # relative imports will cause the import hook to fail. Use our local copy of # ihooks module which does have support for relative imports. if sys.hexversion >= 0x20600b0: _m = ihooks.ModuleImporter.import_module if _m.im_func.func_code.co_argcount == 5: import ihooks_local as ihooks from quixote.ptl.ptl_compile import compile_template, PTL_EXT assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later" def _exec_module_code(code, name, filename): if name in sys.modules: mod = sys.modules[name] # necessary for reload() else: mod = new.module(name) sys.modules[name] = mod mod.__name__ = name mod.__file__ = filename exec code in mod.__dict__ return mod def _timestamp(filename): try: s = os.stat(filename) except OSError: return None return int(s.st_mtime) & 0xffffffff def _load_pyc(name, filename, pyc_filename): try: fp = open(pyc_filename, "rb") except IOError: return None if fp.read(4) == imp.get_magic(): mtime = struct.unpack('<I', fp.read(4))[0] ptl_mtime = _timestamp(filename) if ptl_mtime is not None and mtime == ptl_mtime: code = marshal.load(fp) return _exec_module_code(code, name, filename) return None def _load_ptl(name, filename, file=None): if not file: try: file = open(filename, "rb") except IOError: return None path, ext = os.path.splitext(filename) pyc_filename = path + ".pyc" module = _load_pyc(name, filename, pyc_filename) if module is not None: return module try: output = open(pyc_filename, "wb") except IOError: output = None try: code = compile_template(file, filename, output) except: if output: output.close() os.unlink(pyc_filename) raise else: if output: output.close() return _exec_module_code(code, name, filename) # Constant used to signal a PTL files PTL_FILE = object() class PTLHooks(ihooks.Hooks): def get_suffixes(self): # add our suffixes return [(PTL_EXT, 'r', PTL_FILE)] + imp.get_suffixes() class PTLLoader(ihooks.ModuleLoader): def load_module(self, name, stuff): file, filename, info = stuff (suff, mode, type) = info # If it's a PTL file, load it specially. if type is PTL_FILE: return _load_ptl(name, filename, file) else: # Otherwise, use the default handler for loading return ihooks.ModuleLoader.load_module(self, name, stuff) if sys.hexversion <= 0x20600b0: try: import cimport except ImportError: cimport = None else: # cimport module doesn't handle relative imports cimport = None class cModuleImporter(ihooks.ModuleImporter): def __init__(self, loader=None): self.loader = loader or ihooks.ModuleLoader() cimport.set_loader(self.find_import_module) def find_import_module(self, fullname, subname, path): stuff = self.loader.find_module(subname, path) if not stuff: return None return self.loader.load_module(fullname, stuff) def install(self): self.save_import_module = __builtin__.__import__ self.save_reload = __builtin__.reload if not hasattr(__builtin__, 'unload'): __builtin__.unload = None self.save_unload = __builtin__.unload __builtin__.__import__ = cimport.import_module __builtin__.reload = cimport.reload_module __builtin__.unload = self.unload _installed = False def install(): global _installed if not _installed: hooks = PTLHooks() loader = PTLLoader(hooks) if cimport is not None: importer = cModuleImporter(loader) else: importer = ihooks.ModuleImporter(loader) ihooks.install(importer) _installed = True if __name__ == '__main__': install() --Multipart=_Thu__1_Aug_2013_15_56_38_+0300_DUS.VO_F6I198Dlt Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline _______________________________________________ Quixote-users mailing list [email protected] http://lists.quixote.ca/cgi-bin/mailman/listinfo/quixote-users --Multipart=_Thu__1_Aug_2013_15_56_38_+0300_DUS.VO_F6I198Dlt--