Fix wrong lines information in bytecode (tracebacks, pdb...) generated by compiler module.

component/python-2.7/buildout.cfg

@@ -41,6 +41,7 @@ patch-options = -p1
 patches =
   ${:_profile_base_location_}/tls_sni.patch#c95af105e6e96aaa58a50137595872a0
   ${:_profile_base_location_}/tls_sni_httplib.patch#5c9d00d23b85169df792a936a056cbcc
+  ${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
 url =
   http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
 configure-options =

component/python-2.7/fix_compiler_module_issue_20613.patch

+From af98230024f639d4c719b55d0d330912c3ea7b97 Mon Sep 17 00:00:00 2001
+From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
+Date: Thu, 11 Dec 2014 23:48:22 +0900
+Subject: [PATCH] Revert https://hg.python.org/cpython/raw-rev/42faa8054c3d.
+ This fixes http://bugs.python.org/issue20613.
+
+---
+ Lib/compiler/pyassem.py   | 257 ++++++++++++++++++++++++++++------------------
+ Lib/compiler/pycodegen.py |   2 +-
+ 2 files changed, 159 insertions(+), 100 deletions(-)
+
+diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
+index f52f7d0..4299a17 100644
+--- a/Lib/compiler/pyassem.py
++++ b/Lib/compiler/pyassem.py
+@@ -21,7 +21,6 @@ class FlowGraph:
+             if self.current:
+                 print "end", repr(self.current)
+                 print "    next", self.current.next
+-                print "    prev", self.current.prev
+                 print "   ", self.current.get_children()
+             print repr(block)
+         self.current = block
+@@ -41,12 +40,13 @@ class FlowGraph:
+         if block is None:
+             block = self.newBlock()
+ 
+-        # Note: If the current block ends with an unconditional control
+-        # transfer, then it is techically incorrect to add an implicit
+-        # transfer to the block graph. Doing so results in code generation
+-        # for unreachable blocks.  That doesn't appear to be very common
+-        # with Python code and since the built-in compiler doesn't optimize
+-        # it out we don't either.
++        # Note: If the current block ends with an unconditional
++        # control transfer, then it is incorrect to add an implicit
++        # transfer to the block graph.  The current code requires
++        # these edges to get the blocks emitted in the right order,
++        # however. :-(  If a client needs to remove these edges, call
++        # pruneEdges().
++
+         self.current.addNext(block)
+         self.startBlock(block)
+ 
+@@ -69,6 +69,8 @@ class FlowGraph:
+     def emit(self, *inst):
+         if self._debug:
+             print "\t", inst
++        if inst[0] in ['RETURN_VALUE', 'YIELD_VALUE']:
++            self.current.addOutEdge(self.exit)
+         if len(inst) == 2 and isinstance(inst[1], Block):
+             self.current.addOutEdge(inst[1])
+         self.current.emit(inst)
+@@ -78,9 +80,118 @@ class FlowGraph:
+ 
+         i.e. each node appears before all of its successors
+         """
+-        order = order_blocks(self.entry, self.exit)
++        # XXX make sure every node that doesn't have an explicit next
++        # is set so that next points to exit
++        for b in self.blocks.elements():
++            if b is self.exit:
++                continue
++            if not b.next:
++                b.addNext(self.exit)
++        order = dfs_postorder(self.entry, {})
++        order.reverse()
++        self.fixupOrder(order, self.exit)
++        # hack alert
++        if not self.exit in order:
++            order.append(self.exit)
++
+         return order
+ 
++    def fixupOrder(self, blocks, default_next):
++        """Fixup bad order introduced by DFS."""
++
++        # XXX This is a total mess.  There must be a better way to get
++        # the code blocks in the right order.
++
++        self.fixupOrderHonorNext(blocks, default_next)
++        self.fixupOrderForward(blocks, default_next)
++
++    def fixupOrderHonorNext(self, blocks, default_next):
++        """Fix one problem with DFS.
++
++        The DFS uses child block, but doesn't know about the special
++        "next" block.  As a result, the DFS can order blocks so that a
++        block isn't next to the right block for implicit control
++        transfers.
++        """
++        index = {}
++        for i in range(len(blocks)):
++            index[blocks[i]] = i
++
++        for i in range(0, len(blocks) - 1):
++            b = blocks[i]
++            n = blocks[i + 1]
++            if not b.next or b.next[0] == default_next or b.next[0] == n:
++                continue
++            # The blocks are in the wrong order.  Find the chain of
++            # blocks to insert where they belong.
++            cur = b
++            chain = []
++            elt = cur
++            while elt.next and elt.next[0] != default_next:
++                chain.append(elt.next[0])
++                elt = elt.next[0]
++            # Now remove the blocks in the chain from the current
++            # block list, so that they can be re-inserted.
++            l = []
++            for b in chain:
++                assert index[b] > i
++                l.append((index[b], b))
++            l.sort()
++            l.reverse()
++            for j, b in l:
++                del blocks[index[b]]
++            # Insert the chain in the proper location
++            blocks[i:i + 1] = [cur] + chain
++            # Finally, re-compute the block indexes
++            for i in range(len(blocks)):
++                index[blocks[i]] = i
++
++    def fixupOrderForward(self, blocks, default_next):
++        """Make sure all JUMP_FORWARDs jump forward"""
++        index = {}
++        chains = []
++        cur = []
++        for b in blocks:
++            index[b] = len(chains)
++            cur.append(b)
++            if b.next and b.next[0] == default_next:
++                chains.append(cur)
++                cur = []
++        chains.append(cur)
++
++        while 1:
++            constraints = []
++
++            for i in range(len(chains)):
++                l = chains[i]
++                for b in l:
++                    for c in b.get_children():
++                        if index[c] < i:
++                            forward_p = 0
++                            for inst in b.insts:
++                                if inst[0] == 'JUMP_FORWARD':
++                                    if inst[1] == c:
++                                        forward_p = 1
++                            if not forward_p:
++                                continue
++                            constraints.append((index[c], i))
++
++            if not constraints:
++                break
++
++            # XXX just do one for now
++            # do swaps to get things in the right order
++            goes_before, a_chain = constraints[0]
++            assert a_chain > goes_before
++            c = chains[a_chain]
++            chains.remove(c)
++            chains.insert(goes_before, c)
++
++        del blocks[:]
++        for c in chains:
++            for b in c:
++                blocks.append(b)
++
+     def getBlocks(self):
+         return self.blocks.elements()
+ 
+@@ -94,84 +205,27 @@ class FlowGraph:
+             l.extend(b.getContainedGraphs())
+         return l
+ 
+-
+-def order_blocks(start_block, exit_block):
+-    """Order blocks so that they are emitted in the right order"""
+-    # Rules:
+-    # - when a block has a next block, the next block must be emitted just after
+-    # - when a block has followers (relative jumps), it must be emitted before
+-    #   them
+-    # - all reachable blocks must be emitted
++def dfs_postorder(b, seen):
++    """Depth-first search of tree rooted at b, return in postorder"""
+     order = []
+-
+-    # Find all the blocks to be emitted.
+-    remaining = set()
+-    todo = [start_block]
+-    while todo:
+-        b = todo.pop()
+-        if b in remaining:
+-            continue
+-        remaining.add(b)
+-        for c in b.get_children():
+-            if c not in remaining:
+-                todo.append(c)
+-
+-    # A block is dominated by another block if that block must be emitted
+-    # before it.
+-    dominators = {}
+-    for b in remaining:
+-        if __debug__ and b.next:
+-            assert b is b.next[0].prev[0], (b, b.next)
+-        # Make sure every block appears in dominators, even if no
+-        # other block must precede it.
+-        dominators.setdefault(b, set())
+-        # preceding blocks dominate following blocks
+-        for c in b.get_followers():
+-            while 1:
+-                dominators.setdefault(c, set()).add(b)
+-                # Any block that has a next pointer leading to c is also
+-                # dominated because the whole chain will be emitted at once.
+-                # Walk backwards and add them all.
+-                if c.prev and c.prev[0] is not b:
+-                    c = c.prev[0]
+-                else:
+-                    break
+-
+-    def find_next():
+-        # Find a block that can be emitted next.
+-        for b in remaining:
+-            for c in dominators[b]:
+-                if c in remaining:
+-                    break # can't emit yet, dominated by a remaining block
+-            else:
+-                return b
+-        assert 0, 'circular dependency, cannot find next block'
+-
+-    b = start_block
+-    while 1:
+-        order.append(b)
+-        remaining.discard(b)
+-        if b.next:
+-            b = b.next[0]
++    seen[b] = b
++    for c in b.get_children():
++        if c in seen:
+             continue
+-        elif b is not exit_block and not b.has_unconditional_transfer():
+-            order.append(exit_block)
+-        if not remaining:
+-            break
+-        b = find_next()
++        order = order + dfs_postorder(c, seen)
++    order.append(b)
+     return order
+ 
+-
+ class Block:
+     _count = 0
+ 
+     def __init__(self, label=''):
+         self.insts = []
+-        self.outEdges = set()
++        self.inEdges = misc.Set()
++        self.outEdges = misc.Set()
+         self.label = label
+         self.bid = Block._count
+         self.next = []
+-        self.prev = []
+         Block._count = Block._count + 1
+ 
+     def __repr__(self):
+@@ -187,46 +241,51 @@ class Block:
+ 
+     def emit(self, inst):
+         op = inst[0]
++        if op[:4] == 'JUMP':
++            self.outEdges.add(inst[1])
+         self.insts.append(inst)
+ 
+     def getInstructions(self):
+         return self.insts
+ 
++    def addInEdge(self, block):
++        self.inEdges.add(block)
++
+     def addOutEdge(self, block):
+         self.outEdges.add(block)
+ 
+     def addNext(self, block):
+         self.next.append(block)
+         assert len(self.next) == 1, map(str, self.next)
+-        block.prev.append(self)
+-        assert len(block.prev) == 1, map(str, block.prev)
+ 
+-    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
+-                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP',
+-                        )
++    _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS', 'YIELD_VALUE',
++                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
+ 
+-    def has_unconditional_transfer(self):
+-        """Returns True if there is an unconditional transfer to an other block
+-        at the end of this block. This means there is no risk for the bytecode
+-        executer to go past this block's bytecode."""
++    def pruneNext(self):
++        """Remove bogus edge for unconditional transfers
++
++        Each block has a next edge that accounts for implicit control
++        transfers, e.g. from a JUMP_IF_FALSE to the block that will be
++        executed if the test is true.
++
++        These edges must remain for the current assembler code to
++        work. If they are removed, the dfs_postorder gets things in
++        weird orders.  However, they shouldn't be there for other
++        purposes, e.g. conversion to SSA form.  This method will
++        remove the next edge when it follows an unconditional control
++        transfer.
++        """
+         try:
+             op, arg = self.insts[-1]
+         except (IndexError, ValueError):
+             return
+-        return op in self._uncond_transfer
++        if op in self._uncond_transfer:
++            self.next = []
+ 
+     def get_children(self):
+-        return list(self.outEdges) + self.next
+-
+-    def get_followers(self):
+-        """Get the whole list of followers, including the next block."""
+-        followers = set(self.next)
+-        # Blocks that must be emitted *after* this one, because of
+-        # bytecode offsets (e.g. relative jumps) pointing to them.
+-        for inst in self.insts:
+-            if inst[0] in PyFlowGraph.hasjrel:
+-                followers.add(inst[1])
+-        return followers
++        if self.next and self.next[0] in self.outEdges:
++            self.outEdges.remove(self.next[0])
++        return self.outEdges.elements() + self.next
+ 
+     def getContainedGraphs(self):
+         """Return all graphs contained within this block.
+@@ -387,18 +446,18 @@ class PyFlowGraph(FlowGraph):
+             elif inst[0] != "SET_LINENO":
+                 pc = pc + 3
+             opname = inst[0]
+-            if opname in self.hasjrel:
++            if self.hasjrel.has_elt(opname):
+                 oparg = inst[1]
+                 offset = begin[oparg] - pc
+                 insts[i] = opname, offset
+-            elif opname in self.hasjabs:
++            elif self.hasjabs.has_elt(opname):
+                 insts[i] = opname, begin[inst[1]]
+         self.stage = FLAT
+ 
+-    hasjrel = set()
++    hasjrel = misc.Set()
+     for i in dis.hasjrel:
+         hasjrel.add(dis.opname[i])
+-    hasjabs = set()
++    hasjabs = misc.Set()
+     for i in dis.hasjabs:
+         hasjabs.add(dis.opname[i])
+ 
+diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
+index 6515945..3f641ba 100644
+--- a/Lib/compiler/pycodegen.py
++++ b/Lib/compiler/pycodegen.py
+@@ -706,7 +706,7 @@ class CodeGenerator:
+             self.startBlock(anchor)
+             self.emit('POP_BLOCK')
+             self.setups.pop()
+-            self.nextBlock(end)
++            self.startBlock(end)
+ 
+         self.emit('LOAD_CONST', None)
+ 
+-- 
+2.2.0.rc0.207.ga3a616c
+