Synced up with Python's release21-maint branch, with a bugfix for FOR_LOOP

stack size computation and relative rather than absolute imports.

lib/python/RestrictedPython/compiler/ast.py

@@ -40,8 +40,6 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
-    def getChildNodes(self):
-        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):
@@ -190,7 +188,7 @@ class Sub(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Sub((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Sub(%s, %s)" % (repr(self.left), repr(self.right))
 
 class ListCompIf(Node):
     nodes["listcompif"] = "ListCompIf"
@@ -209,7 +207,7 @@ class Div(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Div((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Div(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Discard(Node):
     nodes["discard"] = "Discard"
@@ -237,7 +235,7 @@ class RightShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "RightShift((%s, %s))" % (repr(self.left), repr(self.right))
+        return "RightShift(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Continue(Node):
     nodes["continue"] = "Continue"
@@ -277,7 +275,7 @@ class LeftShift(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right))
+        return "LeftShift(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Mul(Node):
     nodes["mul"] = "Mul"
@@ -287,16 +285,7 @@ class Mul(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mul((%s, %s))" % (repr(self.left), repr(self.right))
-
-class Yield(Node):
-    nodes["yield"] = "Yield"
-    def __init__(self, value):
-        self.value = value
-    def _getChildren(self):
-        return self.value,
-    def __repr__(self):
-        return "Yield(%s)" % (repr(self.value),)
+        return "Mul(%s, %s)" % (repr(self.left), repr(self.right))
 
 class List(Node):
     nodes["list"] = "List"
@@ -365,7 +354,7 @@ class Mod(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Mod((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Mod(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Class(Node):
     nodes["class"] = "Class"
@@ -463,16 +452,16 @@ class Power(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Power((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Power(%s, %s)" % (repr(self.left), repr(self.right))
 
-class Ellipsis(Node):
-    nodes["ellipsis"] = "Ellipsis"
-    def __init__(self, ):
-        pass
+class Import(Node):
+    nodes["import"] = "Import"
+    def __init__(self, names):
+        self.names = names
     def _getChildren(self):
-        return ()
+        return self.names,
     def __repr__(self):
-        return "Ellipsis()"
+        return "Import(%s)" % (repr(self.names),)
 
 class Return(Node):
     nodes["return"] = "Return"
@@ -491,7 +480,7 @@ class Add(Node):
     def _getChildren(self):
         return self.left, self.right
     def __repr__(self):
-        return "Add((%s, %s))" % (repr(self.left), repr(self.right))
+        return "Add(%s, %s)" % (repr(self.left), repr(self.right))
 
 class Function(Node):
     nodes["function"] = "Function"
@@ -536,14 +525,14 @@ class Subscript(Node):
     def __repr__(self):
         return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs))
 
-class Import(Node):
-    nodes["import"] = "Import"
-    def __init__(self, names):
-        self.names = names
+class Ellipsis(Node):
+    nodes["ellipsis"] = "Ellipsis"
+    def __init__(self, ):
+        pass
     def _getChildren(self):
-        return self.names,
+        return ()
     def __repr__(self):
-        return "Import(%s)" % (repr(self.names),)
+        return "Ellipsis()"
 
 class Print(Node):
     nodes["print"] = "Print"

lib/python/RestrictedPython/compiler/ast.txt

+Module: doc, node
+Stmt: nodes
+Function: name, argnames, defaults, flags, doc, code
+Lambda: argnames, defaults, flags, code
+Class: name, bases, doc, code
+Pass: 
+Break: 
+Continue: 
+For: assign, list, body, else_
+While: test, body, else_
+If: tests, else_
+Exec: expr, locals, globals
+From: modname, names
+Import: names
+Raise: expr1, expr2, expr3
+TryFinally: body, final
+TryExcept: body, handlers, else_
+Return: value
+Const: value
+Print: nodes, dest
+Printnl: nodes, dest
+Discard: expr
+AugAssign: node, op, expr
+Assign: nodes, expr
+AssTuple: nodes
+AssList: nodes
+AssName: name, flags
+AssAttr: expr, attrname, flags
+ListComp: expr, quals
+ListCompFor: assign, list, ifs
+ListCompIf: test
+List: nodes
+Dict: items
+Not: expr
+Compare: expr, ops
+Name: name
+Global: names
+Backquote: expr
+Getattr: expr, attrname
+CallFunc: node, args, star_args = None, dstar_args = None
+Keyword: name, expr
+Subscript: expr, flags, subs
+Ellipsis: 
+Sliceobj: nodes
+Slice: expr, flags, lower, upper
+Assert: test, fail
+Tuple: nodes
+Or: nodes
+And: nodes
+Bitor: nodes
+Bitxor: nodes
+Bitand: nodes
+LeftShift: (left, right)
+RightShift: (left, right)
+Add: (left, right)
+Sub: (left, right)
+Mul: (left, right)
+Div: (left, right)
+Mod: (left, right)
+Power: (left, right)
+UnaryAdd: expr
+UnarySub: expr
+Invert: expr
+
+init(Function):
+    self.varargs = self.kwargs = None
+    if flags & CO_VARARGS:
+        self.varargs = 1
+    if flags & CO_VARKEYWORDS:
+        self.kwargs = 1
+
+init(Lambda):
+    self.varargs = self.kwargs = None
+    if flags & CO_VARARGS:
+        self.varargs = 1
+    if flags & CO_VARKEYWORDS:
+        self.kwargs = 1

lib/python/RestrictedPython/compiler/astgen.py

@@ -81,8 +81,6 @@ class NodeInfo:
         print >> buf, "    def __repr__(self):"
         if self.argnames:
             fmt = COMMA.join(["%s"] * self.nargs)
-            if '(' in self.args:
-                fmt = '(%s)' % fmt
             vals = ["repr(self.%s)" % name for name in self.argnames]
             vals = COMMA.join(vals)
             if self.nargs == 1:
@@ -173,8 +171,6 @@ class Node:
         return flatten(self._getChildren())
     def asList(self):
         return tuple(asList(self.getChildren()))
-    def getChildNodes(self):
-        return [n for n in self.getChildren() if isinstance(n, Node)]
 
 class EmptyNode(Node):
     def __init__(self):

lib/python/RestrictedPython/compiler/consts.py

-# code flags
-CO_VARARGS = 1
-CO_VARKEYWORDS = 2
-
 # operation flags
 OP_ASSIGN = 'OP_ASSIGN'
 OP_DELETE = 'OP_DELETE'
@@ -12,3 +8,9 @@ SC_GLOBAL = 2
 SC_FREE = 3
 SC_CELL = 4
 SC_UNKNOWN = 5
+
+CO_OPTIMIZED = 0x0001
+CO_NEWLOCALS = 0x0002
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+CO_NESTED = 0x0010

lib/python/RestrictedPython/compiler/future.py

@@ -16,14 +16,19 @@ def is_future(stmt):
 
 class FutureParser:
 
-    features = ("nested_scopes", "generators", "division")
+    features = ("nested_scopes",)
     
     def __init__(self):
         self.found = {} # set
 
     def visitModule(self, node):
+        if node.doc is None:
+            off = 0
+        else:
+            off = 1
+
         stmt = node.node
-        for s in stmt.nodes:
+        for s in stmt.nodes[off:]:
             if not self.check_stmt(s):
                 break
 
@@ -62,7 +67,7 @@ def find_futures(node):
 
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
 
     for file in sys.argv[1:]:
         print file

lib/python/RestrictedPython/compiler/misc.py

@@ -39,3 +39,29 @@ class Stack:
         self.stack.append(elt)
     def top(self):
         return self.stack[-1]
+    def __getitem__(self, index): # needed by visitContinue()
+        return self.stack[index]
+
+MANGLE_LEN = 256 # magic constant from compile.c
+
+def mangle(name, klass):
+    if not name.startswith('__'):
+        return name
+    if len(name) + 2 >= MANGLE_LEN:
+        return name
+    if name.endswith('__'):
+        return name
+    try:
+        i = 0
+        while klass[i] == '_':
+            i = i + 1
+    except IndexError:
+        return name
+    klass = klass[i:]
+
+    tlen = len(klass) + len(name)
+    if tlen > MANGLE_LEN:
+        klass = klass[:MANGLE_LEN-tlen]
+
+    return "_%s%s" % (klass, name)
+

lib/python/RestrictedPython/compiler/pyassem.py

 """A flow graph representation for Python bytecode"""
 
 from __future__ import nested_scopes
+
 import dis
 import new
 import string
@@ -8,6 +9,7 @@ import sys
 import types
 
 import misc
+from consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS
 
 def xxx_sort(l):
     l = l[:]
@@ -54,7 +56,7 @@ class FlowGraph:
         # these edges to get the blocks emitted in the right order,
         # however. :-(  If a client needs to remove these edges, call
         # pruneEdges().
-        
+
         self.current.addNext(block)
         self.startBlock(block)
 
@@ -109,13 +111,13 @@ class FlowGraph:
 
         # XXX This is a total mess.  There must be a better way to get
         # the code blocks in the right order.
-        
+
         self.fixupOrderHonorNext(blocks, default_next)
         self.fixupOrderForward(blocks, default_next)
 
     def fixupOrderHonorNext(self, blocks, default_next):
         """Fix one problem with DFS.
-        
+
         The DFS uses child block, but doesn't know about the special
         "next" block.  As a result, the DFS can order blocks so that a
         block isn't next to the right block for implicit control
@@ -195,19 +197,18 @@ class FlowGraph:
             chains.remove(c)
             chains.insert(goes_before, c)
 
-
         del blocks[:]
         for c in chains:
             for b in c:
                 blocks.append(b)
-            
+
     def getBlocks(self):
         return self.blocks.elements()
 
     def getRoot(self):
         """Return nodes appropriate for use with dominator"""
         return self.entry
-    
+
     def getContainedGraphs(self):
         l = []
         for b in self.getBlocks():
@@ -246,7 +247,7 @@ class Block:
     def __str__(self):
         insts = map(str, self.insts)
         return "<block %s %d:\n%s>" % (self.label, self.bid,
-                                       string.join(insts, '\n')) 
+                                       string.join(insts, '\n'))
 
     def emit(self, inst):
         op = inst[0]
@@ -268,7 +269,7 @@ class Block:
         assert len(self.next) == 1, map(str, self.next)
 
     _uncond_transfer = ('RETURN_VALUE', 'RAISE_VARARGS',
-                        'JUMP_ABSOLUTE', 'JUMP_FORWARD')
+                        'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'CONTINUE_LOOP')
 
     def pruneNext(self):
         """Remove bogus edge for unconditional transfers
@@ -312,11 +313,6 @@ class Block:
         return contained
 
 # flags for code objects
-CO_OPTIMIZED = 0x0001
-CO_NEWLOCALS = 0x0002
-CO_VARARGS = 0x0004
-CO_VARKEYWORDS = 0x0008
-CO_NESTED = 0x0010
 
 # the FlowGraph is transformed in place; it exists in one of these states
 RAW = "RAW"
@@ -327,15 +323,17 @@ DONE = "DONE"
 class PyFlowGraph(FlowGraph):
     super_init = FlowGraph.__init__
 
-    def __init__(self, name, filename, args=(), optimized=0):
+    def __init__(self, name, filename, args=(), optimized=0, klass=None):
         self.super_init()
         self.name = name
+        assert isinstance(filename, types.StringType)
         self.filename = filename
         self.docstring = None
         self.args = args # XXX
         self.argcount = getArgCount(args)
+        self.klass = klass
         if optimized:
-            self.flags = CO_OPTIMIZED | CO_NEWLOCALS 
+            self.flags = CO_OPTIMIZED | CO_NEWLOCALS
         else:
             self.flags = 0
         self.consts = []
@@ -364,6 +362,10 @@ class PyFlowGraph(FlowGraph):
         if flag == CO_VARARGS:
             self.argcount = self.argcount - 1
 
+    def checkFlag(self, flag):
+        if self.flags & flag:
+            return 1
+
     def setFreeVars(self, names):
         self.freevars = list(names)
 
@@ -462,7 +464,6 @@ class PyFlowGraph(FlowGraph):
                 insts[i] = opname, offset
             elif self.hasjabs.has_elt(opname):
                 insts[i] = opname, begin[inst[1]]
-        self.stacksize = findDepth(self.insts)
         self.stage = FLAT
 
     hasjrel = misc.Set()
@@ -480,8 +481,7 @@ class PyFlowGraph(FlowGraph):
         for i in range(len(self.insts)):
             t = self.insts[i]
             if len(t) == 2:
-                opname = t[0]
-                oparg = t[1]
+                opname, oparg = t
                 conv = self._converters.get(opname, None)
                 if conv:
                     self.insts[i] = opname, conv(self, oparg)
@@ -501,10 +501,16 @@ class PyFlowGraph(FlowGraph):
         self.closure = self.cellvars + self.freevars
 
     def _lookupName(self, name, list):
-        """Return index of name in list, appending if necessary"""
+        """Return index of name in list, appending if necessary
+
+        This routine uses a list instead of a dictionary, because a
+        dictionary can't store two different keys if the keys have the
+        same value but different types, e.g. 2 and 2L.  The compiler
+        must treat these two separately, so it does an explicit type
+        comparison before comparing the values.
+        """
         t = type(name)
         for i in range(len(list)):
-            # must do a comparison on type first to prevent UnicodeErrors 
             if t == type(list[i]) and list[i] == name:
                 return i
         end = len(list)
@@ -523,9 +529,15 @@ class PyFlowGraph(FlowGraph):
     _convert_STORE_FAST = _convert_LOAD_FAST
     _convert_DELETE_FAST = _convert_LOAD_FAST
 
+    def _convert_LOAD_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
+        return self._lookupName(arg, self.names)
+
     def _convert_NAME(self, arg):
+        if self.klass is None:
+            self._lookupName(arg, self.varnames)
         return self._lookupName(arg, self.names)
-    _convert_LOAD_NAME = _convert_NAME
     _convert_STORE_NAME = _convert_NAME
     _convert_DELETE_NAME = _convert_NAME
     _convert_IMPORT_NAME = _convert_NAME
@@ -557,7 +569,7 @@ class PyFlowGraph(FlowGraph):
     for name, obj in locals().items():
         if name[:9] == "_convert_":
             opname = name[9:]
-            _converters[opname] = obj            
+            _converters[opname] = obj
     del name, obj, opname
 
     def makeByteCode(self):
@@ -587,13 +599,14 @@ class PyFlowGraph(FlowGraph):
 
     def newCodeObject(self):
         assert self.stage == DONE
-        if self.flags == 0:
+        if (self.flags & CO_NEWLOCALS) == 0:
             nlocals = 0
         else:
             nlocals = len(self.varnames)
         argcount = self.argcount
         if self.flags & CO_VARKEYWORDS:
             argcount = argcount - 1
+
         return new.code(argcount, nlocals, self.stacksize, self.flags,
                         self.lnotab.getCode(), self.getConsts(),
                         tuple(self.names), tuple(self.varnames),
@@ -613,7 +626,7 @@ class PyFlowGraph(FlowGraph):
                 elt = elt.getCode()
             l.append(elt)
         return tuple(l)
-            
+
 def isJump(opname):
     if opname[:4] == 'JUMP':
         return 1
@@ -707,17 +720,19 @@ class LineAddrTable:
 
     def getTable(self):
         return string.join(map(chr, self.lnotab), '')
-    
+
 class StackDepthTracker:
     # XXX 1. need to keep track of stack depth on jumps
     # XXX 2. at least partly as a result, this code is broken
 
-    def findDepth(self, insts):
+    def findDepth(self, insts, debug=0):
         depth = 0
         maxDepth = 0
         for i in insts:
             opname = i[0]
-            delta = self.effect.get(opname)
+            if debug:
+                print i,
+            delta = self.effect.get(opname, None)
             if delta is not None:
                 depth = depth + delta
             else:
@@ -732,10 +747,10 @@ class StackDepthTracker:
                     meth = getattr(self, opname, None)
                     if meth is not None:
                         depth = depth + meth(i[1])
-            if depth < 0:
-                depth = 0
             if depth > maxDepth:
                 maxDepth = depth
+            if debug:
+                print depth, maxDepth
         return maxDepth
 
     effect = {
@@ -756,9 +771,8 @@ class StackDepthTracker:
         'DELETE_SUBSCR': -2,
         # PRINT_EXPR?
         'PRINT_ITEM': -1,
-        'LOAD_LOCALS': 1,
         'RETURN_VALUE': -1,
-        'EXEC_STMT': -2,
+        'EXEC_STMT': -3,
         'BUILD_CLASS': -2,
         'STORE_NAME': -1,
         'STORE_ATTR': -2,
@@ -774,23 +788,20 @@ class StackDepthTracker:
         # close enough...
         'SETUP_EXCEPT': 3,
         'SETUP_FINALLY': 3,
-        'FOR_ITER': 1,
+        'FOR_LOOP': 1,
         }
     # use pattern match
     patterns = [
         ('BINARY_', -1),
         ('LOAD_', 1),
         ]
-    
-    # special cases:
-    # UNPACK_SEQUENCE, BUILD_TUPLE,
-    # BUILD_LIST, CALL_FUNCTION, MAKE_FUNCTION, BUILD_SLICE
+
     def UNPACK_SEQUENCE(self, count):
         return count-1
     def BUILD_TUPLE(self, count):
-        return 1-count
+        return -count+1
     def BUILD_LIST(self, count):
-        return 1-count
+        return -count+1
     def CALL_FUNCTION(self, argc):
         hi, lo = divmod(argc, 256)
         return -(lo + hi * 2)
@@ -802,6 +813,9 @@ class StackDepthTracker:
         return self.CALL_FUNCTION(argc)-2
     def MAKE_FUNCTION(self, argc):
         return -argc
+    def MAKE_CLOSURE(self, argc):
+        # XXX need to account for free variables too!
+        return -argc
     def BUILD_SLICE(self, argc):
         if argc == 2:
             return -1
@@ -809,5 +823,5 @@ class StackDepthTracker:
             return -2
     def DUP_TOPX(self, argc):
         return argc
-    
+
 findDepth = StackDepthTracker().findDepth

lib/python/RestrictedPython/compiler/symbols.py

@@ -2,6 +2,7 @@
 
 import ast
 from consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN
+from misc import mangle
 import types
 
 import sys
@@ -36,13 +37,7 @@ class Scope:
     def mangle(self, name):
         if self.klass is None:
             return name
-        if not name.startswith('__'):
-            return name
-        if len(name) + 2 >= MANGLE_LEN:
-            return name
-        if name.endswith('__'):
-            return name
-        return "_%s%s" % (self.klass, name)
+        return mangle(name, self.klass)
 
     def add_def(self, name):
         self.defs[self.mangle(name)] = 1
@@ -295,6 +290,27 @@ class SymbolVisitor:
                 name = name[:i]
             scope.add_def(asname or name)
 
+    def visitGlobal(self, node, scope):
+        for name in node.names:
+            scope.add_global(name)
+
+    def visitAssign(self, node, scope):
+        """Propagate assignment flag down to child nodes.
+
+        The Assign node doesn't itself contains the variables being
+        assigned to.  Instead, the children in node.nodes are visited
+        with the assign flag set to true.  When the names occur in
+        those nodes, they are marked as defs.
+
+        Some names that occur in an assignment target are not bound by
+        the assignment, e.g. a name occurring inside a slice.  The
+        visitor handles these nodes specially; they do not propagate
+        the assign flag to their children.
+        """
+        for n in node.nodes:
+            self.visit(n, scope, 1)
+        self.visit(node.expr, scope)
+
     def visitAssName(self, node, scope, assign=1):
         scope.add_def(node.name)
 
@@ -306,6 +322,13 @@ class SymbolVisitor:
         for n in node.subs:
             self.visit(n, scope, 0)
 
+    def visitSlice(self, node, scope, assign=0):
+        self.visit(node.expr, scope, 0)
+        if node.lower:
+            self.visit(node.lower, scope, 0)
+        if node.upper:
+            self.visit(node.upper, scope, 0)
+
     def visitAugAssign(self, node, scope):
         # If the LHS is a name, then this counts as assignment.
         # Otherwise, it's just use.
@@ -314,15 +337,6 @@ class SymbolVisitor:
             self.visit(node.node, scope, 1) # XXX worry about this
         self.visit(node.expr, scope)
 
-    def visitAssign(self, node, scope):
-        for n in node.nodes:
-            self.visit(n, scope, 1)
-        self.visit(node.expr, scope)
-
-    def visitGlobal(self, node, scope):
-        for name in node.names:
-            scope.add_global(name)
-
     # prune if statements if tests are false
 
     _const_types = types.StringType, types.IntType, types.FloatType
@@ -348,7 +362,8 @@ def list_eq(l1, l2):
 
 if __name__ == "__main__":
     import sys
-    from compiler import parseFile, walk
+    from transformer import parseFile
+    from visitor import walk
     import symtable
 
     def get_names(syms):

lib/python/RestrictedPython/compiler/transformer.py

@@ -153,12 +153,8 @@ class Transformer:
 
     def file_input(self, nodelist):
         doc = self.get_docstring(nodelist, symbol.file_input)
-        if doc is not None:
-            i = 1
-        else:
-            i = 0
         stmts = []
-        for node in nodelist[i:]:
+        for node in nodelist:
             if node[0] != token.ENDMARKER and node[0] != token.NEWLINE:
                 self.com_append_stmt(stmts, node)
         return Module(doc, Stmt(stmts))
@@ -344,11 +340,6 @@ class Transformer:
         n.lineno = nodelist[0][2]
         return n
 
-    def yield_stmt(self, nodelist):
-        n = Yield(self.com_node(nodelist[1]))
-        n.lineno = nodelist[0][2]
-        return n
-
     def raise_stmt(self, nodelist):
         # raise: [test [',' test [',' test]]]
         if len(nodelist) > 5:
@@ -1254,7 +1245,6 @@ _legal_node_types = [
     symbol.continue_stmt,
     symbol.return_stmt,
     symbol.raise_stmt,
-    #symbol.yield_stmt,
     symbol.import_stmt,
     symbol.global_stmt,
     symbol.exec_stmt,
@@ -1281,9 +1271,6 @@ _legal_node_types = [
     symbol.atom,
     ]
 
-if hasattr(symbol, 'yield_stmt'):
-    _legal_node_types.append(symbol.yield_stmt)
-
 _assign_types = [
     symbol.test,
     symbol.and_test,