You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1277 lines
40 KiB
1277 lines
40 KiB
""" |
|
Module that defines a common AST description, independent from Python |
|
version and implementation. Also provides a function ``parse()`` to |
|
generate this common AST by using the builtin ast module and converting |
|
the result. Supports CPython 2.7, CPython 3.2+, Pypy. |
|
|
|
https://github.com/almarklein/commonast |
|
""" |
|
|
|
from __future__ import print_function, absolute_import |
|
|
|
import sys |
|
import ast |
|
import json |
|
import base64 |
|
|
|
if hasattr(base64, "encodebytes"): |
|
encodebytes = base64.encodebytes |
|
decodebytes = base64.decodebytes |
|
else: |
|
encodebytes = base64.encodestring |
|
decodebytes = base64.decodestring |
|
|
|
pyversion = sys.version_info |
|
NoneType = None.__class__ |
|
_Ellipsis = Ellipsis |
|
|
|
if pyversion >= (3, ): |
|
basestring = str # noqa |
|
|
|
|
|
# do some extra asserts when running tests, but not always, for speed |
|
docheck = 'pytest' in sys.modules |
|
|
|
def parse(code, comments=False): |
|
""" Parse Python code to produce a common AST tree. |
|
|
|
Parameters: |
|
code (str): the Python code to parse |
|
comments (bool): if True, will include Comment nodes. Default False. |
|
""" |
|
converter = NativeAstConverter(code) |
|
return converter.convert(comments) |
|
|
|
|
|
class Node(object): |
|
""" Abstract base class for all Nodes. |
|
""" |
|
|
|
__slots__ = ['lineno', 'col_offset'] |
|
|
|
class OPS: |
|
""" Operator enums: """ |
|
# Unary |
|
UAdd = 'UAdd' |
|
USub = 'USub' |
|
Not = 'Not' |
|
Invert = 'Invert' |
|
# Binary |
|
Add = 'Add' |
|
Sub = 'Sub' |
|
Mult = 'Mult' |
|
Div = 'Div' |
|
FloorDiv = 'FloorDiv' |
|
Mod = 'Mod' |
|
Pow = 'Pow' |
|
LShift = 'LShift' |
|
RShift = 'RShift' |
|
BitOr = 'BitOr' |
|
BitXor = 'BitXor' |
|
BitAnd = 'BitAnd' |
|
# Boolean |
|
And = 'And' |
|
Or = 'Or' |
|
|
|
class COMP: |
|
""" Comparison enums: """ |
|
Eq = 'Eq' |
|
NotEq = 'NotEq' |
|
Lt = 'Lt' |
|
LtE = 'LtE' |
|
Gt = 'Gt' |
|
GtE = 'GtE' |
|
Is = 'Is' |
|
IsNot = 'IsNot' |
|
In = 'In' |
|
NotIn = 'NotIn' |
|
|
|
def __init__(self, *args): |
|
names = self.__slots__ |
|
# Checks |
|
assert len(args) == len(names) # check this always |
|
if docheck: |
|
assert not hasattr(self, '__dict__'), 'Nodes must have __slots__' |
|
assert self.__class__ is not Node, 'Node is an abstract class' |
|
for name, val in zip(names, args): |
|
assert not isinstance(val, ast.AST) |
|
if name == 'name': |
|
assert isinstance(val, (basestring, NoneType)), 'name not a string' |
|
elif name == 'op': |
|
assert val in Node.OPS.__dict__ or val in Node.COMP.__dict__ |
|
elif name.endswith('_node'): |
|
assert isinstance(val, (Node, NoneType)), '%r is not a Node' % name |
|
elif name.endswith('_nodes'): |
|
islistofnodes = (isinstance(val, list) and |
|
all(isinstance(n, Node) for n in val)) |
|
assert islistofnodes, '%r is not a list of nodes' % name |
|
else: |
|
assert not isinstance(val, Node), '%r should not be a Node' % name |
|
assert not (isinstance(val, list) and |
|
all(isinstance(n, Node) for n in val)) |
|
# Assign |
|
for name, val in zip(names, args): |
|
setattr(self, name, val) |
|
|
|
def tojson(self, indent=2): |
|
""" Return a string with the JSON representatiom of this AST. |
|
Set indent to None for a more compact representation. |
|
""" |
|
return json.dumps(self._todict(), indent=indent, sort_keys=True) |
|
|
|
@classmethod |
|
def fromjson(cls, text): |
|
""" Classmethod to create an AST tree from JSON. |
|
""" |
|
return Node._fromdict(json.loads(text)) |
|
|
|
@classmethod |
|
def _fromdict(cls, d): |
|
assert '_type' in d |
|
Cls = globals()[d['_type']] |
|
|
|
args = [] |
|
for name in Cls.__slots__: |
|
val = d[name] |
|
if val is None: |
|
pass |
|
elif name.endswith('_node'): |
|
val = Node._fromdict(val) |
|
elif name.endswith('_nodes'): |
|
val = [Node._fromdict(x) for x in val] |
|
elif isinstance(val, basestring): |
|
if val.startswith('BYTES:'): |
|
val = decodebytes(val[6:].encode('utf-8')) |
|
elif val.startswith('COMPLEX:'): |
|
val = complex(val[8:]) |
|
elif pyversion < (3, ): |
|
val = unicode(val) # noqa |
|
args.append(val) |
|
return Cls(*args) |
|
|
|
def _todict(self): |
|
""" Get a dict representing this AST. This is the basis for |
|
creating JSON, but can be used to compare AST trees as well. |
|
""" |
|
d = {} |
|
d['_type'] = self.__class__.__name__ |
|
for name in self.__slots__: |
|
val = getattr(self, name) |
|
if val is None: |
|
pass |
|
elif name.endswith('_node'): |
|
val = val._todict() |
|
elif name.endswith('_nodes'): |
|
val = [x._todict() for x in val] |
|
elif isinstance(self, Bytes) and isinstance(val, bytes): |
|
val = 'BYTES:' + encodebytes(val).decode('utf-8').rstrip() |
|
elif isinstance(self, Num) and isinstance(val, complex): |
|
val = 'COMPLEX:' + repr(val) |
|
d[name] = val |
|
return d |
|
|
|
def __eq__(self, other): |
|
if not isinstance(other, Node): |
|
raise ValueError('Can only compare nodes to other nodes.') |
|
return self._todict() == other._todict() |
|
|
|
def __repr__(self): |
|
names = ', '.join([repr(x) for x in self.__slots__]) |
|
return '<%s with %s at 0x%x>' % (self.__class__.__name__, names, id(self)) |
|
|
|
def __str__(self): |
|
return self.tojson() |
|
|
|
|
|
try: |
|
Node.OPS.__doc__ += ', '.join([x for x in sorted(Node.OPS.__dict__) |
|
if not x.startswith('_')]) |
|
Node.COMP.__doc__ += ', '.join([x for x in sorted(Node.COMP.__dict__) |
|
if not x.startswith('_')]) |
|
except AttributeError: # pragma: no cover |
|
pass # Py < 3.3 |
|
|
|
|
|
## -- (start marker for doc generator) |
|
|
|
## General |
|
|
|
class Comment(Node): |
|
""" |
|
Attributes: |
|
value: the comment string. |
|
""" |
|
__slots__ = 'value', |
|
|
|
class Module(Node): |
|
""" Each code that an AST is created for gets wrapped in a Module node. |
|
|
|
Attributes: |
|
body_nodes: a list of nodes. |
|
""" |
|
__slots__ = 'body_nodes', |
|
|
|
## Literals |
|
|
|
class Num(Node): |
|
""" |
|
Attributes: |
|
value: the number as a native Python object (int, float, or complex). |
|
""" |
|
__slots__ = 'value', |
|
|
|
class Str(Node): |
|
""" |
|
Attributes: |
|
value: the native Python str object. |
|
""" |
|
__slots__ = 'value', |
|
|
|
class FormattedValue(Node): |
|
""" Node representing a single formatting field in an f-string. If the |
|
string contains a single formatting field and nothing else the node can be |
|
isolated, otherwise it appears in JoinedStr. |
|
|
|
Attributes: |
|
value_node: an expression (can be anything). |
|
conversion: a string, '' means no formatting, 's' means !s string |
|
formatting, 'r' means !r repr formatting, 'a' means !a ascii |
|
formatting. |
|
format_node: a JoinedStr node reprensenting the formatting, or None |
|
if no format was specified. Both conversion and format_node can be |
|
set at the same time. |
|
""" |
|
__slots__ = 'value_node', 'conversion', 'format_node' |
|
|
|
class JoinedStr(Node): |
|
""" An f-string, comprising a series of FormattedValue and Str nodes. |
|
|
|
Attributes: |
|
value_nodes: list of Str and FormattedValue nodes. |
|
""" |
|
__slots__ = 'value_nodes', |
|
|
|
class Bytes(Node): |
|
""" |
|
Attributes: |
|
value: the native Python bytes object. |
|
""" |
|
__slots__ = 'value', |
|
|
|
class List(Node): |
|
""" |
|
Attributes: |
|
element_nodes: the items in the list. |
|
""" |
|
__slots__ = 'element_nodes', |
|
|
|
class Tuple(Node): |
|
""" |
|
Attributes: |
|
element_nodes: the items in the tuple. |
|
""" |
|
__slots__ = 'element_nodes', |
|
|
|
class Set(Node): |
|
""" |
|
Attributes: |
|
element_nodes: the items in the set. |
|
""" |
|
__slots__ = 'element_nodes', |
|
|
|
class Dict(Node): |
|
""" |
|
Attributes: |
|
key_nodes: the keys of the dict. |
|
value_nodes: the corresponding values. |
|
""" |
|
__slots__ = 'key_nodes', 'value_nodes' |
|
|
|
class Ellipsis(Node): |
|
""" Represents the ``...`` syntax for the Ellipsis singleton. |
|
""" |
|
__slots__ = () |
|
|
|
class NameConstant(Node): |
|
""" |
|
Attributes: |
|
value: the corresponding native Python object like True, False or None. |
|
""" |
|
__slots__ = 'value', |
|
|
|
## Variables, attributes, indexing and slicing |
|
|
|
class Name(Node): |
|
""" |
|
Attributes: |
|
name: the string name of this variable. |
|
""" |
|
__slots__ = 'name', |
|
|
|
class Starred(Node): |
|
""" A starred variable name, e.g. ``*foo``. Note that this isn't |
|
used to define a function with ``*args`` - FunctionDef nodes have |
|
special fields for that. |
|
|
|
Attributes: |
|
value_node: the value that is starred, typically a Name node. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class Attribute(Node): |
|
""" Attribute access, e.g. ``foo.bar``. |
|
|
|
Attributes: |
|
value_node: The node to get/set an attribute of. Typically a Name node. |
|
attr: a string with the name of the attribute. |
|
""" |
|
__slots__ = 'value_node', 'attr' |
|
|
|
class Subscript(Node): |
|
""" Subscript access, e.g. ``foo[3]``. |
|
|
|
Attributes: |
|
value_node: The node to get/set a subscript of. Typically a Name node. |
|
slice_node: An Index, Slice or ExtSlice node. |
|
""" |
|
__slots__ = 'value_node', 'slice_node' |
|
|
|
class Index(Node): |
|
""" |
|
Attributes: |
|
value_node: Single index. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class Slice(Node): |
|
""" |
|
Attributes: |
|
lower_node: start slice. |
|
upper_node: end slice. |
|
step_node: slice step. |
|
""" |
|
__slots__ = 'lower_node', 'upper_node', 'step_node' |
|
|
|
class ExtSlice(Node): |
|
""" |
|
Attributes: |
|
dim_nodes: list of Index and Slice nodes (of for each dimension). |
|
""" |
|
__slots__ = 'dim_nodes', |
|
|
|
## Expressions |
|
|
|
class Expr(Node): |
|
""" When an expression, such as a function call, appears as a |
|
statement by itself (an expression statement), with its return value |
|
not used or stored, it is wrapped in this container. |
|
|
|
Attributes: |
|
value_node: holds one of the other nodes in this section, or a |
|
literal, a Name, a Lambda, or a Yield or YieldFrom node. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class UnaryOp(Node): |
|
""" A unary operation (e.g. ``-x``, ``not x``). |
|
|
|
Attributes: |
|
op: the operator (an enum from ``Node.OPS``). |
|
right_node: the operand at the right of the operator. |
|
""" |
|
__slots__ = 'op', 'right_node' |
|
|
|
class BinOp(Node): |
|
""" A binary operation (e.g. ``a / b``, ``a + b``). |
|
|
|
Attributes: |
|
op: the operator (an enum from ``Node.OPS``). |
|
left_node: the node to the left of the operator. |
|
right_node: the node to the right of the operator. |
|
""" |
|
__slots__ = 'op', 'left_node', 'right_node' |
|
|
|
class BoolOp(Node): |
|
""" A boolean operator (``and``, ``or``, but not ``not``). |
|
|
|
Attributes: |
|
op: the operator (an enum from ``Node.OPS``). |
|
value_nodes: a list of nodes. ``a``, ``b`` and ``c`` in |
|
``a or b or c``. |
|
""" |
|
__slots__ = 'op', 'value_nodes' |
|
|
|
class Compare(Node): |
|
""" A comparison of two or more values. |
|
|
|
Attributes: |
|
op: the comparison operator (an enum from ``Node.COMP``). |
|
left_node: the node to the left of the operator. |
|
right_node: the node to the right of the operator. |
|
""" |
|
__slots__ = 'op', 'left_node', 'right_node' |
|
|
|
class Call(Node): |
|
""" A function call. |
|
|
|
Attributes: |
|
func_node: Name, Attribute or SubScript node that represents |
|
the function. |
|
arg_nodes: list of nodes representing positional arguments. |
|
kwarg_nodes: list of Keyword nodes representing keyword arguments. |
|
|
|
Note that an argument ``*x`` would be specified as a Starred node |
|
in arg_nodes, and ``**y`` as a Keyword node with a name being ``None``. |
|
""" |
|
__slots__ = ('func_node', 'arg_nodes', 'kwarg_nodes') |
|
|
|
class Keyword(Node): |
|
""" Keyword argument used in a Call. |
|
|
|
Attributes: |
|
name: the (string) name of the argument. Is None for ``**kwargs``. |
|
value_node: the value of the arg. |
|
""" |
|
__slots__ = ('name', 'value_node') |
|
|
|
class IfExp(Node): |
|
""" An expression such as ``a if b else c``. |
|
|
|
Attributes: |
|
test_node: the ``b`` in the above. |
|
body_node: the ``a`` in the above. |
|
else_node: the ``c`` in the above. |
|
""" |
|
__slots__ = 'test_node', 'body_node', 'else_node' |
|
|
|
class ListComp(Node): |
|
""" List comprehension. |
|
|
|
Attributes: |
|
element_node: the part being evaluated for each item. |
|
comp_nodes: a list of Comprehension nodes. |
|
""" |
|
__slots__ = 'element_node', 'comp_nodes' |
|
|
|
class SetComp(Node): |
|
""" Set comprehension. See ListComp. |
|
""" |
|
__slots__ = 'element_node', 'comp_nodes' |
|
|
|
class GeneratorExp(Node): |
|
""" Generor expression. See ListComp. |
|
""" |
|
__slots__ = 'element_node', 'comp_nodes' |
|
|
|
class DictComp(Node): |
|
""" Dict comprehension. |
|
|
|
Attributes: |
|
key_node: the key of the item being evaluated. |
|
value_node: the value of the item being evaluated. |
|
comp_nodes: a list of Comprehension nodes. |
|
""" |
|
__slots__ = 'key_node', 'value_node', 'comp_nodes' |
|
|
|
class Comprehension(Node): |
|
""" Represents a single for-clause in a comprehension. |
|
|
|
Attributes: |
|
target_node: reference to use for each element, typically a |
|
Name or Tuple node. |
|
iter_node: the object to iterate over. |
|
if_nodes: a list of test expressions. |
|
""" |
|
__slots__ = 'target_node', 'iter_node', 'if_nodes' |
|
|
|
|
|
## Statements |
|
|
|
class Assign(Node): |
|
""" Assignment of a value to a variable. |
|
|
|
Attributes: |
|
target_nodes: variables to assign to, Name or SubScript. |
|
value_node: the object to assign. |
|
""" |
|
__slots__ = 'target_nodes', 'value_node' |
|
|
|
class AugAssign(Node): |
|
""" Augmented assignment, such as ``a += 1``. |
|
|
|
Attributes: |
|
target_node: variable to assign to, Name or SubScript. |
|
op: operator enum (e.g. ``Node.OPS.Add``) |
|
value_node: the object to assign. |
|
""" |
|
__slots__ = 'target_node', 'op', 'value_node' |
|
|
|
|
|
class Raise(Node): |
|
""" Raising an exception. |
|
|
|
Attributes: |
|
exc_node: the exception object to be raised, normally a Call |
|
or Name, or None for a standalone raise. |
|
cause_node: the optional part for y in raise x from y. |
|
""" |
|
__slots__ = 'exc_node', 'cause_node' |
|
|
|
class Assert(Node): |
|
""" An assertion. |
|
|
|
Attributes: |
|
test_node: the condition to test. |
|
msg_node: the failure message (commonly a Str node) |
|
""" |
|
__slots__ = 'test_node', 'msg_node' |
|
|
|
class Delete(Node): |
|
""" A del statement. |
|
|
|
Attributes: |
|
target_nodes: the variables to delete, such as Name, Attribute |
|
or Subscript nodes. |
|
""" |
|
__slots__ = 'target_nodes', |
|
|
|
class Pass(Node): |
|
""" Do nothing. |
|
""" |
|
__slots__ = () |
|
|
|
class Import(Node): |
|
""" An import statement. |
|
|
|
Attributes: |
|
root: the name of the module to import from. None if this is |
|
not a from-import. |
|
names: list of (name, alias) tuples, where alias can be None. |
|
level: an integer indicating depth of import. Zero means |
|
absolute import. |
|
""" |
|
__slots__ = 'root', 'names', 'level' |
|
|
|
## Control flow |
|
|
|
class If(Node): |
|
""" An if-statement. |
|
|
|
Note that elif clauses don't have a special representation in the |
|
AST, but rather appear as extra If nodes within the else section |
|
of the previous one. |
|
|
|
Attributes: |
|
test_node: the test, e.g. a Compare node. |
|
body_nodes: the body of the if-statement. |
|
else_nodes: the body of the else-clause of the if-statement. |
|
""" |
|
__slots__ = 'test_node', 'body_nodes', 'else_nodes' |
|
|
|
class For(Node): |
|
""" A for-loop. |
|
|
|
Attributes: |
|
target_node: the variable(s) the loop assigns to. |
|
iter_node: the object to iterate over. |
|
body_nodes: the body of the for-loop. |
|
else_nodes: the body of the else-clause of the for-loop. |
|
""" |
|
__slots__ = 'target_node', 'iter_node', 'body_nodes', 'else_nodes' |
|
|
|
class While(Node): |
|
""" A while-loop. |
|
|
|
Attributes: |
|
test_node: the test to perform on each iteration. |
|
body_nodes: the body of the for-loop. |
|
else_nodes: the body of the else-clause of the for-loop. |
|
""" |
|
__slots__ = 'test_node', 'body_nodes', 'else_nodes' |
|
|
|
class Break(Node): |
|
""" Break from a loop. |
|
""" |
|
__slots__ = () |
|
|
|
class Continue(Node): |
|
""" Continue with next iteration of a loop. |
|
""" |
|
__slots__ = () |
|
|
|
class Try(Node): |
|
""" Try-block. |
|
|
|
Attributes: |
|
body_nodes: the body of the try-block (i.e. the code to try). |
|
handler_nodes: a list of ExceptHandler instances. |
|
else_nodes: the body of the else-clause of the try-block. |
|
finally_nodes: the body of the finally-clause of the try-block. |
|
""" |
|
__slots__ = 'body_nodes', 'handler_nodes', 'else_nodes', 'finally_nodes' |
|
|
|
class ExceptHandler(Node): |
|
""" Single except-clause. |
|
|
|
Attributes: |
|
type_node: the type of exception to catch. Often a Name node |
|
or None to catch all. |
|
name: the string name of the exception object in case of ``as err``. |
|
None otherwise. |
|
body_nodes: the body of the except-clause. |
|
""" |
|
__slots__ = 'type_node', 'name', 'body_nodes' |
|
|
|
class With(Node): |
|
""" A with-block (i.e. a context manager). |
|
|
|
Attributes: |
|
item_nodes: a list of WithItem nodes (i.e. context managers). |
|
body_nodes: the body of the with-block. |
|
""" |
|
__slots__ = 'item_nodes', 'body_nodes' |
|
|
|
class WithItem(Node): |
|
""" A single context manager in a with block. |
|
|
|
Attributes: |
|
expr_node: the expression for the context manager. |
|
as_node: a Name, Tuple or List node representing the ``as foo`` part. |
|
""" |
|
__slots__ = 'expr_node', 'as_node' |
|
|
|
## Function and class definitions |
|
|
|
class FunctionDef(Node): |
|
""" A function definition. |
|
|
|
Attributes: |
|
name: the (string) name of the function. |
|
decorator_nodes: the list of decorators to be applied, stored |
|
outermost first (i.e. the first in the list will be applied |
|
last). |
|
annotation_node: the return annotation (Python 3 only). |
|
arg_nodes: list of Args nodes representing positional arguments. |
|
These *may* have a default value. |
|
kwarg_nodes: list of Arg nodes representing keyword-only arguments. |
|
args_node: an Arg node representing ``*args``. |
|
kwargs_node: an Arg node representing ``**kwargs``. |
|
body_nodes: the body of the function. |
|
""" |
|
__slots__ = ('name', 'decorator_nodes', 'annotation_node', |
|
'arg_nodes', 'kwarg_nodes', 'args_node', 'kwargs_node', |
|
'body_nodes') |
|
|
|
class Lambda(Node): |
|
""" Anonymous function definition. |
|
|
|
Attributes: |
|
arg_nodes: list of Args nodes representing positional arguments. |
|
kwarg_nodes: list of Arg nodes representing keyword-only arguments. |
|
args_node: an Arg node representing ``*args``. |
|
kwargs_node: an Arg node representing ``**kwargs``. |
|
body_node: the body of the function (a single node). |
|
""" |
|
__slots__ = ('arg_nodes', 'kwarg_nodes', 'args_node', 'kwargs_node', |
|
'body_node') |
|
|
|
class AsyncFunctionDef(Node): |
|
""" Asynchronous function definition. |
|
|
|
Same as FunctionDef, but async. |
|
""" |
|
__slots__ = ('name', 'decorator_nodes', 'annotation_node', |
|
'arg_nodes', 'kwarg_nodes', 'args_node', 'kwargs_node', |
|
'body_nodes') |
|
|
|
class Arg(Node): |
|
""" Function argument for a FunctionDef. |
|
|
|
Attributes: |
|
name: the (string) name of the argument. |
|
value_node: the default value of this argument. Can be None. |
|
annotation_node: the annotation for this argument (Python3 only). |
|
""" |
|
|
|
__slots__ = ('name', 'value_node', 'annotation_node') |
|
|
|
class Return(Node): |
|
""" |
|
Attributes: |
|
value_node: the value to return. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class Yield(Node): |
|
""" |
|
Attributes: |
|
value_node: the value to yield. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class YieldFrom(Node): |
|
""" |
|
Attributes: |
|
value_node: the value to yield. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class Await(Node): |
|
""" |
|
Attributes: |
|
value_node: the value to return. |
|
""" |
|
__slots__ = 'value_node', |
|
|
|
class Global(Node): |
|
""" |
|
Attributes: |
|
names: a list of string names to declare global. |
|
""" |
|
__slots__ = 'names', |
|
|
|
class Nonlocal(Node): |
|
""" |
|
Attributes: |
|
names: a list of string names to declare nonlocal. |
|
""" |
|
__slots__ = 'names', |
|
|
|
class ClassDef(Node): |
|
""" A class definition. |
|
|
|
Attributes: |
|
name: a string for the class name. |
|
decorator_nodes: the list of decorators to be applied, as in FunctionDef. |
|
arg_nodes: list of nodes representing base classes. |
|
kwarg_nodes: list of Keyword nodes representing keyword-only arguments. |
|
body_nodes: the body of the class. |
|
|
|
Note that arg_nodes and kwarg_nodes are similar to those in the |
|
Call node. An argument ``*x`` would be specified as a Starred node |
|
in arg_nodes, and ``**y`` as a Keyword node with a name being |
|
``None``. For more information on keyword arguments see |
|
https://www.python.org/dev/peps/pep-3115/. |
|
""" |
|
__slots__ = ('name', 'decorator_nodes', 'arg_nodes', 'kwarg_nodes', 'body_nodes') |
|
|
|
## -- (end marker for doc generator) |
|
|
|
|
|
class NativeAstConverter: |
|
""" Convert ast produced by Python's ast module to common ast. |
|
""" |
|
|
|
def __init__(self, code): |
|
self._root = ast.parse(code) |
|
self._lines =code.splitlines() |
|
self._stack = [] # contains tuple elements: (list_obj, native_nodes) |
|
|
|
def _add_comments(self, container, lineno): |
|
""" Add comment nodes from the last point until the given line number. |
|
""" |
|
linenr1 = self._comment_pointer |
|
linenr2 = lineno |
|
self._comment_pointer = linenr2 + 1 # store for next time |
|
|
|
for i in range(linenr1, linenr2): |
|
line = self._lines[i-1] # lineno's start from 1 |
|
if line.lstrip().startswith('#'): |
|
before, _, comment = line.partition('#') |
|
node = Comment(comment) |
|
node.lineno = i |
|
node.col_offset = len(before) |
|
container.append(node) |
|
|
|
def convert(self, comments=False): |
|
assert not self._stack |
|
self._comment_pointer = 1 |
|
|
|
result = self._convert(self._root) |
|
|
|
while self._stack: |
|
container, native_nodes = self._stack.pop(0) |
|
for native_node in native_nodes: |
|
node = self._convert(native_node) |
|
if comments: |
|
self._add_comments(container, node.lineno) |
|
container.append(node) |
|
|
|
return result |
|
|
|
def _convert(self, n): |
|
|
|
# n is the native node produced by the ast module |
|
if n is None: |
|
return None # but some node attributes can be None |
|
assert isinstance(n, ast.AST) |
|
|
|
# Get converter function |
|
type = n.__class__.__name__ |
|
try: |
|
converter = getattr(self, '_convert_' + type) |
|
except AttributeError: # pragma: no cover |
|
raise RuntimeError('Cannot convert %s nodes.' % type) |
|
# Convert node |
|
val = converter(n) |
|
assert isinstance(val, Node) |
|
# Set its position |
|
val.lineno = getattr(n, 'lineno', 1) |
|
val.col_offset = getattr(n, 'col_offset', 0) |
|
return val |
|
|
|
def _convert_Module(self, n): |
|
node = Module([]) |
|
# Add back the "docstring" that Python removed; this may actually be |
|
# a code snippet and not a module. |
|
self._stack.append((node.body_nodes, n.body)) |
|
return node |
|
|
|
## Literals |
|
|
|
def _convert_Constant(self, n): |
|
val = n.value |
|
if val is None or val is True or val is False: |
|
return NameConstant(val) |
|
if isinstance(val, (int, float, complex)): |
|
return Num(val) |
|
if isinstance(val, str): |
|
return Str(val) |
|
if isinstance(val, bytes): |
|
return Bytes(val) |
|
if val is _Ellipsis: |
|
return Ellipsis() |
|
raise RuntimeError('Cannot convert %s constants.' % type(val).__name__) |
|
|
|
def _convert_Num(self, n): |
|
if pyversion < (3, ) and str(n.n).startswith('-'): |
|
# -4 is a unary sub on 4, dont forget complex numbers |
|
return UnaryOp(Node.OPS.USub, Num(-n.n)) |
|
return Num(n.n) |
|
|
|
def _convert_Str(self, n): |
|
# We check the string prefix here. We only really need it in Python 2, |
|
# because u is not needed in py3, and b and r are resolved by the lexer, |
|
# and f as well (resulting in JoinedStr or FormattedValue). |
|
# Note that the col_offset of the node seems 1 off when the string is |
|
# a key in a dict :/ (PScript issue #15) |
|
if pyversion < (3, ): |
|
line = self._lines[n.lineno-1] |
|
i = n.col_offset |
|
i = i - 1 if (i > 0 and line[i-1] in 'rufb"\'') else i |
|
pre = '' |
|
if line[i] not in '"\'': |
|
pre += line[i] |
|
if line[i + 1] not in '"\'': |
|
pre += line[i + 1] |
|
if 'b' in pre: |
|
return Bytes(n.s) |
|
return Str(n.s) |
|
|
|
def _convert_JoinedStr(self, n): |
|
c = self._convert |
|
return JoinedStr([c(x) for x in n.values]) |
|
|
|
def _convert_FormattedValue(self, n): |
|
conversion = '' if n.conversion < 0 else chr(n.conversion) |
|
return FormattedValue(self._convert(n.value), conversion, |
|
self._convert(n.format_spec)) |
|
|
|
def _convert_Bytes(self, n): |
|
return Bytes(n.s) |
|
|
|
def _convert_List(self, n): |
|
c = self._convert |
|
return List([c(x) for x in n.elts]) |
|
|
|
def _convert_Tuple(self, n): |
|
c = self._convert |
|
return Tuple([c(x) for x in n.elts]) |
|
|
|
def _convert_Set(self, n): |
|
c = self._convert |
|
return Set([c(x) for x in n.elts]) |
|
|
|
def _convert_Dict(self, n): |
|
c = self._convert |
|
return Dict([c(x) for x in n.keys], [c(x) for x in n.values]) |
|
|
|
def _convert_Ellipsis(self, n): |
|
if pyversion < (3, ): |
|
return Index(Ellipsis()) # Ellipses must be wrapped in an index |
|
return Ellipsis() |
|
|
|
def _convert_NameConstant(self, n): |
|
return NameConstant(n.value) |
|
|
|
## Variables, attributes, indexing and slicing |
|
|
|
def _convert_Name(self, n): |
|
if pyversion < (3, 4): # pragma: no cover |
|
M = {'None': None, 'False': False, 'True': True} |
|
if n.id in M: |
|
return NameConstant(M[n.id]) # Python < 3.4 |
|
if pyversion < (3, ) and isinstance(n.ctx , ast.Param): |
|
return Arg(n.id, None, None) |
|
return Name(n.id) |
|
|
|
def _convert_Starred(self, n): |
|
return Starred(self._convert(n.value)) |
|
|
|
def _convert_Attribute(self, n): |
|
return Attribute(self._convert(n.value), n.attr) |
|
|
|
def _convert_Subscript(self, n): |
|
return Subscript(self._convert(n.value), self._convert_index_like(n.slice)) |
|
|
|
def _convert_index_like(self, n): |
|
c = self._convert |
|
if isinstance(n, (ast.Slice, ast.Index, ast.ExtSlice, ast.Ellipsis)): |
|
return c(n) # Python < 3.8 (and also 3.8 on Windows?) |
|
elif isinstance(n, ast.Tuple): |
|
assert isinstance(n, ast.Tuple) |
|
dims = [self._convert_index_like(x) for x in n.elts] |
|
return Tuple(dims) |
|
else: # Num, Unary, Name, or ... |
|
return c(n) |
|
|
|
def _convert_Index(self, n): |
|
return self._convert(n.value) |
|
|
|
def _convert_Slice(self, n): |
|
c = self._convert |
|
step = c(n.step) |
|
if pyversion < (3, ) and isinstance(step, NameConstant) and step.value is None: |
|
if not self._lines[n.step.lineno-1][n.step.col_offset:].startswith('None'): |
|
step = None # silly Python 2 turns a[::] into a[::None] |
|
return Slice(c(n.lower), c(n.upper), step) |
|
|
|
def _convert_ExtSlice(self, n): |
|
return Tuple([self._convert_index_like(x) for x in n.dims]) |
|
|
|
## Expressions |
|
|
|
def _convert_Expr(self, n): |
|
return Expr(self._convert(n.value)) |
|
|
|
def _convert_UnaryOp(self, n): |
|
op = n.op.__class__.__name__ |
|
return UnaryOp(op, self._convert(n.operand)) |
|
|
|
def _convert_BinOp(self, n): |
|
op = n.op.__class__.__name__ |
|
return BinOp(op, self._convert(n.left), self._convert(n.right)) |
|
|
|
def _convert_BoolOp(self, n): |
|
c = self._convert |
|
op = n.op.__class__.__name__ |
|
return BoolOp(op, [c(x) for x in n.values]) # list of value_nodes |
|
|
|
def _convert_Compare(self, n): |
|
c = self._convert |
|
# Get compares and ops |
|
comps = [c(x) for x in ([n.left] + n.comparators)] |
|
ops = [op.__class__.__name__ for op in n.ops] |
|
assert len(ops) == (len(comps) - 1) |
|
# Create our comparison operators |
|
compares = [] |
|
for i in range(len(ops)): |
|
co = Compare(ops[i], comps[i], comps[i+1]) |
|
compares.append(co) |
|
# Return single or wrapped in an AND |
|
assert compares |
|
if len(compares) == 1: |
|
return compares[0] |
|
else: |
|
return BoolOp(Node.OPS.And, compares) |
|
|
|
def _convert_Call(self, n): |
|
c = self._convert |
|
arg_nodes = [c(a) for a in n.args] |
|
kwarg_nodes = [c(a) for a in n.keywords] |
|
|
|
if pyversion < (3, 5): |
|
if n.starargs: |
|
arg_nodes.append(Starred(c(n.starargs))) |
|
if n.kwargs: |
|
kwarg_nodes.append(Keyword(None, c(n.kwargs))) |
|
|
|
return Call(c(n.func), arg_nodes, kwarg_nodes) |
|
|
|
def _convert_keyword(self, n): |
|
return Keyword(n.arg, self._convert(n.value or None)) |
|
|
|
def _convert_IfExp(self, n): |
|
c = self._convert |
|
return IfExp(c(n.test), c(n.body), c(n.orelse)) |
|
|
|
def _convert_ListComp(self, n): |
|
c = self._convert |
|
return ListComp(c(n.elt), [c(x) for x in n.generators]) |
|
|
|
def _convert_SetComp(self, n): |
|
c = self._convert |
|
return SetComp(c(n.elt), [c(x) for x in n.generators]) |
|
|
|
def _convert_GeneratorExp(self, n): |
|
c = self._convert |
|
return GeneratorExp(c(n.elt), [c(x) for x in n.generators]) |
|
|
|
def _convert_DictComp(self, n): |
|
c = self._convert |
|
return DictComp(c(n.key), c(n.value), [c(x) for x in n.generators]) |
|
|
|
def _convert_comprehension(self, n): |
|
c = self._convert |
|
return Comprehension(c(n.target), c(n.iter), [c(x) for x in n.ifs]) |
|
|
|
## Statements |
|
|
|
def _convert_Assign(self, n): |
|
c = self._convert |
|
return Assign([c(x) for x in n.targets], c(n.value)) |
|
|
|
def _convert_AugAssign(self, n): |
|
op = n.op.__class__.__name__ |
|
return AugAssign(self._convert(n.target), op, self._convert(n.value)) |
|
|
|
def _convert_AnnAssign(self, n): |
|
if n.value is None: |
|
raise RuntimeError("Cannot convert AnnAssign nodes with no assignment!") |
|
c = self._convert |
|
return Assign([c(n.target)], c(n.value)) |
|
|
|
def _convert_Print(self, n): # pragma: no cover - Python 2.x compat |
|
c = self._convert |
|
if len(n.values) == 1 and isinstance(n.values[0], ast.Tuple): |
|
arg_nodes = [c(x) for x in n.values[0].elts] |
|
else: |
|
arg_nodes = [c(x) for x in n.values] |
|
kwarg_nodes = [] |
|
if n.dest is not None: |
|
kwarg_nodes.append(Keyword('dest', c(n.dest))) |
|
if not n.nl: |
|
kwarg_nodes.append(Keyword('end', Str(''))) |
|
return Expr(Call(Name('print'), arg_nodes, kwarg_nodes)) |
|
|
|
def _convert_Exec(self, n): # pragma: no cover - Python 2.x compat |
|
c = self._convert |
|
arg_nodes = [c(n.body)] |
|
arg_nodes.append(c(n.globals) or NameConstant(None)) |
|
arg_nodes.append(c(n.locals) or NameConstant(None)) |
|
return Expr(Call(Name('exec'), arg_nodes, [])) |
|
|
|
def _convert_Repr(self, n): # pragma: no cover - Python 2.x compat |
|
c = self._convert |
|
return Call(Name('repr'), [c(n.value)], []) |
|
|
|
def _convert_Raise(self, n): |
|
if pyversion < (3, ): |
|
if n.inst or n.tback: |
|
raise RuntimeError('Commonast does not support old raise syntax') |
|
return Raise(self._convert(n.type), None) |
|
return Raise(self._convert(n.exc), self._convert(n.cause)) |
|
|
|
def _convert_Assert(self, n): |
|
return Assert(self._convert(n.test), self._convert(n.msg)) |
|
|
|
def _convert_Delete(self, n): |
|
c = self._convert |
|
return Delete([c(x) for x in n.targets]) |
|
|
|
def _convert_Pass(self, n): |
|
return Pass() |
|
|
|
def _convert_Import(self, n): |
|
return Import(None, [(x.name, x.asname) for x in n.names], 0) |
|
|
|
def _convert_ImportFrom(self, n): |
|
names = [(x.name, x.asname) for x in n.names] |
|
return Import(n.module, names, n.level) |
|
|
|
## Control flow |
|
|
|
def _convert_If(self, n): |
|
c = self._convert |
|
node = If(c(n.test), [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.else_nodes, n.orelse)) |
|
return node |
|
|
|
def _convert_For(self, n): |
|
c = self._convert |
|
node = For(c(n.target), c(n.iter), [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.else_nodes, n.orelse)) |
|
return node |
|
|
|
def _convert_While(self, n): |
|
c = self._convert |
|
node = While(c(n.test), [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.else_nodes, n.orelse)) |
|
return node |
|
|
|
def _convert_Break(self, n): |
|
return Break() |
|
|
|
def _convert_Continue(self, n): |
|
return Continue() |
|
|
|
def _convert_Try(self, n): |
|
c = self._convert |
|
node = Try([], [c(x) for x in n.handlers], [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.else_nodes, n.orelse)) |
|
self._stack.append((node.finally_nodes, n.finalbody)) |
|
return node |
|
|
|
def _convert_TryFinally(self, n): # pragma: no cover - Py <= 3.2 |
|
c = self._convert |
|
if (len(n.body) == 1) and n.body[0].__class__.__name__ == 'TryExcept': |
|
# un-nesting for try-except-finally |
|
n2 = n.body[0] |
|
node = Try([], [c(x) for x in n2.handlers], [], []) |
|
self._stack.append((node.body_nodes, n2.body)) |
|
self._stack.append((node.else_nodes, n2.orelse)) |
|
self._stack.append((node.finally_nodes, n.finalbody)) |
|
else: |
|
node = Try([], [], [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.finally_nodes, n.finalbody)) |
|
return node |
|
|
|
def _convert_TryExcept(self, n): # pragma: no cover - Py <= 3.2 |
|
c = self._convert |
|
node = Try([], [c(x) for x in n.handlers], [], []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
self._stack.append((node.else_nodes, n.orelse)) |
|
return node |
|
|
|
def _convert_ExceptHandler(self, n): |
|
c = self._convert |
|
name = n.name.id if isinstance(n.name, ast.Name) else n.name |
|
node = ExceptHandler(c(n.type), name, []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
return node |
|
|
|
def _convert_With(self, n): |
|
c = self._convert |
|
if hasattr(n, 'items'): |
|
node = With([c(x) for x in n.items], []) |
|
else: # pragma: no cover - Py < 3.3 |
|
items = [WithItem(c(n.context_expr), c(n.optional_vars))] |
|
while (len(n.body) == 1) and isinstance(n.body[0], n.__class__): |
|
n = n.body[0] |
|
items.append(WithItem(c(n.context_expr), c(n.optional_vars))) |
|
node = With(items, []) |
|
self._stack.append((node.body_nodes, n.body)) |
|
return node |
|
|
|
def _convert_withitem(self, n): |
|
return WithItem(self._convert(n.context_expr), self._convert(n.optional_vars)) |
|
|
|
## Function and class definitions |
|
|
|
def _convert_functiondefs(self, n, cls): |
|
c = self._convert |
|
args = n.args |
|
# Parse arg_nodes and kwarg_nodes |
|
arg_nodes = [c(x) for x in args.args] |
|
for i, default in enumerate(reversed(args.defaults)): |
|
arg_node = arg_nodes[-1-i] |
|
if isinstance(arg_node, Tuple): |
|
raise RuntimeError('Tuple arguments in function def not supported.') |
|
arg_node.value_node = c(default) |
|
if pyversion < (3, ): |
|
kwarg_nodes = [] |
|
else: |
|
kwarg_nodes = [c(x) for x in args.kwonlyargs] |
|
for i, default in enumerate(reversed(args.kw_defaults)): |
|
kwarg_nodes[-1-i].value_node = c(default) |
|
# Parse args_node and kwargs_node |
|
if pyversion < (3, ): |
|
args_node = Arg(args.vararg, None, None) if args.vararg else None |
|
kwargs_node = Arg(args.kwarg, None, None) if args.kwarg else None |
|
elif pyversion < (3, 4): |
|
args_node = kwargs_node = None |
|
if args.vararg: |
|
args_node = Arg(args.vararg, None, c(args.varargannotation)) |
|
if args.kwarg: |
|
kwargs_node = Arg(args.kwarg, None, c(args.kwargannotation)) |
|
else: |
|
args_node = c(args.vararg) |
|
kwargs_node = c(args.kwarg) |
|
|
|
returns = None if pyversion < (3, ) else c(n.returns) |
|
Cls = cls # noqa |
|
node = Cls(n.name, [c(x) for x in n.decorator_list], returns, |
|
arg_nodes, kwarg_nodes, args_node, kwargs_node, []) |
|
if docheck: |
|
assert isinstance(node.args_node, (NoneType, Arg)) |
|
assert isinstance(node.kwargs_node, (NoneType, Arg)) |
|
for x in node.arg_nodes + node.kwarg_nodes: |
|
assert isinstance(x, Arg) |
|
|
|
self._stack.append((node.body_nodes, n.body)) |
|
return node |
|
|
|
def _convert_FunctionDef(self, n): |
|
return self._convert_functiondefs(n, FunctionDef) |
|
|
|
def _convert_Lambda(self, n): |
|
c = self._convert |
|
args = n.args |
|
arg_nodes = [c(x) for x in args.args] |
|
for i, default in enumerate(reversed(args.defaults)): |
|
arg_nodes[-1-i].value_node = c(default) |
|
if pyversion < (3, ): |
|
kwarg_nodes = [] |
|
else: |
|
kwarg_nodes = [c(x) for x in args.kwonlyargs] |
|
for i, default in enumerate(reversed(args.kw_defaults)): |
|
kwarg_nodes[-1-i].value_node = c(default) |
|
|
|
return Lambda(arg_nodes, kwarg_nodes, |
|
c(args.vararg), c(args.kwarg), c(n.body)) |
|
|
|
def _convert_AsyncFunctionDef(self, n): |
|
return self._convert_functiondefs(n, AsyncFunctionDef) |
|
|
|
def _convert_arg(self, n): |
|
# Value is initially None |
|
return Arg(n.arg or None, None, self._convert(n.annotation)) |
|
|
|
def _convert_Return(self, n): |
|
return Return(self._convert(n.value)) |
|
|
|
def _convert_Yield(self, n): |
|
return Yield(self._convert(n.value)) |
|
|
|
def _convert_YieldFrom(self, n): |
|
return YieldFrom(self._convert(n.value)) |
|
|
|
def _convert_Await(self, n): |
|
return Await(self._convert(n.value)) |
|
|
|
def _convert_Global(self, n): |
|
return Global(n.names) |
|
|
|
def _convert_Nonlocal(self, n): |
|
return Nonlocal(n.names) |
|
|
|
def _convert_ClassDef(self, n): |
|
c = self._convert |
|
arg_nodes = [c(a) for a in n.bases] |
|
kwarg_nodes = [] if pyversion < (3, ) else [c(a) for a in n.keywords] |
|
|
|
if getattr(n, 'starargs', None): |
|
arg_nodes.append(Starred(self._convert(n.starargs))) |
|
if getattr(n, 'kwargs', None): |
|
kwarg_nodes.append(Keyword(None, self._convert(n.kwargs))) |
|
|
|
node = ClassDef(n.name, [c(a) for a in n.decorator_list], |
|
arg_nodes, kwarg_nodes, []) |
|
|
|
self._stack.append((node.body_nodes, n.body)) |
|
return node
|
|
|