diff --git a/documentation_files/builtindocumentation.py b/documentation_files/builtindocumentation.py index be998a9f..953da543 100644 --- a/documentation_files/builtindocumentation.py +++ b/documentation_files/builtindocumentation.py @@ -1,694 +1,695 @@ # This file denotes the built-in python library. It is imported into every file which is parsed. class Exception(object): pass class object(): def __init__(self): pass def __new__(cls): pass def __del__(self): pass def __repr__(self): pass def __str__(self): pass def __lt__(self, other): pass def __gt__(self, other): pass def __le__(self, other): pass def __eq__(self, other): pass def __ne__(self, other): pass def __gt__(self, other): pass def __ge__(self, other): pass def __cmp__(self, other): pass def __hash__(self): pass def __nonzero__(self): pass def __unicode__(self): pass def __getattr__(self, name): pass def __setattr__(self, name, value): pass def __delattr__(self, name): pass def __getattribute__(self, name): pass def __get__(self, instance, owner): pass def __set__(self, instance, value): pass def __delete__(self, instance): pass def __instancecheck__(self, instance): pass def __subclasscheck__(self, subclass): pass def __call__(self): pass def __len__(self): pass def __getitem__(self, key): pass def __setitem__(self, key, value): pass def __delitem__(self, key): pass def __iter__(self): pass def __reversed__(self): pass def __contains__(self, item): pass def __getslice__(self, i, j): pass def __delslice__(self, i, j): pass def __add__(self, other): pass def __sub__(self, other): pass def __mul__(self, other): pass def __matmul__(self, other): pass def __rmatmul__(self, other): pass def __imatmul__(self, other): pass def __floordiv__(self, other): pass def __mod__(self, other): pass def __divmod__(self, other): pass def __pow__(self, other): pass def __lshift__(self, other): pass def __rshift__(self, other): pass def __and__(self, other): pass def __xor__(self, other): pass def __or__(self, other): pass def __div__(self, other): pass def __truediv__(self, other): pass def __radd__(self, other): pass def __rsub__(self, other): pass def __rmul__(self, other): pass def __rtruediv__(self, other): pass def __rfloordiv__(self, other): pass def __rmod__(self, other): pass def __rdivmod__(self, other): pass def __rpow__(self, other): pass def __rlshift__(self, other): pass def __rrshift__(self, other): pass def __rand__(self, other): pass def __rxor__(self, other): pass def __ror__(self, other): pass def __iadd__(self, other): pass def __isub__(self, other): pass def __imul__(self, other): pass def __idiv__(self, other): pass def __itruediv__(self, other): pass def __ifloordiv__(self, other): pass def __imod__(self, other): pass def __ipow__(self, other): pass def __ilshift__(self, other): pass def __irshift__(self, other): pass def __iand__(self, other): pass def __ixor__(self, other): pass def __ior__(self, other): pass def __neg__(self): pass def __pos__(self): pass def __abs__(self): pass def __invert__(self): pass def __complex__(self): pass def __int__(self): pass def __long__(self): pass def __float__(self): pass def __oct__(self): pass def __hex__(self): pass def __index__(self): pass def __coerce__(self, other): pass __class__ = str() class basestring(): pass class list(): """! TypeContainer !""" def __init__(self, items): """! returnContentEqualsContentOf ! 0""" return [] def __setitem__(self, key, value): """! addsTypeOfArg ! 1 ! getsType !""" def __getitem__(self, key): """! addsTypeOfArg ! 0""" def append(self,obj): """! addsTypeOfArg ! 0""" def extend(self,obj): """! addsTypeOfArgContent ! 0""" return None def insert(self,i, x): """! getsType !""" return None def pop(self,i): """! getsType !""" return None def index(self,x): return 0 def count(self,x): """! getsList !""" return 0 def sort(self,): """! getsList !""" return [] def reverse(self,): """! getsList !""" return [] def remove(self, x): pass class _io_TextIOWrapper(): def close(self,): return None def detach(self): return self # Not quite def flush(self,): return None def fileno(self,): return 0 def isatty(self,): return True def next(self,): return None def read(self,size = 0): return "" def readable(self): return True def readline(self,size = 0): return "" def readlines(self,sizehint = 0): return [""] def seek(self,offset, whence = 0): return None def seekable(self): return True def tell(self,): return 0 def truncate(self,size = 0): return 0 def write(self,string): return None def writable(self): return True def writelines(self,sequence): return None def __iter__(self): return self def __next__(self): return "" + def __enter__(self): return self buffer = _io_TextIOWrapper() # Not quite closed = True encoding = "" errors = None line_buffering = True mode = "" name = "" newlines = "" softspace = True class dict(): """! TypeContainer ! ! hasTypedKeys !""" def __init__(self, items): """! returnContentEqualsContentOf ! 0""" return {} def __setitem__(self, key, value): """! addsTypeOfArg ! 1 ! addsKeyTypeOfArg ! 0""" def __getitem__(self, key): """! getsType !""" def clear(self,): return None def copy(self,): return {} def fromkeys(self,seq, value = None): """! addsKeyTypeOfArgContent ! 0 ! addsTypeOfArg ! 1""" return {} def get(self,key, default = ""): """! getsType !""" return None def has_key(self,key): return True def items(self,): """! getsListOfBoth !""" return {} def iteritems(self,): """! getsListOfBoth !""" return [] def iterkeys(self,): """! getsListOfKeys !""" return [] def itervalues(self,): """! getsList !""" return [] def keys(self,): """! getsListOfKeys !""" return [] def pop(self,key, default = ""): """! getsType !""" return None def popitem(self,): """! getsBoth !""" return None def setdefault(self,key, default = ""): return None def update(self,other = None): return None def values(self,): """! getsList !""" return [] def viewitems(self,): return None def viewkeys(self,): return None def viewvalues(self,): return None class str(): def __init__(self, obj): pass def __mod__(self, modulo): return str() def __getitem__(self): return "" def __iter__(self): return self def __next__(self): return "" def replace(self,before, after): return "" def capitalize(self,): return "" def center(self,width, fillchar = None): return "" def count(self,substring, start = 0, end = 0): return 0 def encode(self, encoding): return bytes() def endswith(self,suffix, start = 0, end = 0): return True def expandtabs(self,tabsize = 0): return "" def find(self,substring, start = 0, end = 0): return 0 def format(self,*args, **kwargs): return "" def index(self,substring, start = 0, end = 0): return 0 def isalnum(self,): return True def isalpha(self,): return True def isdigit(self,): return True def islower(self,): return True def isspace(self,): return True def istitle(self,): return True def isupper(self,): return True def join(self,iterable): return "" def ljust(self,width, fillchar = ""): return "" def lower(self,): return "" def lstrip(self,chars = ""): return "" def partition(self,seperator): return ("", "", "") def replace(self,old, new, count = 0): return "" def rfind(self,substring, start = 0, end = 0): return 0 def rindex(self,substring, start = 0, end = 0): return 0 def rjust(self,width, fillchar = ""): return "" def rpartition(self,seperator): return ("", "", "") def rsplit(self,seperator = "", maxsplit = 0): return [] def rstrip(self,chars = ""): return "" def split(self,seperator = "", maxsplit = 0): return ["string"] def splitlines(self,keepends = False): return ["string"] def startswith(self,prefix, start = 0, end = 0): return True def strip(self,chars = ""): return "" def swapcase(self,): return "" def title(self,): return "" def translate(self,table, deletechars = ""): return "" def upper(self,): return "" def zfill(self,width): return "" class float(): def bit_length(self,): return 0 def as_integer_ration(self,): return (self,0, 0) def is_integer(self,): return True def hex(self,): return 0x0 def fromhex(self,s): return 0 def __add__(self, other): return float() def __sub__(self, other): return float() def __mul__(self, other): return float() def __div__(self, other): return float() class int(): def __add__(self, other): return int() def __sub__(self, other): return int() def __mul__(self, other): return int() def __div__(self, other): return float() class complex(): real = 3 imag = 5 def __add__(self, other): return complex() def __sub__(self, other): return complex() def __mul__(self, other): return complex() def __div__(self, other): return complex() def __mod__(self, other): return complex() class BaseException: args = () __cause__ = BaseException __context__ = BaseException __traceback__ = TracebackType def __init__(self, *args): ... def with_traceback(self, tb): return self class GeneratorExit(BaseException): ... class KeyboardInterrupt(BaseException): ... class SystemExit(BaseException): code = 0 class Exception(BaseException): ... class ArithmeticError(Exception): ... class EnvironmentError(Exception): errno = 0 strerror = "" filename = "" class LookupError(Exception): ... class RuntimeError(Exception): ... class ValueError(Exception): ... class AssertionError(Exception): ... class AttributeError(Exception): ... class BufferError(Exception): ... class EOFError(Exception): ... class FloatingPointError(ArithmeticError): ... class IOError(EnvironmentError): ... class ImportError(Exception): ... class IndexError(LookupError): ... class KeyError(LookupError): ... class MemoryError(Exception): ... class NameError(Exception): ... class NotImplementedError(RuntimeError): ... class OSError(EnvironmentError): ... class BlockingIOError(OSError): characters_written = 0 class ChildProcessError(OSError): ... class ConnectionError(OSError): ... class BrokenPipeError(ConnectionError): ... class ConnectionAbortedError(ConnectionError): ... class ConnectionRefusedError(ConnectionError): ... class ConnectionResetError(ConnectionError): ... class FileExistsError(OSError): ... class FileNotFoundError(OSError): ... class InterruptedError(OSError): ... class IsADirectoryError(OSError): ... class NotADirectoryError(OSError): ... class PermissionError(OSError): ... class ProcessLookupError(OSError): ... class TimeoutError(OSError): ... class WindowsError(OSError): winerror = 0 class OverflowError(ArithmeticError): ... class ReferenceError(Exception): ... class StopIteration(Exception): value = ... # type: Any class StopAsyncIteration(Exception): value = ... # type: Any class RecursionError(RuntimeError): ... class SyntaxError(Exception): msg = "" lineno = 0 offset = 0 text = "" class IndentationError(SyntaxError): ... class TabError(IndentationError): ... class SystemError(Exception): ... class TypeError(Exception): ... class UnboundLocalError(NameError): ... class UnicodeError(ValueError): ... class UnicodeDecodeError(UnicodeError): encoding = "" object = b"" start = 0 end = 0 reason = "" def __init__(self, __encoding: str, __object: bytes, __start: int, __end: int, __reason: str): ... class UnicodeEncodeError(UnicodeError): encoding = "" object = "" start = 0 end = 0 reason = "" def __init__(self, __encoding: str, __object: str, __start: int, __end: int, __reason: str): ... class UnicodeTranslateError(UnicodeError): ... class ZeroDivisionError(ArithmeticError): ... class Warning(Exception): ... class UserWarning(Warning): ... class DeprecationWarning(Warning): ... class SyntaxWarning(Warning): ... class RuntimeWarning(Warning): ... class FutureWarning(Warning): ... class PendingDeprecationWarning(Warning): ... class ImportWarning(Warning): ... class UnicodeWarning(Warning): ... class BytesWarning(Warning): ... class ResourceWarning(Warning): ... class tuple(): """! IndexedTypeContainer !""" def __mul__(self, other): return tuple() class bytes: def __init__(self, data): pass def __getitem__(self, key): return int() def __iter__(self): return self def __next__(self): return int() def capitalize(self): return bytes() def center(self): return bytes() def count(self): return int() def decode(self, encoding): return str() def endswith(self, data): return True def expandtabs(self): return bytes() def find(self): return int() def fromhex(self, hexdata): return bytes() def index(self): return int() def isalnum(self): return True def isalpha(self): return True def isdigit(self): return True def islower(self): return True def isspace(self): return True def istitle(self): return True def isupper(self): return True def join(self, other): return bytes() def ljust(self, space): return bytes(); def lower(self): return bytes() def lstrip(self): return bytes() def maketrans(self, frm, to): return bytes() def partition(self, separator): return (bytes(), bytes(), bytes()) def replace(self, find, replace): return bytes() def rfind(self, data): return int() def rindex(self, data): return int() def rjust(self, justify): return bytes() def rpartition(self, separator): return (bytes(), bytes(), bytes()) def rsplit(self, separator): return [bytes()] def rstrip(self): return bytes() def split(self, separator): return [bytes()] def splitlines(self): return [bytes()] def startswith(self): return False def strip(self): return bytes() def swapcase(self): return bytes() def title(self): return bytes() def translate(self, table, deletechars=None): return bytes() def upper(self): return bytes() def zfill(self, width): return bytes() class set(): """! TypeContainer !""" def __init__(self, iterable): """! returnContentEqualsContentOf ! 0""" pass def len(self): return 0 def isdisjoint(self, other): return True def issubset(self, other): return True def issuperset(self, other): return True def union(self, other, *others): return set() def intersection(self, other, *others): return set() def difference(self, other, *others): return set() def symmetric_difference(self, other): return set() def copy(self): return set() def update(self, other): pass def intersection_update(self, other, *others): pass def difference_update(self, other, *others): pass def symmetric_difference_update(self, other, *others): pass def add(self, elem): pass def remove(self, elem): pass def discard(self, elem): pass def pop(self): pass def clear(self): pass class frozenset(): """! TypeContainer !""" def __init__(self, iterable): """! returnContentEqualsContentOf ! 0""" pass def len(self): return 0 def isdisjoint(self, other): return True def issubset(self, other): return True def issuperset(self, other): return True def union(self, other, *others): return set() def intersection(self, other, *others): return set() def difference(self, other, *others): return set() def symmetric_difference(self, other): return set() def copy(self): return set() def abs(x): """ Return the absolute value of a number. The argument may be a plain or long integer or a floating point number. If the argument is a complex number, its magnitude is returned.""" return 0 def int(x): return 0 def all(iterable): return True def any(iterable): """Return True if any element of the iterable is true. If the iterable is empty, return False.""" return True def bin(x): """Convert an integer number to a binary string. The result is a valid Python expression. If x is not a Python int object, it has to define an __index__() method that returns an integer.""" return "" def bool(x = False): """Convert a value to a Boolean, using the standard truth testing procedure. If x is false or omitted, this returns False; otherwise it returns True. bool is also a class, which is a subclass of int. Class bool cannot be subclassed further. Its only instances are False and True.""" return True def bytearray(source = None, encoding = None, errors = None): """Return a new array of bytes. The bytearray type is a mutable sequence of integers in the range 0 <= x < 256.""" return bytes() def callable(object): """Return True if the object argument appears callable, False if not. If this returns true, it is still possible that a call fails, but if it is false, calling object will never succeed. Note that classes are callable (calling a class returns a new instance); class instances are callable if they have a __call__() method.""" return True def chr(i): """Return a string of one character whose ASCII code is the integer i. For example, chr(97) returns the string 'a'. This is the inverse of ord(). The argument must be in the range [0..255], inclusive; ValueError will be raised if i is outside that range. See also unichr().""" return "" def classmethod(function): """Return a class method for function.""" return None def cmp(x, y): """Compare the two objects x and y and return an integer according to the outcome. The return value is negative if x < y, zero if x == y and strictly positive if x > y.""" return 0 def compile(source, filename, mode, flags = None, dont_inherit = None): """Compile the source into a code or AST object. Code objects can be executed by an exec statement or evaluated by a call to eval(). source can either be a Unicode string, a Latin-1 encoded string or an AST object. Refer to the ast module documentation for information on how to work with AST objects.""" return None def delattr(obj, name): """This is a relative of setattr(). The arguments are an object and a string. The string must be the name of one of the object’s attributes. The function deletes the named attribute, provided the object allows it. For example, delattr(x, 'foobar') is equivalent to del x.foobar.""" return None def dir(obj = None): """Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object.""" return {"string" : None} def divmod(a, b): """Take two (non complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using long division.""" return 0 def enumerate(sequence, start = 0): """Return an enumerate object. sequence must be a sequence, an iterator, or some other object which supports iteration. ! enumerate ! 0 """ return [(0, 0)] def eval(expression, glob = None, loc = None): """The expression argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the globals and locals dictionaries as global and local namespace.""" return None def execfile(filename, glob = None, loc = None): """This function is similar to the exec statement, but parses a file instead of a string. It is different from the import statement in that it does not use the module administration — it reads the file unconditionally and does not create a new module. """ return None def file(filename, mode = None, bufsize = None): """Constructor function for the file type, described further in section File Objects.""" return _io_TextIOWrapper() def filter(function, iterable): """Construct a list from those elements of iterable for which function returns true. iterable may be either a sequence, a container which supports iteration, or an iterator. If iterable is a string or a tuple, the result also has that type; otherwise it is always a list. If function is None, the identity function is assumed, that is, all elements of iterable that are false are removed.""" return [] def float(x = 0): """Convert a string or a number to floating point.""" return 0.0 def format(value, format_spec = None): """Convert a value to a “formatted” representation, as controlled by format_spec.""" return "" def getattr(obj, name, default = None): """Return the value of the named attribute of object. name must be a string.""" return None def globals(): """Return a dictionary representing the current global symbol table.""" return {} def hasattr(obj, name): """The arguments are an object and a string. The result is True if the string is the name of one of the object’s attributes, False if not. (This is implemented by calling getattr(object, name) and seeing whether it raises an exception or not.)""" return bool def hash(obj): """Return the hash value of the object (if it has one).""" return 0 def hex(x): """Convert an integer number (of any size) to a hexadecimal string.""" return 0x0 def id(obj): """Return the “identity” of an object. This is an integer (or long integer) which is guaranteed to be unique and constant for this object during its lifetime. Two objects with non-overlapping lifetimes may have the same id() value.""" return 0 def input(prompt = None): """ Read a string from standard input. The trailing newline is stripped. The prompt string, if given, is printed to standard output without a trailing newline before reading input. If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. On *nix systems, readline is used if available. """ return "" def isinstance(obj, cls): """Return true if the object argument is an instance of the classinfo argument, or of a (direct, indirect or virtual) subclass thereof.""" return True def issubclass(cls, info): """Return true if class is a subclass (direct, indirect or virtual) of classinfo.""" return True def iter(o, s = None): """Return an iterator object. ! returnContentEqualsContentOf ! 0 """ return [] def len(s): """Return the length (the number of items) of an object. The argument may be a sequence (string, tuple or list) or a mapping (dictionary).""" return 0 def locals(): """Update and return a dictionary representing the current local symbol table.""" return {} def long(x = None, base = None): """Convert a string or number to a long integer.""" return 0 def map(func, iterab): """Apply function to every item of iterable and return a list of the results.""" return [] def max(lst, args = None, key = None): """Return the largest item in an iterable or the largest of two or more arguments.""" return 0 def memoryview(obj): """Return a “memory view” object created from the given argument.""" return None def min(lst, default = None): """Return the smallest item in an iterable or the smallest of two or more arguments.""" return 0 def next(iterator, default = None): """Retrieve the next item from the iterator by calling its next() method.""" return iterator[0] def oct(x): """Convert an integer number (of any size) to an octal string.""" return 0o0 def open(filename, mode = None, bufsize = None): """Open a file, returning an object of the file type described in section File Objects.""" return _io_TextIOWrapper() def ord(c): """Given a string of length one, return an integer representing the Unicode code point of the character when the argument is a unicode object, or the value of the byte when the argument is an 8-bit string.""" return 0 def pow(x, y, z = 0): """Return x to the power y; if z is present, return x to the power y, modulo z.""" return 0.0 def property(fget = 0, fset = 0, fdel = 0, doc = 0): """Return a property attribute for new-style classes (classes that derive from object).""" return 0 def range(start = 0, stop = 0, step = 0): """This is a versatile function to create lists containing arithmetic progressions. It is most often used in for loops. The arguments must be plain integers.""" return [0] def reduce(function, iterable, init = None): """Apply function of two arguments cumulatively to the items of iterable, from left to right, so as to reduce the iterable to a single value.""" return None def reload(module): """Reload a previously imported module.""" return None def repr(object): """Return a string containing a printable representation of an object.""" return "" def reversed(seq): """Return a reverse iterator. ! returnContentEqualsContentOf ! 0""" return None def round(x, n=0): """Return the floating point value number rounded to ndigits digits after the decimal point.""" return 0.0 def setattr(obj, name, value): """This is the counterpart of getattr(). The arguments are an object, a string and an arbitrary value. The string may name an existing attribute or a new attribute. The function assigns the value to the attribute, provided the object allows it.""" return None def slice(start = 0, stop = 0, step = 0): """Return a slice object representing the set of indices specified by range(start, stop, step).""" return slice() def sorted(iterable, cmpre = None, key = None, reverse = False): """Return a new sorted list from the items in iterable. ! returnContentEqualsContentOf ! 0""" return [] def staticmethod(function): """Return a static method for function.""" return function def sum(iterable): """Sums start and the items of an iterable from left to right and returns the total.""" return 0.0 def super(_type, obj = None): """Return a proxy object that delegates method calls to a parent or sibling class of type.""" return None def tuple(iterable = None): """Return a tuple whose items are the same and in the same order as iterable‘s items.""" return () def type(object): """With one argument, return the type of an object.""" return object def unichr(i): """Return the Unicode string of one character whose Unicode code is the integer i.""" return "" def unicode(obj = None, encoding = None, errors = None): """Return the Unicode string version of object.""" return "" def vars(obj): """Return the __dict__ attribute for a module, class, instance, or any other object with a __dict__ attribute.""" return None def xrange(start = 0, stop = 0, step = 0): """This function is very similar to range(), but returns an xrange object instead of a list.""" return [0] def zip(iterable = None): """This function returns a list of tuples, where the i-th tuple contains the i-th element from each of the argument sequences or iterables.""" return [] def __import__(name, globa = None, loca = None, fromlist = None, level = 0): """This function is invoked by the import statement. It can be replaced (by importing the __builtin__ module and assigning to __builtin__.__import__) in order to change semantics of the import statement, but nowadays it is usually simpler to use import hooks (see PEP 302).""" return None def exit(status): return None __name__ = "none" __file__ = "none" __doc__ = "none" __package__ = "none" NotImplemented = None def print(obj, sep='', end='\n', file=open()): pass diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index d5ca7124..54fc3cf0 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1879 +1,1893 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/functiondeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/nonetype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, const ReferencedTopDUContext& updateContext_) { ReferencedTopDUContext updateContext(updateContext_); m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); delete prebuilder; qCDebug(KDEV_PYTHON_DUCHAIN) << "Second declarationbuilder pass"; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Prebuilding declarations"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = nullptr; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(name, name), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. if ( currentVariableDefinition->context != ExpressionAst::Context::Store ) { return nullptr; } return visitVariableDeclaration(currentVariableDefinition->identifier, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(nullptr); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { return currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), nullptr, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = nullptr; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != nullptr ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; RangeInRevision range = editorFindRange(node, node); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = nullptr; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, nullptr, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); - visitVariableDeclaration(node->optionalVars, nullptr, v.lastType()); + auto mgrType = v.lastType(); + auto enterType = mgrType; // If we can't find __enter__(), assume it returns `self` like file objects. + + static const IndexedIdentifier enterId(KDevelop::Identifier("__enter__")); + + DUChainReadLocker lock; + if ( auto enterFunc = dynamic_cast( + Helper::accessAttribute(mgrType, enterId, topContext()))) { + if ( auto enterFuncType = enterFunc->type() ) { + enterType = enterFuncType->returnType(); + } + } + lock.unlock(); + // This may be any assignable expression, e.g. `with foo() as bar[3]: ...` + assignToUnknown(node->optionalVars, enterType); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = nullptr; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), nullptr, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return nullptr; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = nullptr; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(nullptr); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(nullptr); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(nullptr); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = nullptr; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = nullptr; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = nullptr; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return nullptr; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = nullptr; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return nullptr; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return nullptr; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, nullptr, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, nullptr); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(nullptr), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. DUChainWriteLocker lock; previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } visitNodeList(node->arguments->defaultValues); if (node->arguments->vararg) { visitVariableDeclaration(node->arguments->vararg->argumentName); } if (node->arguments->kwarg) { visitVariableDeclaration(node->arguments->kwarg->argumentName); } visitNode(node->body); closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker wlock; auto contentType = Helper::contentOfIterable(argVisitor.lastType(), topContext()); container->addContentType(contentType); v.lastDeclaration()->setType(container); } }; auto docstring = function->comment(); if ( ! docstring.isEmpty() ) { foreach ( const auto& key, items.keys() ) { if ( Helper::docstringContainsHint(docstring, key, &args) ) { items[key](); } } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer called) { DUChainReadLocker lock; auto funcInfo = Helper::functionForCalled(called.data()); auto function = funcInfo.declaration; if ( ! function ) { return; } if ( function->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Note: within this function: // - 'parameters' refers to the parameters of the function definition. // - 'arguments' refers to the arguments of the function call. DUContext* parameterContext = DUChainUtils::argumentContext(function); FunctionType::Ptr functionType = function->type(); if ( ! parameterContext || ! functionType ) { return; } QVector parameters = parameterContext->localDeclarations(); if ( parameters.isEmpty() ) { return; } const int specialParamsCount = (function->vararg() != -1) + (function->kwarg() != -1); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfParam = false; if ( ( function->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! function->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as parameters // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfParam = true; } } lock.unlock(); bool explicitSelfArgument = false; if ( hasSelfParam && ! function->isClassMethod() && node->function->astType == Ast::AttributeAstType ) { // Calling an attribute, e.g. `instance.foo(arg)` or `MyClass.foo(instance, arg)`. ExpressionVisitor valueVisitor(currentContext()); valueVisitor.visitNode(static_cast(node->function)->value); if ( valueVisitor.lastDeclaration().dynamicCast() && valueVisitor.isAlias() ) { // Function is attribute of a class _type_ (not instance), so first arg is used as `self`. explicitSelfArgument = true; } } int currentParamIndex = hasSelfParam; int currentArgumentIndex = explicitSelfArgument; int indexInVararg = -1; int paramsAvailable = qMin(functionType->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentArgumentIndex++ ) { atVararg = atVararg || currentParamIndex == function->vararg(); // Not >=, nonexistent vararg is -1. ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(function->vararg()); IndexedContainer::Ptr varargContainer = parameter->type(); if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functionType->removeArgument(currentParamIndex); functionType->addArgument(newType, currentParamIndex); function->setAbstractType(functionType.cast()); parameters.at(currentParamIndex)->setType(newType); currentParamIndex++; } } // **kwargs is always the last parameter MapType::Ptr kwargsDict; if ( function->kwarg() != -1 ) { kwargsDict = parameters.last()->abstractType().cast(); } lock.unlock(); DUChainWriteLocker wlock; foreach ( KeywordAst* keyword, node->keywords ) { wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); bool matchedNamedParam = false; HintedType::Ptr addType = HintedType::Ptr(new HintedType()); if ( keyword->argumentName ) { openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); for (int ip = currentParamIndex; ip < paramsAvailable; ++ip ) { if ( parameters.at(ip)->identifier().toString() != keyword->argumentName->value ) { continue; } matchedNamedParam = true; auto newType = Helper::mergeTypes(parameters.at(ip)->abstractType(), addType); functionType->removeArgument(ip); functionType->addArgument(newType, ip); parameters.at(ip)->setType(newType); } } else if ( auto unpackedDict = argumentVisitor.lastType().cast() ) { // 'keyword is actually an unpacked dict: `foo(**{'a': 12}). openType(addType); addType->setType(unpackedDict->contentType().abstractType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); } else { // Maybe the dict type wasn't loaded yet, or something else happened. continue; } if ( ! matchedNamedParam && kwargsDict ) { DUChainWriteLocker lock; kwargsDict->addContentType(addType); parameters.last()->setAbstractType(kwargsDict); } } function->setAbstractType(functionType); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(target->identifier, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, nullptr, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { DUChainWriteLocker lock; list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(nullptr); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreadyOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 || (starred == -1 && spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } void DeclarationBuilder::visitAnnotationAssignment(AnnotationAssignmentAst* node) { ExpressionVisitor v(currentContext()); v.visitNode(node->target); v.visitNode(node->value); auto assignType = v.lastType(); // Never mind aliasing, why annotate that? v.visitNode(node->annotation); assignType = Helper::mergeTypes(assignType, v.lastType()); assignToUnknown(node->target, assignType); } void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { visitNodeList(node->decorators); visitNodeList(node->baseClasses); const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, NoTypeRequired); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); auto docstring = getDocstring(node->body); dec->setComment(docstring); if ( ! docstring.isEmpty() ) { // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(docstring, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(docstring, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(docstring, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassTypes.append(type); // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); visitNodeList(node->body); lock.lock(); closeContext(); m_currentClassTypes.removeLast(); closeType(); closeDeclaration(); } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); dec->setStatic(false); dec->setClassMethod(false); dec->setProperty(false); foreach ( auto decorator, node->decorators) { visitNode(decorator); switch (decorator->astType) { case Ast::AttributeAstType: { auto attr = static_cast(decorator)->attribute->value; if ( attr == QStringLiteral("setter") || attr == QStringLiteral("getter") || attr == QStringLiteral("deleter") ) dec->setProperty(true); break; } case Ast::NameAstType: { auto name = static_cast(decorator)->identifier->value; if ( name == QStringLiteral("staticmethod") ) dec->setStatic(true); else if ( name == QStringLiteral("classmethod") ) dec->setClassMethod(true); else if ( name == QStringLiteral("property") ) dec->setProperty(true); break; } default: {} } } visitFunctionArguments(node); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new NoneType())); } dec->setType(type); } if ( ! dec->isStatic() ) { DUContext* args = DUChainUtils::argumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! dec->isClassMethod() ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { static auto noneType = AbstractType::Ptr(new NoneType()); if ( auto function = currentType() ) { // Statements with no explicit value return `None`. auto encountered = noneType; if ( node->value ) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); encountered = v.lastType(); } // Update the containing function's return type DUChainWriteLocker lock; function->setReturnType(Helper::mergeTypes(function->returnType(), encountered)); } else { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && workingOnDeclaration->isClassMethod() ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, AliasDeclarationType); if ( ! m_currentClassTypes.isEmpty() ) { auto classDecl = m_currentClassTypes.last()->declaration(topContext()); decl->setAliasedDeclaration(classDecl); } closeDeclaration(); paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } if ( isFirst && ! workingOnDeclaration->isStatic() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassTypes.last().cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, nullptr, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, nullptr, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/tests/pyduchaintest.cpp b/duchain/tests/pyduchaintest.cpp index ef078557..680b2bf0 100644 --- a/duchain/tests/pyduchaintest.cpp +++ b/duchain/tests/pyduchaintest.cpp @@ -1,1803 +1,1821 @@ /***************************************************************************** * Copyright 2010 (c) Miquel Canes Gonzalez * * Copyright 2012 (c) Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include "duchaindebug.h" #include "pyduchaintest.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" #include "kdevpythonversion.h" QTEST_MAIN(PyDUChainTest) using namespace KDevelop; using namespace Python; PyDUChainTest::PyDUChainTest(QObject* parent): QObject(parent) { assetsDir = QDir(DUCHAIN_PY_DATA_DIR); if (!assetsDir.cd("data")) { qFatal("Failed find data directory for test files. Aborting"); } testDir = QDir(testDirOwner.path()); qputenv("PYTHONPATH", assetsDir.absolutePath().toUtf8()); initShell(); } QList PyDUChainTest::FindPyFiles(QDir& rootDir) { QList foundfiles; rootDir.setFilter(QDir::Files | QDir::Dirs | QDir::NoDot | QDir::NoDotDot); rootDir.setNameFilters(QStringList() << "*.py"); // We only want .py files QDirIterator it(rootDir, QDirIterator::Subdirectories); while(it.hasNext()) { foundfiles.append(it.next()); } return foundfiles; } void PyDUChainTest::init() { QString currentTest = QString(QTest::currentTestFunction()); if (lastTest == currentTest) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Already prepared assets for " << currentTest << ", skipping"; return; } else { lastTest = currentTest; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Preparing assets for test " << currentTest; QDir assetModuleDir = QDir(assetsDir.absolutePath()); if (!assetModuleDir.cd(currentTest)) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Asset directory " << currentTest << " does not exist under " << assetModuleDir.absolutePath() << ". Skipping it."; return; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Searching for python files in " << assetModuleDir.absolutePath(); QList foundfiles = FindPyFiles(assetModuleDir); QString correctionFileDir = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files", QStandardPaths::LocateDirectory); auto correctionFileUrl = QUrl(QDir::cleanPath(correctionFileDir + "/testCorrectionFiles/example.py")); foundfiles.prepend(correctionFileUrl.path()); for ( int i = 0; i < 2; i++ ) { // Parse each file twice, to ensure no parsing-order related bugs appear. // Such bugs will need separate unit tests and should not influence these. foreach(const QString filename, foundfiles) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Parsing asset: " << filename; DUChain::self()->updateContextForUrl(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); } foreach(const QString filename, foundfiles) { DUChain::self()->waitForUpdate(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); } while ( ICore::self()->languageController()->backgroundParser()->queuedCount() > 0 ) { // make sure to wait for all parsejobs to finish QTest::qWait(10); } } } void PyDUChainTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); auto doc_url = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); qCDebug(KDEV_PYTHON_DUCHAIN) << doc_url; DUChain::self()->updateContextForUrl(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); DUChain::self()->waitForUpdate(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } ReferencedTopDUContext PyDUChainTest::parse(const QString& code) { TestFile* testfile = new TestFile(code + "\n", "py", nullptr, testDir.absolutePath().append("/")); createdFiles << testfile; testfile->parse((TopDUContext::Features) (TopDUContext::ForceUpdate | TopDUContext::AST) ); testfile->waitForParsed(2000); if ( testfile->isReady() ) { Q_ASSERT(testfile->topContext()); m_ast = static_cast(testfile->topContext()->ast().data())->ast; return testfile->topContext(); } else Q_ASSERT(false && "Timed out waiting for parser results, aborting all tests"); return nullptr; } PyDUChainTest::~PyDUChainTest() { foreach ( TestFile* f, createdFiles ) { delete f; } testDir.rmdir(testDir.absolutePath()); } void PyDUChainTest::testMultiFromImport() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList a = ctx->findDeclarations(QualifiedIdentifier("a")); QList b = ctx->findDeclarations(QualifiedIdentifier("b")); QVERIFY(! a.isEmpty()); QVERIFY(! b.isEmpty()); QVERIFY(a.first()->abstractType()->toString().endsWith("int")); QVERIFY(b.first()->abstractType()->toString().endsWith("int")); } void PyDUChainTest::testMultiFromImport_data() { QTest::addColumn("code"); QTest::newRow("multiimport") << "import testMultiFromImport.i.localvar1\n" "import testMultiFromImport.i.localvar2\n" "a = testMultiFromImport.i.localvar1\n" "b = testMultiFromImport.i.localvar2\n"; } void PyDUChainTest::testRelativeImport() { QFETCH(QString, code); QFETCH(QString, token); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList t = ctx->findDeclarations(QualifiedIdentifier(token)); QVERIFY(! t.isEmpty()); QVERIFY(t.first()->abstractType()->toString().endsWith(type)); } void PyDUChainTest::testRelativeImport_data() { QTest::addColumn("code"); QTest::addColumn("token"); QTest::addColumn("type"); QTest::newRow(".local") << "from testRelativeImport.m.sm1.go import i1" << "i1" << "int"; QTest::newRow(".init") << "from testRelativeImport.m.sm1.go import i2" << "i2" << "int"; QTest::newRow("..local") << "from testRelativeImport.m.sm1.go import i3" << "i3" << "int"; QTest::newRow("..init") << "from testRelativeImport.m.sm1.go import i4" << "i4" << "int"; QTest::newRow("..sub.local") << "from testRelativeImport.m.sm1.go import i5" << "i5" << "int"; QTest::newRow("..sub.init") << "from testRelativeImport.m.sm1.go import i6" << "i6" << "int"; } void PyDUChainTest::testImportFiles() { QString code = "import testImportFiles\nk = testImportFiles.fromInit()\np = testImportFiles.other.fromOther()"; ReferencedTopDUContext ctx = parse(code.toUtf8()); DUChainReadLocker lock; QVERIFY(ctx); auto k = ctx->findDeclarations(QualifiedIdentifier("k")); auto p = ctx->findDeclarations(QualifiedIdentifier("p")); QCOMPARE(k.size(), 1); QCOMPARE(p.size(), 1); QVERIFY(k.first()->abstractType()); QCOMPARE(k.first()->abstractType()->toString(), QString("fromInit")); QCOMPARE(p.first()->abstractType()->toString(), QString("fromOther")); } void PyDUChainTest::testCrashes() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); QVERIFY(! m_ast->body.isEmpty()); } void PyDUChainTest::testCrashes_data() { QTest::addColumn("code"); QTest::newRow("unicode_char") << "a = \"í\""; QTest::newRow("unicode escape char") << "print(\"\\xe9\")"; QTest::newRow("augassign") << "a = 3\na += 5"; QTest::newRow("delete") << "a = 3\ndel a"; QTest::newRow("double_comprehension") << "q = [[x for x in a] + [x for x in a] for y in b]"; QTest::newRow("for_else") << "for i in range(3): pass\nelse: pass"; QTest::newRow("for_while") << "while i < 4: pass\nelse: pass"; QTest::newRow("ellipsis") << "a[...]"; QTest::newRow("tuple_assign_unknown") << "foo = (unknown, unknown, unknown)"; QTest::newRow("for_assign_unknown") << "for foo, bar, baz in unknown: pass"; QTest::newRow("negative slice index") << "t = (1, 2, 3)\nd = t[-1]"; QTest::newRow("decorator_with_args") << "@foo('bar', 'baz')\ndef myfunc(): pass"; QTest::newRow("non_name_decorator") << "@foo.crazy_decorators\ndef myfunc(): pass"; QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass"; QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)"; QTest::newRow("whatever") << "for attr in updated:\n " " getattr.update"; QTest::newRow("return_outside_function") << "return 3"; QTest::newRow("return_context_outside_function") << "return [x for x in range(3)]"; QTest::newRow("paren_attrib_access") << "a = (xxx or yyy).zzz"; QTest::newRow("func_call") << "a = xxx.func(yyy.zzz)"; QTest::newRow("comprehension_attrib") << "a = [foo for foo in bar].baz"; QTest::newRow("comprehension_attrib2") << "a = [foo.bar for foo in bar]"; QTest::newRow("lambda_cmpr_defarg") << "a = lambda foo=[b for b in (1, 2, 3)]: foo"; QTest::newRow("attrib") << "(sep or ' ').join(xxxx.capitalize() for xxxx in ssss.split(sep))"; QTest::newRow("attrib2") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))"; QTest::newRow("attrib3") << "known_threads = {line.strip()}"; QTest::newRow("attrib4") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}"; QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()"; QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n" ".format(type(meth))) from None"; QTest::newRow("newline_attrib") << "some_instance \\\n" ". attr1 \\\n" ".funcfunc(argarg, arg2arg) \\\n" ".foo"; QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n" "in ambiguous_nucleotide_values.iteritems() \\\n" "if set([codon[0] for codon in codons]).issuperset(set(meanings)))"; QTest::newRow("fancy class range") << "class SchemeLexer(RegexLexer):\n" " valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+'\n" "\n" " tokens = {\n" " 'root' : [\n" " # the comments - always starting with semicolon\n" " # and going to the end of the line\n" " (r';.*$', Comment.Single),\n" "\n" " # whitespaces - usually not relevant\n" " (r'\\s+', Text),\n" "\n" " # numbers\n" " (r'-?\\d+\\.\\d+', Number.Float),\n" " (r'-?\\d+', Number.Integer)\n" " ],\n" " }\n"; QTest::newRow("another fancy range") << "setup_args['data_files'] = [\n" " (os.path.dirname(os.path.join(install_base_dir, pattern)),\n" " [ f for f in glob.glob(pattern) ])\n" " for pattern in patterns\n" "]\n"; QTest::newRow("kwarg_empty_crash") << "def myfun(): return\ncheckme = myfun(kw=something)"; QTest::newRow("stacked_tuple_hang") << "tree = (1,(2,(3,(4,(5,'Foo')))))"; QTest::newRow("stacked_tuple_hang2") << "tree = (257," "(264," "(285," "(259," "(272," "(275," "(1, 'return')))))))"; QTest::newRow("very_large_tuple_hang") << "tree = " "(257," "(264," "(285," "(259," "(1, 'def')," "(1, 'f')," "(260, (7, '('), (8, ')'))," "(11, ':')," "(291," "(4, '')," "(5, '')," "(264," "(265," "(266," "(272," "(275," "(1, 'return')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302, (303, (304, (305, (2, '1'))))))))))))))))))," "(264," "(265," "(266," "(272," "(276," "(1, 'yield')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302," "(303, (304, (305, (2, '1'))))))))))))))))))," "(4, '')))," "(6, '')))))," "(4, '')," "(0, ''))))"; QTest::newRow("attribute_hang") << "s = \"123\"\n" "s = s.replace(u'ł', 'l').\\\n" "replace(u'ó', 'o').\\\n" "replace(u'ą', 'a').\\\n" "replace(u'ę', 'e').\\\n" "replace(u'ś', 's').\\\n" "replace(u'ż', 'z').\\\n" "replace(u'ź', 'z').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo')\n"; QTest::newRow("function context range crash") << "def myfunc(arg):\n foo = 3 + \\\n[x for x in range(20)]"; QTest::newRow("decorator comprehension crash") << "@implementer_only(interfaces.ISSLTransport,\n" " *[i for i in implementedBy(tcp.Client)\n" " if i != interfaces.ITLSTransport])\n" "class Client(tcp.Client):\n" " pass\n"; QTest::newRow("comprehension_as_default_crash") << "def foo(bar = [item for (_, item) in items()]):\n return"; QTest::newRow("try_except") << "try: pass\nexcept: pass"; QTest::newRow("try_except_type") << "try: pass\nexcept FooException: pass"; QTest::newRow("try_except_type_as") << "try: pass\nexcept FooException as bar: pass"; QTest::newRow("import_missing") << "from this_does_not_exist import nor_does_this"; QTest::newRow("list_append_missing") << "foo = []\nfoo.append(missing)"; QTest::newRow("list_append_missing_arg") << "foo = []\nfoo.append()"; QTest::newRow("list_extend_missing") << "foo = []\nfoo.extend(missing)"; QTest::newRow("list_extend_missing_arg") << "foo = []\nfoo.extend()"; QTest::newRow("method_of_call_with_list_arg") << "class MyClass:\n" " def bar(self): pass\n" "def foo(x):\n" " return MyClass()\n" "foo([0]).bar()"; QTest::newRow("unpacked_dict_kwarg") << "def foo(arg): pass\nfoo(**{'arg': 2})"; QTest::newRow("negative_container_hints") << "class Evil:\n" " def aa(self, arg):\n" " \"\"\"! addsTypeOfArgContent ! -1\"\"\"\n" " def bb(self, arg):\n" " \"\"\"! addsTypeOfArg ! -2\"\"\"\n" " def cc(self, arg):\n" " \"\"\"! returnContentEqualsContentOf ! -3\"\"\"\n" "e = Evil()\n" "z = [e.aa(1), e.bb(2), e.cc(3)]"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("comprehension_in_fstring") << "def crash(): return f'expr={ {x: y for x, y in [(1, 2), ]}}'"; #endif QTest::newRow("comprehension_in_lambda") << "lambda foo: [bar for bar in foo]"; QTest::newRow("definition_in_baseclass_1") << "class Foo(lambda x: 1): pass"; QTest::newRow("definition_in_baseclass_2") << "class Foo([x for x in (1, 2)]): pass"; } void PyDUChainTest::testClassVariables() { ReferencedTopDUContext ctx = parse("class c():\n myvar = 3;\n def meth(self):\n print(myvar)"); QVERIFY(ctx.data()); DUChainWriteLocker lock(DUChain::lock()); CursorInRevision relevantPosition(3, 10); DUContext* c = ctx->findContextAt(relevantPosition); QVERIFY(c); int useIndex = c->findUseAt(relevantPosition); if ( useIndex != -1 ) { QVERIFY(useIndex < c->usesCount()); const Use* u = &(c->uses()[useIndex]); QVERIFY(!u->usedDeclaration(c->topContext())); } } void PyDUChainTest::testWarnNewNotCls() { QFETCH(QString, code); QFETCH(int, probs); ReferencedTopDUContext ctx = parse(code); DUChainReadLocker lock; QCOMPARE(ctx->problems().count(), probs); } void PyDUChainTest::testWarnNewNotCls_data() { QTest::addColumn("code"); QTest::addColumn("probs"); QTest::newRow("check_for_new_first_arg_cls") << "class c():\n def __new__(clf, other):\n pass" << 1; QTest::newRow("check_for_new_first_arg_cls_0") << "class c():\n def __new__(cls, other):\n pass" << 0; QTest::newRow("check_first_arg_class_self") << "class c():\n def test(seff, masik):\n pass" << 1; QTest::newRow("check_first_arg_class_self_0") << "class c():\n def test(self, masik):\n pass" << 0; } // this is actually for both binary and boolean operators void PyDUChainTest::testBinaryOperatorsUnsure() { QFETCH(QString, code); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), type); } void PyDUChainTest::testBinaryOperatorsUnsure_data() { QTest::addColumn("code"); QTest::addColumn("type"); QTest::newRow("check_unsure_type_0") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3.5;\ny = 3;\ncheckme = x * y;" << "unsure (float, int)"; QTest::newRow("check_unsure_type_1") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_2") << "class c():\n pass;\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_3") << "class c():\n pass;\nclass d():\n pass;\nx = c();\nx = d();\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_4") << "checkme = True or False" << "bool"; QTest::newRow("check_unsure_type_5") << "a = 'foo'; checkme = a or 'bar';" << "str"; QTest::newRow("check_unsure_type_6") << "class A(): pass\nclass B(): pass;\ncheckme = A() or B() or 42;" << "unsure (A, B, int)"; } void PyDUChainTest::testFlickering() { QFETCH(QStringList, code); QFETCH(int, before); QFETCH(int, after); TestFile f(code[0], "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx = f.topContext(); QVERIFY(ctx); DUChainWriteLocker lock(DUChain::lock()); int count = ctx->localDeclarations().size(); qDebug() << "Declaration count before: " << count; QVERIFY(count == before); lock.unlock(); f.setFileContents(code[1]); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ctx = f.topContext(); QVERIFY(ctx); lock.lock(); count = ctx->localDeclarations().size(); qDebug() << "Declaration count afterwards: " << count; QVERIFY(count == after); foreach(Declaration* dec, ctx->localDeclarations()) { qDebug() << dec->toString() << dec->range(); qDebug() << dec->uses().size(); } } void PyDUChainTest::testFlickering_data() { QTest::addColumn("code"); QTest::addColumn("before"); QTest::addColumn("after"); QTest::newRow("declaration_flicker") << ( QStringList() << "a=2\n" << "b=3\na=2\n" ) << 1 << 2; } void PyDUChainTest::testCannotOverwriteBuiltins() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testCannotOverwriteBuiltins_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("list_assign") << "class list(): pass\ncheckme = []\ncheckme.append(3)" << "list of int"; QTest::newRow("str_assign") << "str = 5; checkme = 'Foo'" << "str"; QTest::newRow("str_assign2") << "class Foo: pass\nstr = Foo; checkme = 'Foo'" << "str"; QTest::newRow("str_assign3") << "from testCannotOverwriteBuiltins.i import Foo as str\ncheckme = 'Foo'" << "str"; QTest::newRow("for") << "for str in [1, 2, 3]: pass\ncheckme = 'Foo'" << "str"; QTest::newRow("assert") << "assert isinstance(str, int)\ncheckme = 'Foo'" << "str"; QTest::newRow("assert2") << "assert isinstance(str, int)\ncheckme = 3" << "int"; QTest::newRow("can_have_custom") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open()" << "int"; QTest::newRow("can_have_custom2") << "from testCannotOverwriteBuiltins import mod\ncheckme = open().read()" << "str"; QTest::newRow("can_have_custom3") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open().read()" << "mixed"; } void PyDUChainTest::testVarKWArgs() { ReferencedTopDUContext ctx = parse("def myfun(arg, *vararg, **kwarg):\n pass\n pass"); DUChainWriteLocker lock; QVERIFY(ctx); DUContext* func = ctx->findContextAt(CursorInRevision(1, 0)); QVERIFY(func); QVERIFY(! func->findDeclarations(QualifiedIdentifier("arg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("vararg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("kwarg")).isEmpty()); QVERIFY(func->findDeclarations(QualifiedIdentifier("vararg")).first()->abstractType()->toString().startsWith("tuple")); QCOMPARE(func->findDeclarations(QualifiedIdentifier("kwarg")).first()->abstractType()->toString(), QString("dict of str : unknown")); } void PyDUChainTest::testSimple() { QFETCH(QString, code); QFETCH(int, decls); QFETCH(int, uses); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVector< Declaration* > declarations = ctx->localDeclarations(); QCOMPARE(declarations.size(), decls); int usesCount = 0; foreach(Declaration* d, declarations) { usesCount += d->uses().size(); QVERIFY(d->abstractType()); } QCOMPARE(usesCount, uses); } void PyDUChainTest::testSimple_data() { QTest::addColumn("code"); QTest::addColumn("decls"); QTest::addColumn("uses"); QTest::newRow("assign") << "b = 2;" << 1 << 0; QTest::newRow("assign_str") << "b = 'hola';" << 1 << 0; QTest::newRow("op") << "a = 3; b = a+2;" << 2 << 1; QTest::newRow("bool") << "a = True" << 1 << 0; QTest::newRow("op") << "a = True and True;" << 1 << 0; } class AttributeRangeTestVisitor : public AstDefaultVisitor { public: bool found; KTextEditor::Range searchingForRange; QString searchingForIdentifier; void visitAttribute(AttributeAst* node) override { auto r = KTextEditor::Range(0, node->startCol, 0, node->endCol); qDebug() << "Found attr: " << r << node->attribute->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->attribute->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitAttribute(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found func: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; qDebug() << node->arguments->vararg << node->arguments->kwarg; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } if ( node->arguments->vararg ) { auto r = KTextEditor::Range(0, node->arguments->vararg->startCol, 0, node->arguments->vararg->startCol+node->arguments->vararg->argumentName->value.length()); qDebug() << "Found vararg: " << node->arguments->vararg->argumentName->value << r; if ( r == searchingForRange && node->arguments->vararg->argumentName->value == searchingForIdentifier ) { found = true; return; } } if ( node->arguments->kwarg ) { auto r = KTextEditor::Range(0, node->arguments->kwarg->startCol, 0, node->arguments->kwarg->startCol+node->arguments->kwarg->argumentName->value.length()); qDebug() << "Found kwarg: " << node->arguments->kwarg->argumentName->value << r; if ( r == searchingForRange && node->arguments->kwarg->argumentName->value == searchingForIdentifier ) { found = true; return; } } AstDefaultVisitor::visitFunctionDefinition(node); } void visitClassDefinition(ClassDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found cls: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitClassDefinition(node); } void visitImport(ImportAst* node) override { foreach ( const AliasAst* name, node->names ) { if ( name->name ) { qDebug() << "found import" << name->name->value << name->name->range(); } if ( name->name && name->name->value == searchingForIdentifier && name->name->range() == searchingForRange ) { found = true; return; } if ( name->asName ) { qDebug() << "found import" << name->asName->value << name->asName->range(); } if ( name->asName && name->asName->value == searchingForIdentifier && name->asName->range() == searchingForRange ) { found = true; return; } } } }; void PyDUChainTest::testRanges() { QFETCH(QString, code); QFETCH(int, expected_amount_of_variables); Q_UNUSED(expected_amount_of_variables); QFETCH(QStringList, column_ranges); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); for ( int i = 0; i < column_ranges.length(); i++ ) { int scol = column_ranges.at(i).split(",")[0].toInt(); int ecol = column_ranges.at(i).split(",")[1].toInt(); QString identifier = column_ranges.at(i).split(",")[2]; auto r = KTextEditor::Range(0, scol, 0, ecol); AttributeRangeTestVisitor* visitor = new AttributeRangeTestVisitor(); visitor->searchingForRange = r; visitor->searchingForIdentifier = identifier; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("attr_dot_name_hash", "Insufficiently magic hack", Continue); QCOMPARE(visitor->found, true); delete visitor; } } void PyDUChainTest::testRanges_data() { QTest::addColumn("code"); QTest::addColumn("expected_amount_of_variables"); QTest::addColumn("column_ranges"); QTest::newRow("attr_two_attributes") << "base.attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_binary") << "base.attr + base.attr" << 4 << ( QStringList() << "5,8,attr" << "17,20,attr" ); QTest::newRow("attr_same") << "aaa.aaa.aaa + aaa.aaa.aaa" << 6 << ( QStringList() << "4,6,aaa" << "8,10,aaa" << "18,20,aaa" << "22,24,aaa" ); QTest::newRow("attr_three_attributes") << "base.attr.subattr" << 3 << ( QStringList() << "5,8,attr" << "10,16,subattr" ); QTest::newRow("attr_functionCall") << "base.attr().subattr" << 3 << ( QStringList() << "5,8,attr" << "12,18,subattr" ); QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" ); QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" ); QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" ); QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" << "25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr"); QTest::newRow("attr_two_in_call") << "func(inst.aaa, inst.bbbb)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,bbbb" ); QTest::newRow("attr_two_in_call_same") << "func(inst.aaa, inst.aaaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,aaaa" ); QTest::newRow("attr_two_in_call_same2") << "func(inst.aaa, inst.aaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,22,aaa" ); QTest::newRow("attr_of_string_slash") << "'/'.join(a)" << 1 << ( QStringList() << "4,7,join" ); QTest::newRow("attr_of_string_in_list") << "[\"*{0}*\".format(foo)]" << 1 << ( QStringList() << "9,14,format" ); QTest::newRow("attr_of_call_in_list") << "[foo().format(foo)]" << 1 << ( QStringList() << "7,12,format" ); QTest::newRow("attr_parentheses") << "(\"foo\" + \"foo\").capitalize()" << 1 << ( QStringList() << "16,25,capitalize" ); QTest::newRow("attr_commented_name") << "base.attr # attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_name_in_strings") << "'attr' + base['attr'].attr # attr" << 4 << ( QStringList() << "22,25,attr" ); QTest::newRow("attr_dot_hash_in_strings") << "'.foo#' + base['.#'].attr # attr" << 4 << ( QStringList() << "21,24,attr" ); QTest::newRow("attr_dot_name_hash") << "base['.attr#'].attr" << 4 << ( QStringList() << "15,18,attr" ); QTest::newRow("string_parentheses") << "(\"asdf\".join())" << 1 << ( QStringList() << "8,11,join" ); QTest::newRow("string_parentheses2") << "(\"asdf\").join()" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("string_parentheses3") << "(\"asdf\".join()).join()" << 2 << ( QStringList() << "8,11,join" << "16,19,join" ); QTest::newRow("string_parentheses4") << "(\"asdf\".join()+2).join()" << 2 << ( QStringList() << "8,11,join" << "18,21,join" ); QTest::newRow("string_parentheses_call") << "f(\"asdf\".join())" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" ); QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" ); QTest::newRow("classdef_range") << "class cls(): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance") << "class cls(parent1, parent2): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance_spaces") << "class cls( parent1, parent2 ):pass" << 1 << ( QStringList() << "12,14,cls" ); QTest::newRow("vararg_kwarg") << "def func(*vararg, **kwargs): pass" << 2 << ( QStringList() << "10,16,vararg" << "20,26,kwargs" ); QTest::newRow("import") << "import sys" << 1 << ( QStringList() << "7,10,sys" ); QTest::newRow("import2") << "import i.localvar1" << 1 << ( QStringList() << "7,18,i.localvar1" ); QTest::newRow("import3") << "import sys as a" << 1 << ( QStringList() << "13,14,a" ); } class TypeTestVisitor : public AstDefaultVisitor { public: QString searchingForType; TopDUContextPointer ctx; bool found; void visitName(NameAst* node) override { if ( node->identifier->value != "checkme" ) return; QList decls = ctx->findDeclarations(QualifiedIdentifier(node->identifier->value)); if ( ! decls.length() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration found for " << node->identifier->value; return; } Declaration* d = decls.last(); QVERIFY(d->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << "found: " << node->identifier->value << "is" << d->abstractType()->toString() << "should be" << searchingForType; if ( d->abstractType()->toString().replace("__kdevpythondocumentation_builtin_", "").startsWith(searchingForType) ) { found = true; return; } }; }; void PyDUChainTest::testTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("tuple_func", "no suitable docstring hint", Continue); QEXPECT_FAIL("tuple_add", "not implemented", Continue); QEXPECT_FAIL("tuple_mul", "not implemented", Continue); QEXPECT_FAIL("return_builtin_iterator", "fake builtin iter()", Continue); QEXPECT_FAIL("parent_constructor_arg_type", "Not enough passes?", Continue); QEXPECT_FAIL("init_class_no_decl", "aliasing info lost", Continue); QEXPECT_FAIL("property_wrong", "visitCall uses declaration if no type", Continue); QEXPECT_FAIL("property_setter", "very basic property support", Continue); QCOMPARE(visitor->found, true); } void PyDUChainTest::testTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("annotate_decl") << "checkme: int" << "int"; QTest::newRow("annotate_assign") << "checkme: int = 3.5" << "unsure (float, int)"; #endif QTest::newRow("listtype") << "checkme = []" << "list"; QTest::newRow("listtype_func") << "checkme = list()" << "list"; QTest::newRow("listtype_with_contents") << "checkme = [1, 2, 3, 4, 5]" << "list of int"; QTest::newRow("listtype_extended") << "some_misc_var = []; checkme = some_misc_var" << "list"; QTest::newRow("dicttype") << "checkme = {}" << "dict"; QTest::newRow("dicttype_get") << "d = {0.4:5}; checkme = d.get(0)" << "int"; QTest::newRow("dicttype_func") << "checkme = dict()" << "dict"; QTest::newRow("dicttype_extended") << "some_misc_var = {}; checkme = some_misc_var" << "dict"; QTest::newRow("tuple") << "checkme = ()" << "tuple of ()"; QTest::newRow("tuple_func") << "checkme = tuple((1, 2.3))" << "tuple of (int, float)"; QTest::newRow("tuple_with_contents") << "checkme = 1, 2.3" << "tuple of (int, float)"; QTest::newRow("tuple_extended") << "some_misc_var = (); checkme = some_misc_var" << "tuple of ()"; QTest::newRow("tuple_max_display") << "checkme = 1,2,3,4,5,6" << "tuple of (int, int, int, int, int, ...)"; QTest::newRow("bool") << "checkme = True" << "bool"; QTest::newRow("float") << "checkme = 3.7" << "float"; QTest::newRow("int") << "checkme = 3" << "int"; QTest::newRow("str") << "checkme = \"foo\"" << "str"; QTest::newRow("bytes") << "checkme = b\"foo\"" << "bytes"; QTest::newRow("function_arg_scope") << "class Foo:\n" " a = 3\n" " def func(self, x=a):\n" " return x\n" "f = Foo()\n" "checkme = f.func()" << "int"; + QTest::newRow("with") << "with open('foo') as f: checkme = f.read()" << "str"; + QTest::newRow("with_list_target") << "bar = [1, 2, 3]\n" + "with open('foo') as bar[1]: checkme = bar[1].read()" << "str"; + QTest::newRow("with_attr_target") << "bar = object()\n" + "with open('foo') as bar.zep: checkme = bar.zep.read()" << "str"; + QTest::newRow("with_nonself_enter") << // From https://bugs.kde.org/show_bug.cgi?id=399534 + "class Mgr:\n" + " def __enter__(self): return 42\n" + " def __exit__(self, *args): pass\n" + "with Mgr() as asd:\n" + " checkme = asd" << "int"; + QTest::newRow("with_tuple_target") << + "class Mgr:\n" + " def __enter__(self): return (42, 3.4)\n" + " def __exit__(self, *args): pass\n" + "with Mgr() as (aa, bb):\n" + " checkme = bb" << "float"; + QTest::newRow("arg_after_vararg") << "def func(x, y, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("arg_after_vararg_with_default") << "def func(x=5, y=3, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("class_scope_end_inside") << "a = str()\nclass M:\n" " a = 2\n foo = a\n" "checkme = M().foo" << "int"; QTest::newRow("class_scope_end_outside") << "a = str()\nclass M:\n a = 2\ncheckme = a" << "str"; QTest::newRow("list_access_right_open_slice") << "some_list = []; checkme = some_list[2:]" << "list"; QTest::newRow("list_access_left_open_slice") << "some_list = []; checkme = some_list[:2]" << "list"; QTest::newRow("list_access_closed_slice") << "some_list = []; checkme = some_list[2:17]" << "list"; QTest::newRow("list_access_step") << "some_list = []; checkme = some_list[::2]" << "list"; QTest::newRow("list_access_singleItem") << "some_list = []; checkme = some_list[42]" << "mixed"; QTest::newRow("funccall_number") << "def foo(): return 3; \ncheckme = foo();" << "int"; QTest::newRow("funccall_string") << "def foo(): return 'a'; \ncheckme = foo();" << "str"; QTest::newRow("funccall_list") << "def foo(): return []; \ncheckme = foo();" << "list"; QTest::newRow("funccall_dict") << "def foo(): return {}; \ncheckme = foo();" << "dict"; QTest::newRow("funccall_no_return") << "def foo(): pass\ncheckme = foo()" << "None"; QTest::newRow("funccall_def_return") << "def foo(): return\ncheckme = foo()" << "None"; QTest::newRow("funccall_maybe_def_return") << "def foo():\n if False: return\n return 7\ncheckme = foo()" << "unsure (None, int)"; QTest::newRow("tuple1") << "checkme, foo = 3, \"str\"" << "int"; QTest::newRow("tuple2") << "foo, checkme = 3, \"str\"" << "str"; QTest::newRow("tuple2_negative_index") << "foo = (1, 2, 'foo')\ncheckme = foo[-1]" << "str"; QTest::newRow("tuple_type") << "checkme = 1, 2" << "tuple"; QTest::newRow("tuple_rhs_unpack") << "foo = 1, 2.5\nbar = 1, *foo, 2\ncheckme = bar[2]" << "float"; QTest::newRow("dict_iteritems") << "d = {1:2, 3:4}\nfor checkme, k in d.iteritems(): pass" << "int"; QTest::newRow("enumerate_key") << "d = [str(), str()]\nfor checkme, value in enumerate(d): pass" << "int"; QTest::newRow("enumerate_value") << "d = [str(), str()]\nfor key, checkme in enumerate(d): pass" << "str"; QTest::newRow("dict_enumerate") << "d = {1:2, 3:4}\nfor key, checkme in enumerate(d.values()): pass" << "int"; QTest::newRow("dict_assign_twice") << "d = dict(); d[''] = 0; d = dict(); d[''] = 0; checkme = d" << "unsure (dict of str : int, dict)"; QTest::newRow("class_method_import") << "class c:\n attr = \"foo\"\n def m():\n return attr;\n return 3;\ni=c()\ncheckme=i.m()" << "int"; QTest::newRow("getsListDocstring") << "foo = [1, 2, 3]\ncheckme = foo.reverse()" << "list of int"; QTest::newRow("str_iter") << "checkme = [char for char in 'Hello, world!']" << "list of str"; QTest::newRow("str_subscript") << "checkme = 'Hello, world!'[0]" << "str"; QTest::newRow("bytes_iter") << "checkme = [byte for byte in b'Hello, world!']" << "list of int"; QTest::newRow("bytes_subscript") << "checkme = b'Hello, world!'[0]" << "int"; QTest::newRow("fromAssertIsinstance") << "class c(): pass\ncheckme = mixed()\nassert isinstance(checkme, c)\n" << "c"; QTest::newRow("fromAssertIsinstanceInvalid") << "class c(): pass\ncheckme = mixed()\nassert isinstance(c, checkme)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid2") << "class c(): pass\ncheckme = mixed()\nassert isinstance(D, c)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid3") << "checkme = int()\nassert isinstance(checkme, X)\n" << "int"; QTest::newRow("fromAssertIsinstanceInvalid4") << "checkme = int()\nassert isinstance(checkme)\n" << "int"; QTest::newRow("fromAssertType") << "class c(): pass\ncheckme = mixed()\nassert type(checkme) == c\n" << "c"; QTest::newRow("fromIfType") << "class c(): pass\ncheckme = mixed()\nif type(checkme) == c: pass\n" << "c"; QTest::newRow("fromIfIsinstance") << "class c(): pass\ncheckme = mixed()\nif isinstance(checkme, c): pass\n" << "c"; QTest::newRow("diff_local_classattr") << "class c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; QTest::newRow("diff_local_classattr2") << "local=3\nclass c(): attr = 1\ninst=c()\ncheckme = c.local" << "mixed"; QTest::newRow("diff_local_classattr3") << "attr=3.5\nclass c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; // QTest::newRow("class_method_self") << "class c:\n def func(checkme, arg, arg2):\n pass\n" << "c"; // QTest::newRow("funccall_dict") << "def foo(): return foo; checkme = foo();" << (uint) IntegralType::TypeFunction; // With only one subbed value we get a FormattedValue node QTest::newRow("fstring_formattedvalue") << "name = 'Jim'; checkme = f'{name}'" << "str"; // Otherwise a JoinedString, with FormattedValues as children. QTest::newRow("fstring_joinedstring") << "name = 'Jim'; checkme = f'Hello, {name}! Your name is {name}.'" << "str"; QTest::newRow("tuple_simple") << "mytuple = 3, 5.5\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_simple2") << "mytuple = 3, 5.5\nfoobar, checkme = mytuple" << "float"; QTest::newRow("tuple_simple3") << "mytuple = 3, 5.5, \"str\", 3, \"str\"\na, b, c, d, checkme = mytuple" << "str"; QTest::newRow("tuple_single") << "checkme = 4," << "tuple"; QTest::newRow("tuple_single2") << "checkme, = 4," << "int"; QTest::newRow("tuple_single3") << "mytuple = 4,\ncheckme, = mytuple" << "int"; QTest::newRow("tuple_ext_unpack") << "mytuple = 3, 5.5\nfoobar, *starred, checkme = mytuple" << "float"; QTest::newRow("tuple_ext_unpack2") << "mytuple = 3, 5.5\nfoobar, *checkme, another = mytuple" << "list"; QTest::newRow("tuple_ext_unpack3") << "mytuple = 3, 5.5\nfoobar, *checkme = mytuple" << "list of float"; QTest::newRow("tuple_ext_unpack4") << "mytuple = 3, 5.5\n*checkme, = mytuple" << "list of unsure (int, float)"; QTest::newRow("tuple_nested") << "mytuple = 3, ('foo', 5.5)\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_nested2") << "mytuple = 3, ('foo', 5.5)\nfoobar, (checkme, other) = mytuple" << "str"; QTest::newRow("tuple_nested3") << "mytuple = ((7, 'foo'), 5.5), 3\n((baz, checkme), other), foo = mytuple" << "str"; QTest::newRow("tuple_nested_ext") << "mytuple = (2, ('foo', 'bar', 6), 7)\na, (b, *checkme, c), *d = mytuple" << "list of str"; QTest::newRow("tuple_multi_assign") << "mytuple = 2, 'foo'\ncheckme = a = mytuple" << "tuple"; QTest::newRow("tuple_multi_assign2") << "mytuple = 2, 'foo'\ncheckme, a = b = mytuple" << "int"; QTest::newRow("list_unpack") << "mylist = [1, 2, 3]\ncheckme, b, c = mylist" << "int"; QTest::newRow("list_unpack2") << "mylist = [1, 'x', 3]\ncheckme, b, c = mylist" << "unsure (int, str)"; QTest::newRow("list_ext_unpack") << "mylist = [1, 2, 3]\n*checkme, foo = mylist" << "list of int"; QTest::newRow("list_ext_unpack2") << "mylist = [1, 'x', 3]\n*checkme, foo = mylist" << "list of unsure (int, str)"; QTest::newRow("if_expr_sure") << "checkme = 3 if 7 > 9 else 5" << "int"; QTest::newRow("unary_op") << "checkme = -42" << "int"; QTest::newRow("tuple_funcret") << "def myfun(): return 3, 5\ncheckme, a = myfun()" << "int"; QTest::newRow("tuple_funcret2") << "def myfun():\n t = 3, 5\n return t\ncheckme, a = myfun()" << "int"; QTest::newRow("yield") << "def myfun():\n yield 3\ncheckme = myfun()" << "list of int"; QTest::newRow("yield_twice") << "def myfun():\n yield 3\n yield 'foo'\ncheckme = myfun()" << "list of unsure (int, str)"; // this is mostly a check that it doesn't crash QTest::newRow("yield_return") << "def myfun():\n return 3\n yield 'foo'\ncheckme = myfun()" << "unsure (int, list of str)"; QTest::newRow("lambda") << "x = lambda t: 3\ncheckme = x()" << "int"; QTest::newRow("lambda_failure") << "x = lambda t: 3\ncheckme = t" << "mixed"; QTest::newRow("function_arg_tuple") << "def func(*arg):\n foo, bar = arg\n return bar\ncheckme = func(3, 5)" << "int"; QTest::newRow("function_arg_tuple2") << "def func(*arg):\n return arg[-1]\ncheckme = func(3, \"Foo\")" << "str"; QTest::newRow("tuple_indexaccess") << "t = 3, 5.5\ncheckme = t[0]" << "int"; QTest::newRow("tuple_indexaccess2") << "t = 3, 5.5\ncheckme = t[1]" << "float"; QTest::newRow("tuple_indexaccess3") << "t = 3, 4\ncheckme = t[1]" << "int"; QTest::newRow("tuple_indexaccess4") << "t = 3, 4.5\ncheckme = t[2]" << "unsure (int, float)"; QTest::newRow("tuple_indexaccess_neg") << "t = 3, 4.5; checkme = t[-1]" << "float"; QTest::newRow("tuple_indexaccess_neg2") << "t = 3, 4.5; checkme = t[-2]" << "int"; QTest::newRow("tuple_indexaccess_neg3") << "t = 3, 4.5; checkme = t[-3]" << "unsure (int, float)"; QTest::newRow("tuple_slice") << "t = 3, 'q', 4.5; checkme = t[-3: 2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_normal") << "t = 1, 2.3, 'a', {}; checkme = t[1:3]" << "tuple of (float, str)"; QTest::newRow("tuple_slice_defstart") << "t = 1, 2.3, 'a', {}; checkme = t[:3]" << "tuple of (int, float, str)"; QTest::newRow("tuple_slice_defstop") << "t = 1, 2.3, 'a', {}; checkme = t[1:]" << "tuple of (float, str, dict)"; QTest::newRow("tuple_slice_defboth") << "t = 1, 2.3, 'a', {}; checkme = t[:]" << "tuple of (int, float, str, dict)"; QTest::newRow("tuple_slice_step") << "t = 1, 2.3, 'a', {}; checkme = t[0:3:2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_reverse") << "t = 1, 2.3, 'a', {}; checkme = t[3:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstart") << "t = 1, 2.3, 'a', {}; checkme = t[:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[2::-1]" << "tuple of (str, float, int)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[::-1]" << "tuple of (dict, str, float, int)"; QTest::newRow("tuple_slice_no_elems") << "t = 1, 2.3, 'a', {}; checkme = t[1:1]" << "tuple of ()"; // TODO unsure-tuples. QTest::newRow("tuple_slice_not_literal") << "n = 2; t = 1, 2.3, 'a', {}; checkme = t[0:n]" << "tuple of ()"; // These are allowed, for whatever reason. QTest::newRow("tuple_slice_past_range") << "t = 1, 2.3; checkme = t[-999999999:8888888888]" << "tuple of (int, float)"; QTest::newRow("tuple_slice_wrong_direction") << "t = 1, 2.3, 'a'; checkme = t[0:3:-1]" << "tuple of ()"; // This isn't. QTest::newRow("tuple_slice_zero_step") << "t = 1, 2.3; checkme = t[::0]" << "tuple of ()"; QTest::newRow("tuple_add") << "t, u = (3,), ('q', 4.5); checkme = t + u" << "tuple of (int, str, float)"; QTest::newRow("tuple_mul") << "t = 3, 4.5; checkme = t * 2" << "tuple of (int, float, int, float)"; QTest::newRow("dict_unsure") << "t = dict(); t = {3: str()}\ncheckme = t[1].capitalize()" << "str"; QTest::newRow("unsure_attr_access") << "d = str(); d = 3; checkme = d.capitalize()" << "str"; QTest::newRow("class_create_var") << "class c: pass\nd = c()\nd.foo = 3\ncheckme = d.foo" << "int"; QTest::newRow("tuple_loop") << "t = [(1, \"str\")]\nfor checkme, a in t: pass" << "int"; QTest::newRow("no_hints_type") << "def myfun(arg): arg = 3; return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("hints_type") << "def myfun(arg): return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("args_type") << "def myfun(*args): return args[0]\ncheckme = myfun(3)" << "int"; QTest::newRow("kwarg_type") << "def myfun(**kwargs): return kwargs['a']\ncheckme = myfun(a=3)" << "int"; QTest::newRow("dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\ncheckme = foo(**{'a': 12})" << "int"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("dict_norm_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2)" << "unsure (int, float)"; QTest::newRow("multi_dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2, **{'c': ''})" << "unsure (int, float, str)"; #endif QTest::newRow("named_arg_type") << "def myfun(arg): return arg\ncheckme = myfun(arg=3)" << "int"; QTest::newRow("arg_args_type") << "def myfun(arg, *args): return args[0]\n" "checkme = myfun(3, str())" << "str"; QTest::newRow("arg_kwargs_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(12, a=str())" << "str"; QTest::newRow("named_kwargs_type_1") << "def myfun(arg, **kwargs): return arg\n" "checkme = myfun(arg=12, a=str())" << "int"; QTest::newRow("named_kwargs_type_2") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(arg=12, a=str())" << "str"; QTest::newRow("kwargs_named_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(a=str(), arg=12)" << "str"; QTest::newRow("varied_args_type_1") << "def myfun(arg, *args, **kwargs): return arg\n" "checkme = myfun(1, 1.5, a=str())" << "int"; QTest::newRow("varied_args_type_2") << "def myfun(arg, *args, **kwargs): return args[0]\n" "checkme = myfun(1, 1.5, a=str())" << "float"; QTest::newRow("varied_args_type_3") << "def myfun(arg, *args, **kwargs): return kwargs['a']\n" "checkme = myfun(1, 1.5, a=str())" << "str"; QTest::newRow("nested_arg_name_type") << "def foo(xxx):\n" " def bar(xxx): return xxx\n" " return bar('test')\n" "checkme = foo(10)\n" << "str"; QTest::newRow("method_args_type_1") << "class MyClass:\n" " def method(self, arg): return self\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("method_args_type_2") << "class MyClass:\n" " def method(self, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("clsmethod_args_type_1") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return cls\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("clsmethod_args_type_2") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_args_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_vararg_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg, *args): return args[0]\n" "checkme = MyClass().method(12, 2.5)" << "float"; QTest::newRow("method_explicit_self") << "class MyClass:\n" " def method(self, arg): return arg\n" "instance = MyClass()\n" "checkme = MyClass.method(instance, 12)" << "int"; QTest::newRow("method_vararg_explicit_self") << "class MyClass:\n" " def foo(self, arg, *args): return args[0]\n" "mc = MyClass()\n" "checkme = MyClass.foo(mc, 'str', 3, 4.5)" << "int"; QTest::newRow("clsmethod_explicit_self") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg1, arg2): return arg2\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "int"; QTest::newRow("staticmethod_explicit_self") << "class MyClass:\n" " @staticmethod\n" " def method(arg1, arg2): return arg1\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "str"; QTest::newRow("parent_constructor_arg_type") << "class Base:\n" // https://bugs.kde.org/show_bug.cgi?id=369364 " def __init__(self, foo):\n" " self.foo = foo\n" "class Derived(Base):\n" " def __init__(self, foo):\n" " Base.__init__(self, foo)\n" "instance = Derived('string')\n" "checkme = instance.foo" << "str"; QTest::newRow("nested_class_self_inside") << "class Foo:\n" " def foo(self):\n" " class Bar:\n" " def bar(self): return self\n" " return Bar().bar()\n" "checkme = Foo().foo()\n" << "Foo::foo::Bar"; QTest::newRow("nested_class_self_after") << "class Foo:\n" " class Bar: pass\n" " def foo(self): return self\n" "checkme = Foo().foo()\n" << "Foo"; QTest::newRow("tuple_unsure") << "q = (3, str())\nq=(str(), 3)\ncheckme, _ = q" << "unsure (int, str)"; QTest::newRow("custom_iterable") << "class Gen2:\n" " def __iter__(self): return self\n" " def __next__(self): return 'blah'\n" "for checkme in Gen2(): pass" << "str"; QTest::newRow("separate_iterator") << "class Foo:\n" " def __iter__(self): return Bar()\n" " def __next__(self): return 'blah'\n" // Not used (or shouldn't be!) "class Bar:\n" " def __next__(self): return {1}\n" "checkme = [a for a in Foo()]" << "list of set of int"; QTest::newRow("return_builtin_iterator") << "class Gen2:\n" " contents = [1, 2, 3]\n" " def __iter__(self): return iter(Gen2.contents)\n" "for checkme in Gen2(): pass" << "int"; QTest::newRow("init_class") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "checkme = Foo()\n" << "Foo"; QTest::newRow("init_class_no_decl") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "a = [Foo]\n" "checkme = a[0]()\n" << "Foo"; QTest::newRow("call_class") << "class Foo:\n" " def __call__(self):\n" " return 0\n" "f = Foo()\n" "checkme = f()\n" << "int"; QTest::newRow("call_class_no_decl") << "class Foo:\n" " def __call__(self): return 1.5\n" "a = [Foo()]\n" "checkme = a[0]()" << "float"; QTest::newRow("classmethod") << "class Foo:\n" " @classmethod\n" " def foo(cls):\n" " k = cls()\n" " return k\n" "f = Foo.foo()\n" "checkme = f\n" << "Foo"; QTest::newRow("property_getter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" "checkme = Foo().bar" << "int"; QTest::newRow("property_wrong") << "class Foo:\n" " @property\n" " def bar(self): return True\n" "checkme = Foo().bar()" << "mixed"; QTest::newRow("property_setter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" " @bar.setter\n" " def bar(self, value): return 18.3\n" // Return should be ignored "checkme = Foo().bar" << "int"; QTest::newRow("tuple_listof") << "l = [(1, 2), (3, 4)]\ncheckme = l[1][0]" << "int"; QTest::newRow("getitem") << "class c:\n def __getitem__(self, slice): return 3.14\na = c()\ncheckme = a[2]" << "float"; QTest::newRow("constructor_type_deduction") << "class myclass:\n" "\tdef __init__(self, param): self.foo=param\n" "checkme = myclass(3).foo" << "int"; QTest::newRow("simpe_type_deduction") << "def myfunc(arg): return arg\n" "checkme = myfunc(3)" << "int"; QTest::newRow("functionCall_functionArg_part1") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "checkme = f1()" << "str"; QTest::newRow("functionCall_functionArg_part2") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = identity(getstr)\n" "checkme = f1()\n" << "str"; QTest::newRow("functionCall_functionArg_full") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "f2 = identity(getstr)\n" "a = getstr()\n" "b = f1()\n" "checkme = f2()\n" << "str"; QTest::newRow("vararg_before_other_args") << "def myfun(a, b, *z, x): return z[0]\n" "checkme = myfun(False, False, 1, x = False)" << "int"; QTest::newRow("vararg_before_other_args2") << "def myfun(a, b, *z, x): return z[3]\n" "checkme = myfun(False, False, 1, 2, 3, \"str\", x = False)" << "str"; QTest::newRow("vararg_constructor") << "class myclass():\n" " def __init__(self, *arg): self.prop = arg[0]\n" "obj = myclass(3, 5); checkme = obj.prop" << "int"; QTest::newRow("declaration_order_var") << "aaa = 2\n" "checkme = aaa" << "int"; QTest::newRow("declaration_order_var2") << "checkme = aaa\n" "aaa = 2" << "mixed"; QTest::newRow("declaration_order_func_defarg") << "aaa = 2\n" "def foo(x=aaa): return x\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_defarg2") << "def foo(x=aaa): return x\n" "aaa = 2\n" "checkme = foo()" << "mixed"; QTest::newRow("declaration_order_func_body") << "aaa = 2\n" "def foo(): return aaa\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_body2") << "def foo(): return aaa\n" "aaa = 2\n" "checkme = foo()" << "int"; QTest::newRow("global_variable") << "a = 3\n" "def f1():\n" " global a\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_variable2") << "a = 3\n" "def f1():\n" " global a\n" " a = \"str\"\n" " return a\n" "checkme = f1()\n" << "str"; QTest::newRow("global_scope_variable") << "a = 3\n" "def f1():\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_no_toplevel_dec") << "def f1():\n" " global a\n a = 3\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("top_level_vs_class_attr") << "var = 3\n" "class MyClass:\n" " var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("top_level_vs_instance_attr") << "var = 3\n" "class MyClass:\n" " def __init__(self): self.var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("intermediate_vs_class/instance_attrs") << "def func():\n" " aa, bb = 3, 4\n" " class Foo:\n" " aa = 'a'\n" " def __init__(self):\n" " self.bb = 'b'\n" " def foo(self):\n" " return aa, bb\n" " return Foo().foo()\n" "checkme = func()" << "tuple of (int, int)"; QTest::newRow("top_level_vs_nested_class_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " aaa = 1\n" " class Bar:\n" " bbb = 2\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // Bar.bbb is visible here, Foo.aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; QTest::newRow("top_level_vs_nested_instance_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " def __init__(self): self.aaa = 1\n" " class Bar:\n" " def __init__(self): self.bbb = 1\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // self.bbb is visible here, Foo().aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; } typedef QPair pair; void PyDUChainTest::testImportDeclarations() { QFETCH(QString, code); QFETCH(QStringList, expectedDecls); QFETCH(bool, shouldBeAliased); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); foreach ( const QString& expected, expectedDecls ) { bool found = false; QString name = expected; const auto decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); qCDebug(KDEV_PYTHON_DUCHAIN) << "FOUND DECLARATIONS:"; foreach ( const pair& current, decls ) { qCDebug(KDEV_PYTHON_DUCHAIN) << current.first->toString() << current.first->identifier().identifier().byteArray() << name; } foreach ( const pair& current, decls ) { if ( ! ( current.first->identifier().identifier().byteArray() == name ) ) continue; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found: " << current.first->toString() << " for " << name; AliasDeclaration* isAliased = dynamic_cast(current.first); if ( isAliased && shouldBeAliased ) { found = true; // TODO fixme } else if ( ! isAliased && ! shouldBeAliased ) { found = true; } } QVERIFY(found); } } void PyDUChainTest::testProblemCount() { QFETCH(QString, code); QFETCH(int, problemsCount); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QEXPECT_FAIL("fstring_visit_inside", "Ranges are broken so we don't visit the expression", Continue); QCOMPARE(ctx->problems().size(), problemsCount); } void PyDUChainTest::testProblemCount_data() { QTest::addColumn("code"); QTest::addColumn("problemsCount"); QTest::newRow("list_comp") << "[foo for foo in range(3)]" << 0; QTest::newRow("list_comp_wrong") << "[bar for foo in range(3)]" << 1; QTest::newRow("list_comp_staticmethod") << "class A:\n @staticmethod\n def func(cls):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_decorator") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_wrong") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("list_comp_staticmethod_wrong") << "class A:\n @staticmethod\n def func(cls):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("misplaced_return_plain") << "return" << 1; QTest::newRow("misplaced_return_value") << "return 15" << 1; QTest::newRow("misplaced_return_class") << "class A:\n return 25" << 1; QTest::newRow("correct_return") << "def foo():\n return" << 0; QTest::newRow("lambda_argument_outside") << "def bar():\n lambda foo: 3\n foo" << 1; QTest::newRow("use_found_at_decl") << "foo = 3" << 0; QTest::newRow("fstring_visit_inside") << "checkme = f'{name}'" << 1; } void PyDUChainTest::testImportDeclarations_data() { QTest::addColumn("code"); QTest::addColumn("expectedDecls"); QTest::addColumn("shouldBeAliased"); QTest::newRow("from_import") << "from testImportDeclarations.i import checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("import") << "import testImportDeclarations.i" << ( QStringList() << "testImportDeclarations" ) << false; QTest::newRow("import_as") << "import testImportDeclarations.i as checkme" << ( QStringList() << "checkme" ) << false; QTest::newRow("from_import_as") << "from testImportDeclarations.i import checkme as checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("from_import_missing") << "from testImportDeclarations.i import missing as checkme" << ( QStringList() ) << true; } typedef QPair p; void PyDUChainTest::testAutocompletionFlickering() { TestFile f("foo = 3\nfoo2 = 2\nfo", "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx1 = f.topContext(); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx1); auto decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); QList declIds; foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); f.setFileContents("foo = 3\nfoo2 = 2\nfoo"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx2 = f.topContext(); QVERIFY(ctx2); lock.lock(); auto decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@1: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); qDebug() << "========================="; TestFile g("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfo", "py"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx1 = g.topContext(); lock.lock(); QVERIFY(ctx1); decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); declIds.clear(); foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); g.setFileContents("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfoo"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx2 = g.topContext(); QVERIFY(ctx2); lock.lock(); decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@2: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); } void PyDUChainTest::testFunctionHints() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testFunctionHints_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("func_return_type") << "def myfun(arg) -> int: pass\ncheckme = myfun(\"3\")" << "unsure (None, int)"; QTest::newRow("argument_type") << "def myfun(arg : int): return arg\ncheckme = myfun(foobar)" << "int"; QTest::newRow("argument_type_only_if_typeof") << "def myfun(arg : 3): return arg\ncheckme = myfun(foobar)" << "mixed"; } void PyDUChainTest::testHintedTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testHintedTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("simple_hint") << "def myfunc(x): return x\ncheckme = myfunc(3)" << "int"; QTest::newRow("hint_unsure") << "def myfunc(x): return x\nmyfunc(3.5)\ncheckme = myfunc(3)" << "unsure (float, int)"; QTest::newRow("unsure_attribute") << "def myfunc(x): return x.capitalize()\nmyfunc(3.5)\ncheckme = myfunc(str())" << "str"; } void PyDUChainTest::testOperators() { QFETCH(QString, code); QFETCH(QString, expectedType); code.prepend("from testOperators.example import *\n\n"); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QVERIFY(visitor->found); } void PyDUChainTest::testOperators_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("add") << "checkme = Example() + Example()" << "Add"; QTest::newRow("sub") << "checkme = Example() - Example()" << "Sub"; QTest::newRow("mul") << "checkme = Example() * Example()" << "Mul"; QTest::newRow("floordiv") << "checkme = Example() // Example()" << "Floordiv"; QTest::newRow("mod") << "checkme = Example() % Example()" << "Mod"; QTest::newRow("pow") << "checkme = Example() ** Example()" << "Pow"; QTest::newRow("lshift") << "checkme = Example() << Example()" << "Lshift"; QTest::newRow("rshift") << "checkme = Example() >> Example()" << "Rshift"; QTest::newRow("and") << "checkme = Example() & Example()" << "And"; QTest::newRow("xor") << "checkme = Example() ^ Example()" << "Xor"; QTest::newRow("or") << "checkme = Example() | Example()" << "Or"; } void PyDUChainTest::testFunctionArgs() { ReferencedTopDUContext ctx = parse("def ASDF(arg1, arg2):\n" " arg1 = arg2"); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVERIFY(m_ast); // dumpDUContext(ctx); QCOMPARE(ctx->childContexts().size(), 2); DUContext* funcArgCtx = ctx->childContexts().first(); QCOMPARE(funcArgCtx->type(), DUContext::Function); QCOMPARE(funcArgCtx->localDeclarations().size(), 2); QVERIFY(!funcArgCtx->owner()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); QCOMPARE(decl->type()->arguments().length(), 2); qDebug() << decl->type()->arguments().length() << 2; DUContext* funcBodyCtx = ctx->childContexts().last(); QCOMPARE(funcBodyCtx->type(), DUContext::Other); QVERIFY(funcBodyCtx->owner()); QVERIFY(funcBodyCtx->localDeclarations().isEmpty()); } void PyDUChainTest::testInheritance() { QFETCH(QString, code); QFETCH(int, expectedBaseClasses); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); auto decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); bool found = false; bool classDeclFound = false; foreach ( const p& item, decls ) { if ( item.first->identifier().toString() == "B" ) { auto klass = dynamic_cast(item.first); QVERIFY(klass); QCOMPARE(klass->baseClassesSize(), static_cast(expectedBaseClasses)); classDeclFound = true; } if ( item.first->identifier().toString() == "checkme" ) { QCOMPARE(item.first->abstractType()->toString(), QString("int")); found = true; } } QVERIFY(found); QVERIFY(classDeclFound); } void PyDUChainTest::testInheritance_data() { QTest::addColumn("code"); QTest::addColumn("expectedBaseClasses"); QTest::newRow("simple") << "class A():\n\tattr = 3\n\nclass B(A):\n\tpass\n\ninst=B()\ncheckme = inst.attr" << 1; QTest::newRow("context_import_prereq") << "import testInheritance.i\ninst=testInheritance.i.testclass()\n" "checkme = inst.attr\nclass B(): pass" << 1; // 1 because object QTest::newRow("context_import") << "import testInheritance.i\n\nclass B(testInheritance.i.testclass):\n" "\ti = 4\n\ninst=B()\ncheckme = inst.attr" << 1; } void PyDUChainTest::testClassContextRanges() { QString code = "class my_class():\n pass\n \n \n \n \n"; ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; DUContext* classContext = ctx->findContextAt(CursorInRevision(5, 0)); QVERIFY(classContext); QVERIFY(classContext->type() == DUContext::Class); } void PyDUChainTest::testContainerTypes() { QFETCH(QString, code); QFETCH(QString, contenttype); QFETCH(bool, use_type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList decls = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(decls.length() > 0); QVERIFY(decls.first()->abstractType()); if ( ! use_type ) { auto type = ListType::Ptr::dynamicCast(decls.first()->abstractType()); QVERIFY(type); QVERIFY(type->contentType()); QCOMPARE(type->contentType().abstractType()->toString(), contenttype); } else { QVERIFY(decls.first()->abstractType()); QEXPECT_FAIL("dict_of_int_call", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("dict_from_tuples", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("comprehension_shadowing_ms", "Nothing is foolproof to a sufficiently capable fool", Continue); QEXPECT_FAIL("comprehension_shadowing_nest2", "See above", Continue); QCOMPARE(decls.first()->abstractType()->toString(), contenttype); } } void PyDUChainTest::testContainerTypes_data() { QTest::addColumn("code"); QTest::addColumn("contenttype"); QTest::addColumn("use_type"); QTest::newRow("list_of_int") << "checkme = [1, 2, 3]" << "int" << false; QTest::newRow("list_from_unpacked") << "foo = [1.3]\ncheckme = [1, *foo, 3]" << "unsure (int, float)" << false; QTest::newRow("list_of_int_call") << "checkme = list([1, 2, 3])" << "int" << false; QTest::newRow("list_from_tuple") << "checkme = list((1, 2, 3))" << "int" << false; QTest::newRow("list_from_dict") << "checkme = list({'a':1, 'b':2})" << "str" << false; // Gets key type! QTest::newRow("list_from_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = list(MyClass())" << "float" << false; QTest::newRow("generator") << "checkme = [i for i in [1, 2, 3]]" << "int" << false; QTest::newRow("list_access") << "list = [1, 2, 3]\ncheckme = list[0]" << "int" << true; QTest::newRow("set_of_int") << "checkme = {1, 2, 3}" << "int" << false; QTest::newRow("set_of_int_call") << "checkme = set({1, 2, 3})" << "int" << false; QTest::newRow("set_from_tuple") << "checkme = set((1, 2, 3))" << "int" << false; QTest::newRow("set_generator") << "checkme = {i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_of_str_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; QTest::newRow("frozenset_of_int_call") << "checkme = frozenset({1, 2, 3})" << "int" << false; QTest::newRow("dict_of_int") << "checkme = {a:1, b:2, c:3}" << "int" << false; QTest::newRow("dict_of_int_call") << "checkme = dict({'a':1, 'b':2, 'c':3})" << "dict of str : int" << true; QTest::newRow("dict_from_tuples") << "checkme = dict([('a', 1), ('b', 2)])" << "dict of str : int" << true; QTest::newRow("dict_generator") << "checkme = {\"Foo\":i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_access") << "list = {'a':1, 'b':2, 'c':3}\ncheckme = list[0]" << "int" << true; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("set_from_unpacked") << "foo = [1.3]\ncheckme = {1, *foo, 3}" << "unsure (int, float)" << false; QTest::newRow("dict_from_unpacked") << "checkme = {**{'a': 1}}" << "dict of str : int" << true; QTest::newRow("dict_from_varied") << "checkme = {**{'a': 1}, 1: 1.5}" << "dict of unsure (str, int) : unsure (int, float)" << true; #endif QTest::newRow("generator_attribute") << "checkme = [item.capitalize() for item in ['foobar']]" << "str" << false; QTest::newRow("cannot_change_type") << "checkme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("cannot_change_type2") << "[1, 2, 3].append(5)\ncheckme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("list_append") << "d = []\nd.append(3)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend") << "d = []; q = [int()]\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_tuple") << "d = []; q = (1, 2)\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = []\ncheckme.extend(MyClass())" << "float" << false; QTest::newRow("for_loop") << "d = [3]\nfor item in d:\n checkme = item" << "int" << true; QTest::newRow("for_loop_unsure") << "d = [3, \"foo\"]\nfor item in d:\n checkme = item" << "unsure (int, str)" << true; QTest::newRow("for_loop_tuple_1") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = a" << "int" << true; QTest::newRow("for_loop_tuple_2") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = b" << "float" << true; QTest::newRow("for_loop_tuple_unsure") << "d = [(3, 3.5), (3.5, 3)]\nfor a, b in d:\n checkme = b" << "unsure (float, int)" << true; // Proposed by Nicolás Alvarez; why not? https://bugs.kde.org/show_bug.cgi?id=359915 QTest::newRow("comprehension_messy") << "users = {'a':19, 'b':42, 'c':35}\n" "sorted_list = sorted(users.items(), key=lambda kv: (-kv[1], kv[0]))\n" "checkme = [k for r,(k,v) in enumerate(sorted_list, 1)]" << "list of str" << true; QTest::newRow("comprehension_multiline") << "checkme = [a for\n a in \n (1, 2)]" << "list of int" << true; QTest::newRow("comprehension_multistage") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for bar in nested for foo in bar]" << "list of int" << true; QTest::newRow("comprehension_shadowing_ms") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in nested for foo in foo]" << "list of int" << true; QTest::newRow("comprehension_shadowing_nest1") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in [foo for foo in nested]]" << "list of tuple of (int, int)" << true; QTest::newRow("comprehension_shadowing_nest2") << "nested = (1, 2), (3, 4)\n" "checkme = [[foo for foo in foo] for foo in nested]" << "list of list of int" << true; // From https://bugs.kde.org/show_bug.cgi?id=359912 QTest::newRow("subscript_multi") << "class Middle:\n def __getitem__(self, key):\n return str()\n" "class Outer:\n def __getitem__(self, key):\n return Middle()\n" "aaa = Outer()\ncheckme = aaa[0][0]" << "str" << true; QTest::newRow("subscript_func_call") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "def bar():\n return Foo()\n" "checkme = bar()[0]" << "str" << true; QTest::newRow("subscript_unknown_index") << "a = 1,str()\ncheckme = a[5-4]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure") << "a = 1,2\na=[str()]\ncheckme = a[0]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure_getitem") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "class Bar:\n def __getitem__(self, key):\n return float()\n" "a = Foo()\na=Bar()\na=[1,2]\ncheckme = a[1]" << "unsure (str, float, int)" << true; } void PyDUChainTest::testVariableCreation() { QFETCH(QString, code); QFETCH(QStringList, expected_local_declarations); QFETCH(QStringList, expected_types); ReferencedTopDUContext top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto localDecls = top->localDeclarations(); QVector localDeclNames; for ( const Declaration* d: localDecls ) { localDeclNames.append(d->identifier().toString()); } Q_ASSERT(expected_types.size() == expected_local_declarations.size()); int offset = 0; for ( const QString& expected : expected_local_declarations ) { int index = localDeclNames.indexOf(expected); QVERIFY(index != -1); QVERIFY(localDecls[index]->abstractType()); QCOMPARE(localDecls[index]->abstractType()->toString(), expected_types[offset]); offset++; } } void PyDUChainTest::testVariableCreation_data() { QTest::addColumn("code"); QTest::addColumn("expected_local_declarations"); QTest::addColumn("expected_types"); QTest::newRow("simple") << "a = 3" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("tuple_wrong") << "a, b = 3" << QStringList{"a", "b"} << QStringList{"mixed", "mixed"}; QTest::newRow("tuple_unpack_inplace") << "a, b = 3, 5.5" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_indirect") << "c = 3, 3.5\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_stacked_inplace") << "a, (b, c) = 1, (2, 3.5)" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("tuple_unpack_stacked_indirect") << "d = 3.5, (3, 1)\na, (b, c) = d" << QStringList{"a", "b", "c"} << QStringList{"float", "int", "int"}; QTest::newRow("unpack_from_list_inplace") << "a, b = [1, 2, 3]" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_from_list_indirect") << "c = [1, 2, 3]\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_custom_iterable") << "class Foo:\n" " def __iter__(self): return self\n" " def __next__(self): return 1.5\n" "a, *b = Foo()" << QStringList{"a", "b"} << QStringList {"float", "list of float"}; QTest::newRow("for_loop_simple") << "for i in range(3): pass" << QStringList{"i"} << QStringList{"int"}; QTest::newRow("for_loop_unpack") << "for a, b in [(3, 5.1)]: pass" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("for_loop_stacked") << "for a, (b, c) in [(1, (2, 3.5))]: pass" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("for_loop_tuple") << "for a in 1, 2: pass" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("for_loop_dict") << "for a in {'foo': 1}: pass" << QStringList{"a"} << QStringList{"str"}; } void PyDUChainTest::testCleanupMultiplePasses() { for ( int j = 0; j < 20; j++ ) { ReferencedTopDUContext top = parse("from testCleanupMultiplePasses import foo\ndef fonc(): return 3+2j\nfoo.foo.func = fonc"); } } void PyDUChainTest::testManyDeclarations() { ReferencedTopDUContext top = parse("from testManyDeclarations import test\nk=test.Foo()"); } void PyDUChainTest::testComments() { QFETCH(QString, code); auto top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto decls = top->findDeclarations(QualifiedIdentifier("a")); QCOMPARE(decls.size(), 1); auto a = decls.first(); QCOMPARE(a->comment(), QByteArray("comment")); decls = top->findDeclarations(QualifiedIdentifier("b")); if ( decls.isEmpty() ) { decls = top->childContexts().last()->findDeclarations(QualifiedIdentifier("b")); } auto b = decls.first(); QCOMPARE(b->comment(), QByteArray()); } void PyDUChainTest::testComments_data() { QTest::addColumn("code"); QTest::newRow("variable") << "b=5\n\"\"\"comment\"\"\"\na=5\nb=5"; QTest::newRow("function") << "def a():\n \"\"\"comment\"\"\"\n b=5"; QTest::newRow("class") << "class a:\n \"\"\"comment\"\"\"\n b=5"; } diff --git a/parser/ast.h b/parser/ast.h index b290c5a8..378af0bd 100644 --- a/parser/ast.h +++ b/parser/ast.h @@ -1,783 +1,783 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ // The Python 3.4 Language Reference was used as basis for this AST #ifndef PYTHON_AST_H #define PYTHON_AST_H #include #include #include #include #include "parserexport.h" #include "kdevpythonversion.h" namespace KDevelop { class DUContext; } namespace Python { class StatementAst; class FunctionDefinitionAst; class AssignmentAst; class PassAst; class NonlocalAst; class ExpressionAst; class NameAst; class CallAst; class AttributeAst; class ArgumentsAst; class KeywordAst; class ExpressionAst; class StatementAst; class Ast; class ExceptionHandlerAst; class AliasAst; class ComprehensionAst; class SliceAstBase; class SliceAst; } namespace Python { // Base class for all other Abstract Syntax Tree classes class KDEVPYTHONPARSER_EXPORT Ast { public: enum AstType { StatementAstType, FunctionDefinitionAstType, AssignmentAstType, PassAstType, NonlocalAstType, ArgumentsAstType, ArgAstType, KeywordAstType, ClassDefinitionAstType, ReturnAstType, DeleteAstType, ForAstType, WhileAstType, IfAstType, WithAstType, WithItemAstType, RaiseAstType, TryAstType, ImportAstType, ImportFromAstType, GlobalAstType, BreakAstType, ContinueAstType, AssertionAstType, AugmentedAssignmentAstType, AnnotationAssignmentAstType, LastStatementType, ExpressionAstType, // everything below is an expression AwaitAstType, NameAstType, NameConstantAstType, CallAstType, AttributeAstType, ExtendedSliceAstType, DictionaryComprehensionAstType, BooleanOperationAstType, BinaryOperationAstType, UnaryOperationAstType, LambdaAstType, IfExpressionAstType, // the short one, if a then b else c DictAstType, SetAstType, ListComprehensionAstType, SetComprehensionAstType, GeneratorExpressionAstType, YieldAstType, CompareAstType, NumberAstType, StringAstType, JoinedStringAstType, FormattedValueAstType, BytesAstType, SubscriptAstType, StarredAstType, ListAstType, TupleAstType, YieldFromAstType, ComprehensionAstType, SliceAstType, EllipsisAstType, IndexAstType, LastExpressionType, // keep this at the end of the expr ast list CodeAstType, ExceptionHandlerAstType, AliasAstType, // for imports IdentifierAstType, LastAstType // the largest one, not valid! }; enum BooleanOperationTypes { BooleanAnd = 1, BooleanOr, BooleanInvalidOperation }; enum OperatorTypes { OperatorAdd = 1, OperatorSub, OperatorMult, #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) OperatorMatMult, #endif OperatorDiv, OperatorMod, OperatorPow, OperatorLeftShift, OperatorRightShift, OperatorBitwiseOr, OperatorBitwiseXor, OperatorBitwiseAnd, OperatorFloorDivision, OperatorInvalid }; enum UnaryOperatorTypes { UnaryOperatorInvert = 1, UnaryOperatorNot, UnaryOperatorAdd, UnaryOperatorSub, UnaryOperatorInvalid }; enum ComparisonOperatorTypes { ComparisonOperatorEquals = 1, ComparisonOperatorNotEquals, ComparisonOperatorLessThan, ComparisonOperatorLessThanEqual, ComparisonOperatorGreaterThan, ComparisonOperatorGreaterThanEqual, ComparisonOperatorIs, ComparisonOperatorIsNot, ComparisonOperatorIn, ComparisonOperatorNotIn, ComparisonOperatorInvalid }; Ast(Ast* parent, AstType type); Ast(); Ast* parent = nullptr; AstType astType; bool isExpression() const { return astType >= ExpressionAstType && astType <= LastExpressionType; } void copyRange(const Ast* other) { startCol = other->startCol; endCol = other->endCol; startLine = other->startLine; endLine = other->endLine; } bool appearsBefore(const Ast* other) { return startLine < other->startLine || ( startLine == other->startLine && startCol < other->startCol ); } const KTextEditor::Range range() const { return KTextEditor::Range(startLine, startCol, endLine, endCol); }; const KTextEditor::Cursor start() const { return {startLine, startCol}; } const KTextEditor::Cursor end() const { return {endLine, endCol}; } bool isChildOf(Ast* other) const { const Ast* parent = this; while ( parent ) { if ( parent == other ) { return true; } parent = parent->parent; } return false; } int startCol; int startLine; int endCol; int endLine; bool hasUsefulRangeInformation; KDevelop::DUContext* context; }; class KDEVPYTHONPARSER_EXPORT Identifier : public Ast { public: Identifier(QString value); Identifier operator=(const Identifier& other) { value = other.value; startCol = other.startCol; endCol = other.endCol; startLine = other.startLine; endLine = other.endLine; parent = other.parent; hasUsefulRangeInformation = other.hasUsefulRangeInformation; return *this; }; bool operator==(const Identifier& rhs) const { return value == rhs.value; }; bool operator==(const QString& rhs) const { return value == rhs; }; void setEndColumn() { endCol = startCol + value.length() - 1; } operator QString() const { return value; }; QString value; }; // this replaces ModuleAst class KDEVPYTHONPARSER_EXPORT CodeAst : public Ast { public: CodeAst(); ~CodeAst(); typedef QSharedPointer Ptr; QList body; Identifier* name; // module name }; /** Statement classes **/ class KDEVPYTHONPARSER_EXPORT StatementAst : public Ast { public: StatementAst(Ast* parent, AstType type); }; class KDEVPYTHONPARSER_EXPORT FunctionDefinitionAst : public StatementAst { public: FunctionDefinitionAst(Ast* parent); Identifier* name; ArgumentsAst* arguments; QList decorators; QList body; ExpressionAst* returns; bool async; }; class KDEVPYTHONPARSER_EXPORT ClassDefinitionAst : public StatementAst { public: ClassDefinitionAst(Ast* parent); Identifier* name; QList baseClasses; QList body; QList decorators; }; class KDEVPYTHONPARSER_EXPORT ReturnAst : public StatementAst { public: ReturnAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT DeleteAst : public StatementAst { public: DeleteAst(Ast* parent); QList targets; }; class KDEVPYTHONPARSER_EXPORT AssignmentAst : public StatementAst { public: AssignmentAst(Ast* parent); QList targets; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT AugmentedAssignmentAst : public StatementAst { public: AugmentedAssignmentAst(Ast* parent); ExpressionAst* target; Ast::OperatorTypes op; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT AnnotationAssignmentAst : public StatementAst { public: AnnotationAssignmentAst(Ast* parent); ExpressionAst* target; ExpressionAst* value; ExpressionAst* annotation; }; class KDEVPYTHONPARSER_EXPORT ForAst : public StatementAst { public: ForAst(Ast* parent); ExpressionAst* target; ExpressionAst* iterator; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT WhileAst : public StatementAst { public: WhileAst(Ast* parent); ExpressionAst* condition; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT IfAst : public StatementAst { public: IfAst(Ast* parent); ExpressionAst* condition; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT WithItemAst : public Ast { public: WithItemAst(Ast* parent); ExpressionAst* contextExpression; - NameAst* optionalVars; + ExpressionAst* optionalVars; }; class KDEVPYTHONPARSER_EXPORT WithAst : public StatementAst { public: WithAst(Ast* parent); QList body; QList items; }; class KDEVPYTHONPARSER_EXPORT RaiseAst : public StatementAst { public: RaiseAst(Ast* parent); ExpressionAst* type; // TODO check what the other things in the grammar actually are and add them }; class KDEVPYTHONPARSER_EXPORT TryAst : public StatementAst { public: TryAst(Ast* parent); QList body; QList handlers; QList orelse; QList finally; }; class KDEVPYTHONPARSER_EXPORT AssertionAst : public StatementAst { public: AssertionAst(Ast* parent); ExpressionAst* condition; ExpressionAst* message; }; class KDEVPYTHONPARSER_EXPORT ImportAst : public StatementAst { public: ImportAst(Ast* parent); QList names; }; class KDEVPYTHONPARSER_EXPORT ImportFromAst : public StatementAst { public: ImportFromAst(Ast* parent); Identifier* module; QList names; int level; }; class KDEVPYTHONPARSER_EXPORT GlobalAst : public StatementAst { public: GlobalAst(Ast* parent); QList names; }; // TODO what's stmt::Expr(expr value) in the grammar and what do we need it for? class KDEVPYTHONPARSER_EXPORT BreakAst : public StatementAst { public: BreakAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT ContinueAst : public StatementAst { public: ContinueAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT PassAst : public StatementAst { public: PassAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT NonlocalAst : public StatementAst { public: NonlocalAst(Ast* parent); }; /** Expression classes **/ class KDEVPYTHONPARSER_EXPORT ExpressionAst : public Ast { public: ExpressionAst(Ast* parent, AstType type = Ast::ExpressionAstType); enum Context { Load = 1, // the object is read Store = 2, // the object is written Delete = 3, // the object is deleted Invalid = -1 }; ExpressionAst* value; // WARNING this is not set in most cases! }; class KDEVPYTHONPARSER_EXPORT AwaitAst : public ExpressionAst { public: AwaitAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT YieldFromAst : public ExpressionAst { public: YieldFromAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT BooleanOperationAst : public ExpressionAst { public: BooleanOperationAst(Ast* parent); Ast::BooleanOperationTypes type; QList values; }; class KDEVPYTHONPARSER_EXPORT BinaryOperationAst : public ExpressionAst { public: BinaryOperationAst(Ast* parent); Ast::OperatorTypes type; ExpressionAst* lhs; ExpressionAst* rhs; inline QString methodName() const { switch ( type ) { case Python::Ast::OperatorAdd: return QLatin1String("__add__"); case Python::Ast::OperatorBitwiseAnd: return QLatin1String("__and__"); case Python::Ast::OperatorBitwiseOr: return QLatin1String("__or__"); case Python::Ast::OperatorBitwiseXor: return QLatin1String("__xor__"); case Python::Ast::OperatorDiv: return QLatin1String("__div__"); case Python::Ast::OperatorFloorDivision: return QLatin1String("__floordiv__"); case Python::Ast::OperatorLeftShift: return QLatin1String("__lshift__"); case Python::Ast::OperatorMod: return QLatin1String("__mod__"); case Python::Ast::OperatorMult: return QLatin1String("__mul__"); #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Python::Ast::OperatorMatMult: return QLatin1String("__matmul__"); #endif case Python::Ast::OperatorPow: return QLatin1String("__pow__"); case Python::Ast::OperatorRightShift: return QLatin1String("__rshift__"); case Python::Ast::OperatorSub: return QLatin1String("__sub__"); case Python::Ast::OperatorInvalid: // fallthrough default: return QString(); } }; // incremental methods, for e.g. a += 3 inline QString incMethodName() const { QString name = methodName(); if ( name.size() < 3 ) { return name; } name.insert(2, QLatin1Char('i')); return name; } }; class KDEVPYTHONPARSER_EXPORT UnaryOperationAst : public ExpressionAst { public: UnaryOperationAst(Ast* parent); Ast::UnaryOperatorTypes type; ExpressionAst* operand; }; class KDEVPYTHONPARSER_EXPORT LambdaAst : public ExpressionAst { public: LambdaAst(Ast* parent); ArgumentsAst* arguments; ExpressionAst* body; }; class KDEVPYTHONPARSER_EXPORT IfExpressionAst : public ExpressionAst { public: IfExpressionAst(Ast* parent); ExpressionAst* condition; ExpressionAst* body; ExpressionAst* orelse; }; class KDEVPYTHONPARSER_EXPORT DictAst : public ExpressionAst { public: DictAst(Ast* parent); QList keys; // WARNING: Can contain null elements: `{**other}` QList values; }; class KDEVPYTHONPARSER_EXPORT SetAst : public ExpressionAst { public: SetAst(Ast* parent); QList elements; }; class KDEVPYTHONPARSER_EXPORT ListComprehensionAst : public ExpressionAst { public: ListComprehensionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT SetComprehensionAst : public ExpressionAst { public: SetComprehensionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT DictionaryComprehensionAst : public ExpressionAst { public: DictionaryComprehensionAst(Ast* parent); ExpressionAst* key; ExpressionAst* value; QList generators; }; class KDEVPYTHONPARSER_EXPORT GeneratorExpressionAst : public ExpressionAst { public: GeneratorExpressionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT CompareAst : public ExpressionAst { public: CompareAst(Ast* parent); ExpressionAst* leftmostElement; QList operators; QList comparands; }; // TODO whats this exactly? class KDEVPYTHONPARSER_EXPORT ReprAst : public ExpressionAst { public: ReprAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT NumberAst : public ExpressionAst { public: NumberAst(Ast* parent); long value; // only used for ints bool isInt; // otherwise it's a float }; class KDEVPYTHONPARSER_EXPORT StringAst : public ExpressionAst { public: StringAst(Ast* parent); QString value; bool usedAsComment; }; class KDEVPYTHONPARSER_EXPORT JoinedStringAst : public ExpressionAst { public: JoinedStringAst(Ast* parent); QList values; }; class KDEVPYTHONPARSER_EXPORT FormattedValueAst : public ExpressionAst { public: FormattedValueAst(Ast* parent); ExpressionAst* value; int conversion; ExpressionAst* formatSpec; }; class KDEVPYTHONPARSER_EXPORT BytesAst : public ExpressionAst { public: BytesAst(Ast* parent); QString value; }; class KDEVPYTHONPARSER_EXPORT YieldAst : public ExpressionAst { public: YieldAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT NameAst : public ExpressionAst { public: NameAst(Ast* parent); Identifier* identifier; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT NameConstantAst : public ExpressionAst { public: NameConstantAst(Ast* parent); enum NameConstantTypes { False, True, None, Invalid // should not happen }; NameConstantTypes value; }; class KDEVPYTHONPARSER_EXPORT CallAst : public ExpressionAst { public: CallAst(Ast* parent); ExpressionAst* function; QList arguments; QList keywords; }; class KDEVPYTHONPARSER_EXPORT AttributeAst : public ExpressionAst { public: AttributeAst(Ast* parent); ExpressionAst* value; Identifier* attribute; ExpressionAst::Context context; int depth; }; class KDEVPYTHONPARSER_EXPORT SubscriptAst : public ExpressionAst { public: SubscriptAst(Ast* parent); ExpressionAst* value; SliceAstBase* slice; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT StarredAst : public ExpressionAst { public: StarredAst(Ast* parent); ExpressionAst* value; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT ListAst : public ExpressionAst { public: ListAst(Ast* parent); QList elements; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT TupleAst : public ExpressionAst { public: TupleAst(Ast* parent); QList elements; ExpressionAst::Context context; }; /** Slice classes **/ class KDEVPYTHONPARSER_EXPORT SliceAstBase : public Ast { public: SliceAstBase(Ast* parent, AstType type); }; class KDEVPYTHONPARSER_EXPORT EllipsisAst : public SliceAstBase { public: EllipsisAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT SliceAst : public SliceAstBase { public: SliceAst(Ast* parent); ExpressionAst* lower; ExpressionAst* upper; ExpressionAst* step; }; class KDEVPYTHONPARSER_EXPORT ExtendedSliceAst : public SliceAstBase { public: ExtendedSliceAst(Ast* parent); QList dims; }; class KDEVPYTHONPARSER_EXPORT IndexAst : public SliceAstBase { public: IndexAst(Ast* parent); ExpressionAst* value; }; /** Independent classes **/ class KDEVPYTHONPARSER_EXPORT ArgAst : public Ast { public: ArgAst(Ast* parent); Identifier* argumentName; ExpressionAst* annotation; }; class KDEVPYTHONPARSER_EXPORT ArgumentsAst : public Ast { public: ArgumentsAst(Ast* parent); QList arguments; QList kwonlyargs; QList defaultValues; ArgAst* vararg; ArgAst* kwarg; }; class KDEVPYTHONPARSER_EXPORT KeywordAst : public Ast { public: KeywordAst(Ast* parent); Identifier* argumentName; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT ComprehensionAst : public Ast { public: ComprehensionAst(Ast* parent); ExpressionAst* target; ExpressionAst* iterator; QList conditions; }; class KDEVPYTHONPARSER_EXPORT ExceptionHandlerAst : public Ast { public: ExceptionHandlerAst(Ast* parent); ExpressionAst* type; Identifier* name; QList body; }; class KDEVPYTHONPARSER_EXPORT AliasAst : public Ast { public: AliasAst(Ast* parent); Identifier* name; Identifier* asName; }; } #endif diff --git a/parser/generated.h b/parser/generated.h index 1087ac2e..9061deb0 100644 --- a/parser/generated.h +++ b/parser/generated.h @@ -1,792 +1,792 @@ /* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ #include #include "kdevpythonversion.h" class PythonAstTransformer { public: CodeAst* ast; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } // Shift lines by some fixed amount inline int tline(int line) { if ( line == -99999 ) { // don't touch the marker return -99999; } return line; }; private: QStack nodeStack; Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } Ast* visitNode(_alias* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; AliasAst* v = new AliasAst(parent()); v->name = node->name ? new Python::Identifier(PyUnicodeObjectToQString(node->name)) : nullptr; v->asName = node->asname ? new Python::Identifier(PyUnicodeObjectToQString(node->asname)) : nullptr; return v; } Ast* visitNode(_arg* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgAst* v = new ArgAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; if ( v->argumentName ) { v->argumentName->startCol = node->col_offset; v->startCol = v->argumentName->startCol; v->argumentName->startLine = tline(node->lineno - 1); v->startLine = v->argumentName->startLine; v->argumentName->endCol = node->col_offset + v->argumentName->value.length() - 1; v->endCol = v->argumentName->endCol; v->argumentName->endLine = tline(node->lineno - 1); v->endLine = v->argumentName->endLine; ranges_copied = true; } nodeStack.push(v); v->annotation = static_cast(visitNode(node->annotation)); nodeStack.pop(); return v; } Ast* visitNode(_arguments* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgumentsAst* v = new ArgumentsAst(parent()); nodeStack.push(v); v->vararg = static_cast(visitNode(node->vararg)); nodeStack.pop(); nodeStack.push(v); v->kwarg = static_cast(visitNode(node->kwarg)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_arg, ArgAst>(node->args); nodeStack.pop(); nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); nodeStack.push(v); v->kwonlyargs = visitNodeList<_arg, ArgAst>(node->kwonlyargs); nodeStack.pop(); return v; } Ast* visitNode(_comprehension* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ComprehensionAst* v = new ComprehensionAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->iter)); nodeStack.pop(); nodeStack.push(v); v->conditions = visitNodeList<_expr, ExpressionAst>(node->ifs); nodeStack.pop(); return v; } Ast* visitNode(_excepthandler* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case ExceptHandler_kind: { ExceptionHandlerAst* v = new ExceptionHandlerAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.ExceptHandler.type)); nodeStack.pop(); v->name = node->v.ExceptHandler.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ExceptHandler.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ExceptHandler.body); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _excepthandler AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_expr* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Await_kind: { AwaitAst* v = new AwaitAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Await.value)); nodeStack.pop(); result = v; break; } #endif case BoolOp_kind: { BooleanOperationAst* v = new BooleanOperationAst(parent()); v->type = (ExpressionAst::BooleanOperationTypes) node->v.BoolOp.op; nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.BoolOp.values); nodeStack.pop(); result = v; break; } case BinOp_kind: { BinaryOperationAst* v = new BinaryOperationAst(parent()); v->type = (ExpressionAst::OperatorTypes) node->v.BinOp.op; nodeStack.push(v); v->lhs = static_cast(visitNode(node->v.BinOp.left)); nodeStack.pop(); nodeStack.push(v); v->rhs = static_cast(visitNode(node->v.BinOp.right)); nodeStack.pop(); result = v; break; } case UnaryOp_kind: { UnaryOperationAst* v = new UnaryOperationAst(parent()); v->type = (ExpressionAst::UnaryOperatorTypes) node->v.UnaryOp.op; nodeStack.push(v); v->operand = static_cast(visitNode(node->v.UnaryOp.operand)); nodeStack.pop(); result = v; break; } case Lambda_kind: { LambdaAst* v = new LambdaAst(parent()); nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.Lambda.args)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Lambda.body)); nodeStack.pop(); result = v; break; } case IfExp_kind: { IfExpressionAst* v = new IfExpressionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.IfExp.test)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.IfExp.body)); nodeStack.pop(); nodeStack.push(v); v->orelse = static_cast(visitNode(node->v.IfExp.orelse)); nodeStack.pop(); result = v; break; } case Dict_kind: { DictAst* v = new DictAst(parent()); nodeStack.push(v); v->keys = visitNodeList<_expr, ExpressionAst>(node->v.Dict.keys); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Dict.values); nodeStack.pop(); result = v; break; } case Set_kind: { SetAst* v = new SetAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Set.elts); nodeStack.pop(); result = v; break; } case ListComp_kind: { ListComprehensionAst* v = new ListComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.ListComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.ListComp.generators); nodeStack.pop(); result = v; break; } case SetComp_kind: { SetComprehensionAst* v = new SetComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.SetComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.SetComp.generators); nodeStack.pop(); result = v; break; } case DictComp_kind: { DictionaryComprehensionAst* v = new DictionaryComprehensionAst(parent()); nodeStack.push(v); v->key = static_cast(visitNode(node->v.DictComp.key)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.DictComp.value)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.DictComp.generators); nodeStack.pop(); result = v; break; } case GeneratorExp_kind: { GeneratorExpressionAst* v = new GeneratorExpressionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.GeneratorExp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.GeneratorExp.generators); nodeStack.pop(); result = v; break; } case Yield_kind: { YieldAst* v = new YieldAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Yield.value)); nodeStack.pop(); result = v; break; } case Compare_kind: { CompareAst* v = new CompareAst(parent()); nodeStack.push(v); v->leftmostElement = static_cast(visitNode(node->v.Compare.left)); nodeStack.pop(); for ( int _i = 0; _i < node->v.Compare.ops->size; _i++ ) { v->operators.append((ExpressionAst::ComparisonOperatorTypes) node->v.Compare.ops->elements[_i]); } nodeStack.push(v); v->comparands = visitNodeList<_expr, ExpressionAst>(node->v.Compare.comparators); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION < QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop();};if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop();}; result = v; break; } #endif case Num_kind: { NumberAst* v = new NumberAst(parent()); v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n); result = v; break; } case Str_kind: { StringAst* v = new StringAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Str.s); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case JoinedStr_kind: { JoinedStringAst* v = new JoinedStringAst(parent()); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.JoinedStr.values); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case FormattedValue_kind: { FormattedValueAst* v = new FormattedValueAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.FormattedValue.value)); nodeStack.pop(); v->conversion = node->v.FormattedValue.conversion; nodeStack.push(v); v->formatSpec = static_cast(visitNode(node->v.FormattedValue.format_spec)); nodeStack.pop(); result = v; break; } #endif case Bytes_kind: { BytesAst* v = new BytesAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Bytes.s); result = v; break; } case Attribute_kind: { AttributeAst* v = new AttributeAst(parent()); v->attribute = node->v.Attribute.attr ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Attribute.attr)) : nullptr; if ( v->attribute ) { v->attribute->startCol = node->col_offset; v->startCol = v->attribute->startCol; v->attribute->startLine = tline(node->lineno - 1); v->startLine = v->attribute->startLine; v->attribute->endCol = node->col_offset + v->attribute->value.length() - 1; v->endCol = v->attribute->endCol; v->attribute->endLine = tline(node->lineno - 1); v->endLine = v->attribute->endLine; ranges_copied = true; } nodeStack.push(v); v->value = static_cast(visitNode(node->v.Attribute.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Attribute.ctx; result = v; break; } case Subscript_kind: { SubscriptAst* v = new SubscriptAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Subscript.value)); nodeStack.pop(); nodeStack.push(v); v->slice = static_cast(visitNode(node->v.Subscript.slice)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Subscript.ctx; result = v; break; } case Starred_kind: { StarredAst* v = new StarredAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Starred.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Starred.ctx; result = v; break; } case Name_kind: { NameAst* v = new NameAst(parent()); v->identifier = node->v.Name.id ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Name.id)) : nullptr; if ( v->identifier ) { v->identifier->startCol = node->col_offset; v->startCol = v->identifier->startCol; v->identifier->startLine = tline(node->lineno - 1); v->startLine = v->identifier->startLine; v->identifier->endCol = node->col_offset + v->identifier->value.length() - 1; v->endCol = v->identifier->endCol; v->identifier->endLine = tline(node->lineno - 1); v->endLine = v->identifier->endLine; ranges_copied = true; } v->context = (ExpressionAst::Context) node->v.Name.ctx; result = v; break; } case List_kind: { ListAst* v = new ListAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.List.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.List.ctx; result = v; break; } case Tuple_kind: { TupleAst* v = new TupleAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Tuple.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Tuple.ctx; result = v; break; } case Ellipsis_kind: { EllipsisAst* v = new EllipsisAst(parent()); result = v; break; } case NameConstant_kind: { NameConstantAst* v = new NameConstantAst(parent()); v->value = node->v.NameConstant.value == Py_None ? NameConstantAst::None : node->v.NameConstant.value == Py_False ? NameConstantAst::False : NameConstantAst::True; result = v; break; } case YieldFrom_kind: { YieldFromAst* v = new YieldFromAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.YieldFrom.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _expr AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_keyword* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; KeywordAst* v = new KeywordAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; nodeStack.push(v); v->value = static_cast(visitNode(node->value)); nodeStack.pop(); return v; } Ast* visitNode(_slice* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Slice_kind: { SliceAst* v = new SliceAst(parent()); nodeStack.push(v); v->lower = static_cast(visitNode(node->v.Slice.lower)); nodeStack.pop(); nodeStack.push(v); v->upper = static_cast(visitNode(node->v.Slice.upper)); nodeStack.pop(); nodeStack.push(v); v->step = static_cast(visitNode(node->v.Slice.step)); nodeStack.pop(); result = v; break; } case ExtSlice_kind: { ExtendedSliceAst* v = new ExtendedSliceAst(parent()); nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop(); result = v; break; } case Index_kind: { IndexAst* v = new IndexAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Index.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _slice AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_stmt* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Expr_kind: { ExpressionAst* v = new ExpressionAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Expr.value)); nodeStack.pop(); result = v; break; } case FunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.FunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.FunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.FunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.FunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.FunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.FunctionDef.returns)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.AsyncFunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.AsyncFunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.AsyncFunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.AsyncFunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.AsyncFunctionDef.returns)); nodeStack.pop(); v->async = true; result = v; break; } #endif case ClassDef_kind: { ClassDefinitionAst* v = new ClassDefinitionAst(parent()); v->name = node->v.ClassDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ClassDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->baseClasses = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.bases); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ClassDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.decorator_list); nodeStack.pop(); result = v; break; } case Return_kind: { ReturnAst* v = new ReturnAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Return.value)); nodeStack.pop(); result = v; break; } case Delete_kind: { DeleteAst* v = new DeleteAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Delete.targets); nodeStack.pop(); result = v; break; } case Assign_kind: { AssignmentAst* v = new AssignmentAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Assign.targets); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Assign.value)); nodeStack.pop(); result = v; break; } case AugAssign_kind: { AugmentedAssignmentAst* v = new AugmentedAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AugAssign.target)); nodeStack.pop(); v->op = (ExpressionAst::OperatorTypes) node->v.AugAssign.op; nodeStack.push(v); v->value = static_cast(visitNode(node->v.AugAssign.value)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case AnnAssign_kind: { AnnotationAssignmentAst* v = new AnnotationAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AnnAssign.target)); nodeStack.pop(); nodeStack.push(v); v->annotation = static_cast(visitNode(node->v.AnnAssign.annotation)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.AnnAssign.value)); nodeStack.pop(); result = v; break; } #endif case For_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.For.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.For.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.For.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.For.orelse); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFor_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AsyncFor.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.AsyncFor.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFor.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.AsyncFor.orelse); nodeStack.pop(); result = v; break; } #endif case While_kind: { WhileAst* v = new WhileAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.While.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.While.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.While.orelse); nodeStack.pop(); result = v; break; } case If_kind: { IfAst* v = new IfAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.If.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.If.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.If.orelse); nodeStack.pop(); result = v; break; } case With_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.With.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.With.items); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncWith_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncWith.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.AsyncWith.items); nodeStack.pop(); result = v; break; } #endif case Raise_kind: { RaiseAst* v = new RaiseAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.Raise.exc)); nodeStack.pop(); result = v; break; } case Try_kind: { TryAst* v = new TryAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.Try.body); nodeStack.pop(); nodeStack.push(v); v->handlers = visitNodeList<_excepthandler, ExceptionHandlerAst>(node->v.Try.handlers); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.Try.orelse); nodeStack.pop(); nodeStack.push(v); v->finally = visitNodeList<_stmt, Ast>(node->v.Try.finalbody); nodeStack.pop(); result = v; break; } case Assert_kind: { AssertionAst* v = new AssertionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.Assert.test)); nodeStack.pop(); nodeStack.push(v); v->message = static_cast(visitNode(node->v.Assert.msg)); nodeStack.pop(); result = v; break; } case Import_kind: { ImportAst* v = new ImportAst(parent()); nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.Import.names); nodeStack.pop(); result = v; break; } case ImportFrom_kind: { ImportFromAst* v = new ImportFromAst(parent()); v->module = node->v.ImportFrom.module ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ImportFrom.module)) : nullptr; if ( v->module ) { v->module->startCol = node->col_offset; v->startCol = v->module->startCol; v->module->startLine = tline(node->lineno - 1); v->startLine = v->module->startLine; v->module->endCol = node->col_offset + v->module->value.length() - 1; v->endCol = v->module->endCol; v->module->endLine = tline(node->lineno - 1); v->endLine = v->module->endLine; ranges_copied = true; } nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.ImportFrom.names); nodeStack.pop(); v->level = node->v.ImportFrom.level; result = v; break; } case Global_kind: { GlobalAst* v = new GlobalAst(parent()); for ( int _i = 0; _i < node->v.Global.names->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyUnicodeObjectToQString( static_cast(node->v.Global.names->elements[_i]) )); v->names.append(id); } result = v; break; } case Break_kind: { BreakAst* v = new BreakAst(parent()); result = v; break; } case Continue_kind: { ContinueAst* v = new ContinueAst(parent()); result = v; break; } case Pass_kind: { PassAst* v = new PassAst(parent()); result = v; break; } case Nonlocal_kind: { NonlocalAst* v = new NonlocalAst(parent()); result = v; break; } default: qWarning() << "Unsupported _stmt AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_withitem* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; WithItemAst* v = new WithItemAst(parent()); nodeStack.push(v); v->contextExpression = static_cast(visitNode(node->context_expr)); nodeStack.pop(); - nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->optional_vars)); nodeStack.pop(); + nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->optional_vars)); nodeStack.pop(); return v; } }; /* * End generated code */ diff --git a/parser/python36.sdef b/parser/python36.sdef index fa4308c6..f53ff1c7 100644 --- a/parser/python36.sdef +++ b/parser/python36.sdef @@ -1,102 +1,102 @@ COMMENT;This file specifies rules for a conversion of a python (C) to a plugin-internal (C++) syntax tree;; COMMENT;All lines are terminated by a double semicolon token, newlines are ignored.;; COMMENT;-> stands for "convert AST", => for "convert AST list", ~> for "make an identifier", :> is an assignment w/o conversion, and *> is an enum cast.;; COMMENT;$> is a string assignment, +> is a "mindless assignment, like, it just does v->arg = node->arg. COMMENT;_> is a Singleton Assignment (converts Py_True, Py_False, Py_None to enum) COMMENT;the CODE statement can be used to add custom code.;; RULE_FOR _stmt;KIND Expr_kind;ACTIONS create|ExpressionAst set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND FunctionDef_kind;ACTIONS create|FunctionDefinitionAst set|name~>name set|arguments->ArgumentsAst,args set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list set|returns->ExpressionAst,returns;; RULE_FOR _stmt;KIND AsyncFunctionDef_kind;ACTIONS create|FunctionDefinitionAst set|name~>name set|arguments->ArgumentsAst,args set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list set|returns->ExpressionAst,returns;SINCE 3.5;CODE v->async = true;; RULE_FOR _stmt;KIND ClassDef_kind;ACTIONS create|ClassDefinitionAst set|name~>name set|baseClasses=>ExpressionAst,bases set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list;; RULE_FOR _stmt;KIND Return_kind;ACTIONS create|ReturnAst set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND Delete_kind;ACTIONS create|DeleteAst set|targets=>ExpressionAst,targets;; RULE_FOR _stmt;KIND Assign_kind;ACTIONS create|AssignmentAst set|targets=>ExpressionAst,targets set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND AugAssign_kind;ACTIONS create|AugmentedAssignmentAst set|target->ExpressionAst,target set|op*>OperatorTypes,op set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND AnnAssign_kind;ACTIONS create|AnnotationAssignmentAst set|target->ExpressionAst,target set|annotation->ExpressionAst,annotation set|value->ExpressionAst,value;SINCE 3.6;; RULE_FOR _stmt;KIND For_kind;ACTIONS create|ForAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND AsyncFor_kind;ACTIONS create|ForAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|body=>Ast,body set|orelse=>Ast,orelse;SINCE 3.5;; RULE_FOR _stmt;KIND While_kind;ACTIONS create|WhileAst set|condition->ExpressionAst,test set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND If_kind;ACTIONS create|IfAst set|condition->ExpressionAst,test set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND With_kind;ACTIONS create|WithAst set|body=>Ast,body set|items=>WithItemAst,items;; RULE_FOR _stmt;KIND AsyncWith_kind;ACTIONS create|WithAst set|body=>Ast,body set|items=>WithItemAst,items;SINCE 3.5;; COMMENT;FIXME: the struct Raise has changed, so the following line is likely wrong;; RULE_FOR _stmt;KIND Raise_kind;ACTIONS create|RaiseAst set|type->ExpressionAst,exc;; RULE_FOR _stmt;KIND Try_kind;ACTIONS create|TryAst set|body=>Ast,body set|handlers=>ExceptionHandlerAst,handlers set|orelse=>Ast,orelse set|finally=>Ast,finalbody;; RULE_FOR _stmt;KIND Assert_kind;ACTIONS create|AssertionAst set|condition->ExpressionAst,test set|message->ExpressionAst,msg;; RULE_FOR _stmt;KIND Import_kind;ACTIONS create|ImportAst set|names=>AliasAst,names;; RULE_FOR _stmt;KIND ImportFrom_kind;ACTIONS create|ImportFromAst set|module~>module set|names=>AliasAst,names set|level:>level;; RULE_FOR _stmt;KIND Global_kind;ACTIONS create|GlobalAst set|names=>Identifier,names;; RULE_FOR _stmt;KIND Break_kind;ACTIONS create|BreakAst;; RULE_FOR _stmt;KIND Continue_kind;ACTIONS create|ContinueAst;; RULE_FOR _stmt;KIND Pass_kind;ACTIONS create|PassAst;; RULE_FOR _stmt;KIND Nonlocal_kind;ACTIONS create|NonlocalAst;; RULE_FOR _expr;KIND Await_kind;ACTIONS create|AwaitAst set|value->ExpressionAst,value;SINCE 3.5;; RULE_FOR _expr;KIND BoolOp_kind;ACTIONS create|BooleanOperationAst set|type*>BooleanOperationTypes,op set|values=>ExpressionAst,values;; RULE_FOR _expr;KIND BinOp_kind;ACTIONS create|BinaryOperationAst set|type*>OperatorTypes,op set|lhs->ExpressionAst,left set|rhs->ExpressionAst,right;; RULE_FOR _expr;KIND UnaryOp_kind;ACTIONS create|UnaryOperationAst set|type*>UnaryOperatorTypes,op set|operand->ExpressionAst,operand;; RULE_FOR _expr;KIND Lambda_kind;ACTIONS create|LambdaAst set|arguments->ArgumentsAst,args set|body->ExpressionAst,body;; RULE_FOR _expr;KIND IfExp_kind;ACTIONS create|IfExpressionAst set|condition->ExpressionAst,test set|body->ExpressionAst,body set|orelse->ExpressionAst,orelse;; RULE_FOR _expr;KIND Dict_kind;ACTIONS create|DictAst set|keys=>ExpressionAst,keys set|values=>ExpressionAst,values;; RULE_FOR _expr;KIND Set_kind;ACTIONS create|SetAst set|elements=>ExpressionAst,elts;; RULE_FOR _expr;KIND ListComp_kind;ACTIONS create|ListComprehensionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND SetComp_kind;ACTIONS create|SetComprehensionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND DictComp_kind;ACTIONS create|DictionaryComprehensionAst set|key->ExpressionAst,key set|value->ExpressionAst,value set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND GeneratorExp_kind;ACTIONS create|GeneratorExpressionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND Yield_kind;ACTIONS create|YieldAst set|value->ExpressionAst,value;; RULE_FOR _expr;KIND Compare_kind;ACTIONS create|CompareAst set|leftmostElement->ExpressionAst,left set|operators#>ComparisonOperatorTypes,ops set|comparands=>ExpressionAst,comparators;; RULE_FOR _expr;KIND Call_kind;ACTIONS create|CallAst set|function->ExpressionAst,func set|arguments=>ExpressionAst,args set|keywords=>KeywordAst,keywords;SINCE 3.5;; RULE_FOR _expr;KIND Call_kind;ACTIONS create|CallAst set|function->ExpressionAst,func set|arguments=>ExpressionAst,args set|keywords=>KeywordAst,keywords;BEFORE 3.5; CODE /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */ if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop(); }; if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop(); };; RULE_FOR _expr;KIND Num_kind;ACTIONS create|NumberAst;CODE v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n);; RULE_FOR _expr;KIND Str_kind;ACTIONS create|StringAst set|value$>s;; RULE_FOR _expr;KIND JoinedStr_kind;ACTIONS create|JoinedStringAst set|values=>ExpressionAst,values;SINCE 3.6;; RULE_FOR _expr;KIND FormattedValue_kind;ACTIONS create|FormattedValueAst set|value->ExpressionAst,value set|conversion:>conversion set|formatSpec->ExpressionAst,format_spec;SINCE 3.6;; RULE_FOR _expr;KIND Bytes_kind;ACTIONS create|BytesAst set|value$>s;; RULE_FOR _expr;KIND Attribute_kind;ACTIONS create|AttributeAst set|attribute~>attr set|value->ExpressionAst,value set|context*>Context,ctx;; RULE_FOR _expr;KIND Subscript_kind;ACTIONS create|SubscriptAst set|value->ExpressionAst,value set|slice->SliceAst,slice set|context*>Context,ctx;; RULE_FOR _expr;KIND Starred_kind;ACTIONS create|StarredAst set|value->ExpressionAst,value set|context*>Context,ctx;; RULE_FOR _expr;KIND Name_kind;ACTIONS create|NameAst set|identifier~>id set|context*>Context,ctx;; RULE_FOR _expr;KIND List_kind;ACTIONS create|ListAst set|elements=>ExpressionAst,elts set|context*>Context,ctx;; RULE_FOR _expr;KIND Tuple_kind;ACTIONS create|TupleAst set|elements=>ExpressionAst,elts set|context*>Context,ctx;; RULE_FOR _expr;KIND Ellipsis_kind;ACTIONS create|EllipsisAst;; RULE_FOR _expr;KIND NameConstant_kind;ACTIONS create|NameConstantAst set|value_>value;; RULE_FOR _expr;KIND YieldFrom_kind;ACTIONS create|YieldFromAst set|value->ExpressionAst,value;; RULE_FOR _slice;KIND Slice_kind;ACTIONS create|SliceAst set|lower->ExpressionAst,lower set|upper->ExpressionAst,upper set|step->ExpressionAst,step;; RULE_FOR _slice;KIND ExtSlice_kind;ACTIONS create|ExtendedSliceAst set|dims=>SliceAst,dims;; RULE_FOR _slice;KIND Index_kind;ACTIONS create|IndexAst set|value->ExpressionAst,value;; RULE_FOR _comprehension;KIND any;ACTIONS create|ComprehensionAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|conditions=>ExpressionAst,ifs;; RULE_FOR _excepthandler;KIND ExceptHandler_kind;ACTIONS create|ExceptionHandlerAst set|type->ExpressionAst,type set|name~>name set|body=>Ast,body;; RULE_FOR _arguments;KIND any;ACTIONS create|ArgumentsAst set|vararg->ArgAst,vararg set|kwarg->ArgAst,kwarg set|arguments=>ArgAst,args set|defaultValues=>ExpressionAst,defaults set|kwonlyargs=>ArgAst,kwonlyargs;; RULE_FOR _arg;KIND any;ACTIONS create|ArgAst set|argumentName~>arg set|annotation->ExpressionAst,annotation;; RULE_FOR _keyword;KIND any;ACTIONS create|KeywordAst set|argumentName~>arg set|value->ExpressionAst,value;; RULE_FOR _alias;KIND any;ACTIONS create|AliasAst set|name~>name set|asName~>asname;; -RULE_FOR _withitem;KIND any; ACTIONS create|WithItemAst set|contextExpression->ExpressionAst,context_expr set|optionalVars->NameAst,optional_vars;; +RULE_FOR _withitem;KIND any; ACTIONS create|WithItemAst set|contextExpression->ExpressionAst,context_expr set|optionalVars->ExpressionAst,optional_vars;;