diff --git a/documentation_files/builtindocumentation.py b/documentation_files/builtindocumentation.py index 7d3f4598..2d4094be 100644 --- a/documentation_files/builtindocumentation.py +++ b/documentation_files/builtindocumentation.py @@ -1,665 +1,677 @@ # This file denotes the built-in python library. It is imported into every file which is parsed. class Exception(object): pass class object(): def __init__(self): pass def __new__(self): pass def __del__(self): pass def __repr__(self): pass def __str__(self): pass def __lt__(self, other): pass def __gt__(self, other): pass def __le__(self, other): pass def __eq__(self, other): pass def __ne__(self, other): pass def __gt__(self, other): pass def __ge__(self, other): pass def __cmp__(self, other): pass def __hash__(self): pass def __nonzero__(self): pass def __unicode__(self): pass def __getattr__(self, name): pass def __setattr__(self, name, value): pass def __delattr__(self, name): pass def __getattribute__(self, name): pass def __get__(self, instance, owner): pass def __set__(self, instance, value): pass def __delete__(self, instance): pass def __instancecheck__(self, instance): pass def __subclasscheck__(self, subclass): pass def __call__(self): pass def __len__(self): pass def __getitem__(self, key): pass def __setitem__(self, key, value): pass def __delitem__(self, key): pass def __iter__(self): pass def __reversed__(self): pass def __contains__(self, item): pass def __getslice__(self, i, j): pass def __delslice__(self, i, j): pass def __add__(self, other): pass def __sub__(self, other): pass def __mul__(self, other): pass def __matmul__(self, other): pass def __rmatmul__(self, other): pass def __imatmul__(self, other): pass def __floordiv__(self, other): pass def __mod__(self, other): pass def __divmod__(self, other): pass def __pow__(self, other): pass def __lshift__(self, other): pass def __rshift__(self, other): pass def __and__(self, other): pass def __xor__(self, other): pass def __or__(self, other): pass def __div__(self, other): pass def __truediv__(self, other): pass def __radd__(self, other): pass def __rsub__(self, other): pass def __rmul__(self, other): pass def __rtruediv__(self, other): pass def __rfloordiv__(self, other): pass def __rmod__(self, other): pass def __rdivmod__(self, other): pass def __rpow__(self, other): pass def __rlshift__(self, other): pass def __rrshift__(self, other): pass def __rand__(self, other): pass def __rxor__(self, other): pass def __ror__(self, other): pass def __iadd__(self, other): pass def __isub__(self, other): pass def __imul__(self, other): pass def __idiv__(self, other): pass def __itruediv__(self, other): pass def __ifloordiv__(self, other): pass def __imod__(self, other): pass def __ipow__(self, other): pass def __ilshift__(self, other): pass def __irshift__(self, other): pass def __iand__(self, other): pass def __ixor__(self, other): pass def __ior__(self, other): pass def __neg__(self): pass def __pos__(self): pass def __abs__(self): pass def __invert__(self): pass def __complex__(self): pass def __int__(self): pass def __long__(self): pass def __float__(self): pass def __oct__(self): pass def __hex__(self): pass def __index__(self): pass def __coerce__(self, other): pass __class__ = str() class basestring(): pass @TypeContainer class list(): """! TypeContainer !""" def __init__(self, items): """! returnContentEqualsContentOf ! 0""" return [] def __setitem__(self, key, value): """! addsTypeOfArg ! 1 ! getsType !""" def __getitem__(self, key): """! addsTypeOfArg ! 0""" def append(self,obj): """! addsTypeOfArg ! 0""" def extend(self,obj): """! addsTypeOfArgContent ! 0""" return None def insert(self,i, x): """! getsType !""" return None def pop(self,i): """! getsType !""" return None def index(self,x): return 0 def count(self,x): """! getsList !""" return 0 def sort(self,): """! getsList !""" return [] def reverse(self,): """! getsList !""" return [] def remove(self, x): pass class _io_TextIOWrapper(): def close(self,): return None def flush(self,): return None def fileno(self,): return None def isatty(self,): return True def next(self,): return None def read(self,size = 0): return "" def readline(self,size = 0): return "" def readlines(self,sizehint = 0): return [""] def xreadlines(self,): return None def seek(self,offset, whence = 0): return None def tell(self,): return None def truncate(self,size = 0): return None def write(self,string): return None def writelines(self,sequence): return None closed = True errors = None mode = None name = "" newlines = "" softspace = True class dict(): """! TypeContainer ! ! hasTypedKeys !""" def __init__(self, items): """! returnContentEqualsContentOf ! 0""" return {} def __setitem__(self, key, value): """! addsTypeOfArg ! 1 ! addsKeyTypeOfArg ! 0""" def __getitem__(self, key): """! getsType !""" def clear(self,): return None def copy(self,): return {} def fromkeys(self,seq, value = None): """! addsKeyTypeOfArgContent ! 0 ! addsTypeOfArg ! 1""" return {} def get(self,key, default = ""): """! getsType !""" return None def has_key(self,key): return True def items(self,): """! getsListOfBoth !""" return {} def iteritems(self,): """! getsListOfBoth !""" return [] def iterkeys(self,): """! getsListOfKeys !""" return [] def itervalues(self,): """! getsList !""" return [] def keys(self,): """! getsListOfKeys !""" return [] def pop(self,key, default = ""): """! getsType !""" return None def popitem(self,): """! getsBoth !""" return None def setdefault(self,key, default = ""): return None def update(self,other = None): return None def values(self,): """! getsList !""" return [] def viewitems(self,): return None def viewkeys(self,): return None def viewvalues(self,): return None class str(): def __init__(self, obj): pass def __mod__(self, modulo): return str() def __getitem__(self): return "" def __iter__(self): return self def __next__(self): return "" def replace(self,before, after): return "" def capitalize(self,): return "" def center(self,width, fillchar = None): return "" def count(self,substring, start = 0, end = 0): return 0 def encode(self, encoding): return bytes() def endswith(self,suffix, start = 0, end = 0): return True def expandtabs(self,tabsize = 0): return "" def find(self,substring, start = 0, end = 0): return 0 def format(self,*args, **kwargs): return "" def index(self,substring, start = 0, end = 0): return 0 def isalnum(self,): return True def isalpha(self,): return True def isdigit(self,): return True def islower(self,): return True def isspace(self,): return True def istitle(self,): return True def isupper(self,): return True def join(self,iterable): return "" def ljust(self,width, fillchar = ""): return "" def lower(self,): return "" def lstrip(self,chars = ""): return "" def partition(self,seperator): return ("", "", "") def replace(self,old, new, count = 0): return "" def rfind(self,substring, start = 0, end = 0): return 0 def rindex(self,substring, start = 0, end = 0): return 0 def rjust(self,width, fillchar = ""): return "" def rpartition(self,seperator): return ("", "", "") def rsplit(self,seperator = "", maxsplit = 0): return [] def rstrip(self,chars = ""): return "" def split(self,seperator = "", maxsplit = 0): return ["string"] def splitlines(self,keepends = False): return ["string"] def startswith(self,prefix, start = 0, end = 0): return True def strip(self,chars = ""): return "" def swapcase(self,): return "" def title(self,): return "" def translate(self,table, deletechars = ""): return "" def upper(self,): return "" def zfill(self,width): return "" class float(): def bit_length(self,): return 0 def as_integer_ration(self,): return (self,0, 0) def is_integer(self,): return True def hex(self,): return 0x0 def fromhex(self,s): return 0 def __add__(self, other): return float() def __sub__(self, other): return float() def __mul__(self, other): return float() def __div__(self, other): return float() class int(): def __add__(self, other): return int() def __sub__(self, other): return int() def __mul__(self, other): return int() def __div__(self, other): return float() class complex(): real = 3 imag = 5 def __add__(self, other): return complex() def __sub__(self, other): return complex() def __mul__(self, other): return complex() def __div__(self, other): return complex() def __mod__(self, other): return complex() class BaseException(): args = () class NameError(BaseException): pass class AttributeError(BaseException): pass class IndexError(BaseException): pass class IOError(BaseException): pass class StandardError(BaseException): pass class Exception(BaseException): pass class StopIteration(BaseException): pass class StopAsyncIteration(BaseException): pass class BufferError(BaseException): pass class LookupError(BaseException): pass class EnvironmentError(BaseException): pass class AssertionError(BaseException): pass class EOFError(BaseException): pass class FloatingPointError(BaseException): pass class GeneratorExit(BaseException): pass class ImportError(BaseException): pass class KeyError(BaseException): pass class MemoryError(BaseException): pass class NotImplementedError(BaseException): pass class OSError(BaseException): pass class OverflowError(BaseException): pass class ReferenceError(BaseException): pass class RuntimeError(BaseException): pass class StopIteration(BaseException): pass class SyntaxError(BaseException): pass class IndentationError(BaseException): pass class TabError(BaseException): pass class SystemError(BaseException): pass class SystemExit(BaseException): pass class TypeError(BaseException): pass class UnboundLocalError(BaseException): pass class UnicodeError(BaseException): pass class UnicodeEncodeError(BaseException): pass class UnicodeDecodeError(BaseException): pass class UnicodeTranslateError(BaseException): pass class ValueError(BaseException): pass class ZeroDivisionError(BaseException): pass class Warning(): pass class UserWarning(Warning): pass class DeprecationWarning(Warning): pass class KeyboardInterrupt(Exception): pass class PendingDeprecationWarning(Warning): pass class RuntimeWarning(Warning): pass class FutureWarning(Warning): pass class ImportWarning(Warning): pass class UnicodeWarning(Warning): pass class tuple(): """! IndexedTypeContainer !""" def __mul__(self, other): return tuple() class bytes: def __init__(self, data): pass def __getitem__(self, key): return int() def __iter__(self): return self def __next__(self): return int() def capitalize(self): return bytes() def center(self): return bytes() def count(self): return int() def decode(self, encoding): return str() def endswith(self, data): return True def expandtabs(self): return bytes() def find(self): return int() def fromhex(self, hexdata): return bytes() def index(self): return int() def isalnum(self): return True def isalpha(self): return True def isdigit(self): return True def islower(self): return True def isspace(self): return True def istitle(self): return True def isupper(self): return True def join(self, other): return bytes() def ljust(self, space): return bytes(); def lower(self): return bytes() def lstrip(self): return bytes() def maketrans(self, frm, to): return bytes() def partition(self, separator): return (bytes(), bytes(), bytes()) def replace(self, find, replace): return bytes() def rfind(self, data): return int() def rindex(self, data): return int() def rjust(self, justify): return bytes() def rpartition(self, separator): return (bytes(), bytes(), bytes()) def rsplit(self, separator): return [bytes()] def rstrip(self): return bytes() def split(self, separator): return [bytes()] def splitlines(self): return [bytes()] def startswith(self): return False def strip(self): return bytes() def swapcase(self): return bytes() def title(self): return bytes() def translate(self, table, deletechars=None): return bytes() def upper(self): return bytes() def zfill(self, width): return bytes() class set(): """! TypeContainer !""" - def __init__(self, objects): + def __init__(self, iterable): """! returnContentEqualsContentOf ! 0""" pass def len(self): return 0 def isdisjoint(self, other): return True def issubset(self, other): return True def issuperset(self, other): return True def union(self, other, *others): return set() def intersection(self, other, *others): return set() def difference(self, other, *others): return set() def symmetric_difference(self, other): return set() def copy(self): return set() def update(self, other): pass def intersection_update(self, other, *others): pass def difference_update(self, other, *others): pass def symmetric_difference_update(self, other, *others): pass def add(self, elem): pass def remove(self, elem): pass def discard(self, elem): pass def pop(self): pass def clear(self): pass +class frozenset(): + """! TypeContainer !""" + def __init__(self, iterable): + """! returnContentEqualsContentOf ! 0""" + pass + def len(self): return 0 + def isdisjoint(self, other): return True + def issubset(self, other): return True + def issuperset(self, other): return True + def union(self, other, *others): return set() + def intersection(self, other, *others): return set() + def difference(self, other, *others): return set() + def symmetric_difference(self, other): return set() + def copy(self): return set() + def abs(x): """ Return the absolute value of a number. The argument may be a plain or long integer or a floating point number. If the argument is a complex number, its magnitude is returned.""" return 0 def int(x): return 0 def all(iterable): return True def any(iterable): """Return True if any element of the iterable is true. If the iterable is empty, return False.""" return True def bin(x): """Convert an integer number to a binary string. The result is a valid Python expression. If x is not a Python int object, it has to define an __index__() method that returns an integer.""" return "" def bool(x = False): """Convert a value to a Boolean, using the standard truth testing procedure. If x is false or omitted, this returns False; otherwise it returns True. bool is also a class, which is a subclass of int. Class bool cannot be subclassed further. Its only instances are False and True.""" return True def bytearray(source = None, encoding = None, errors = None): """Return a new array of bytes. The bytearray type is a mutable sequence of integers in the range 0 <= x < 256.""" return bytes() def callable(object): """Return True if the object argument appears callable, False if not. If this returns true, it is still possible that a call fails, but if it is false, calling object will never succeed. Note that classes are callable (calling a class returns a new instance); class instances are callable if they have a __call__() method.""" return True def chr(i): """Return a string of one character whose ASCII code is the integer i. For example, chr(97) returns the string 'a'. This is the inverse of ord(). The argument must be in the range [0..255], inclusive; ValueError will be raised if i is outside that range. See also unichr().""" return "" def classmethod(function): """Return a class method for function.""" return None def cmp(x, y): """Compare the two objects x and y and return an integer according to the outcome. The return value is negative if x < y, zero if x == y and strictly positive if x > y.""" return 0 def compile(source, filename, mode, flags = None, dont_inherit = None): """Compile the source into a code or AST object. Code objects can be executed by an exec statement or evaluated by a call to eval(). source can either be a Unicode string, a Latin-1 encoded string or an AST object. Refer to the ast module documentation for information on how to work with AST objects.""" return None def delattr(obj, name): """This is a relative of setattr(). The arguments are an object and a string. The string must be the name of one of the object’s attributes. The function deletes the named attribute, provided the object allows it. For example, delattr(x, 'foobar') is equivalent to del x.foobar.""" return None def dir(obj = None): """Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object.""" return {"string" : None} def divmod(a, b): """Take two (non complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using long division.""" return 0 def enumerate(sequence, start = 0): """Return an enumerate object. sequence must be a sequence, an iterator, or some other object which supports iteration. ! enumerate ! 0 """ return [(0, 0)] def eval(expression, glob = None, loc = None): """The expression argument is parsed and evaluated as a Python expression (technically speaking, a condition list) using the globals and locals dictionaries as global and local namespace.""" return None def execfile(filename, glob = None, loc = None): """This function is similar to the exec statement, but parses a file instead of a string. It is different from the import statement in that it does not use the module administration — it reads the file unconditionally and does not create a new module. """ return None def file(filename, mode = None, bufsize = None): """Constructor function for the file type, described further in section File Objects.""" return _io_TextIOWrapper() def filter(function, iterable): """Construct a list from those elements of iterable for which function returns true. iterable may be either a sequence, a container which supports iteration, or an iterator. If iterable is a string or a tuple, the result also has that type; otherwise it is always a list. If function is None, the identity function is assumed, that is, all elements of iterable that are false are removed.""" return [] def float(x = 0): """Convert a string or a number to floating point.""" return 0.0 def format(value, format_spec = None): """Convert a value to a “formatted” representation, as controlled by format_spec.""" return "" -def frozenset(iterable = None): - """Return a new frozenset object, optionally with elements taken from iterable.""" - return set() def getattr(obj, name, default = None): """Return the value of the named attribute of object. name must be a string.""" return None def globals(): """Return a dictionary representing the current global symbol table.""" return {} def hasattr(obj, name): """The arguments are an object and a string. The result is True if the string is the name of one of the object’s attributes, False if not. (This is implemented by calling getattr(object, name) and seeing whether it raises an exception or not.)""" return bool def hash(obj): """Return the hash value of the object (if it has one).""" return 0 def hex(x): """Convert an integer number (of any size) to a hexadecimal string.""" return 0x0 def id(obj): """Return the “identity” of an object. This is an integer (or long integer) which is guaranteed to be unique and constant for this object during its lifetime. Two objects with non-overlapping lifetimes may have the same id() value.""" return 0 def input(prompt = None): """Equivalent to eval(raw_input(prompt)).""" return None def isinstance(obj, cls): """Return true if the object argument is an instance of the classinfo argument, or of a (direct, indirect or virtual) subclass thereof.""" return True def issubclass(cls, info): """Return true if class is a subclass (direct, indirect or virtual) of classinfo.""" return True def iter(o, s = None): """Return an iterator object. ! returnContentEqualsContentOf ! 0 """ return [] def len(s): """Return the length (the number of items) of an object. The argument may be a sequence (string, tuple or list) or a mapping (dictionary).""" return 0 def locals(): """Update and return a dictionary representing the current local symbol table.""" return {} def long(x = None, base = None): """Convert a string or number to a long integer.""" return 0 def map(func, iterab): """Apply function to every item of iterable and return a list of the results.""" return [] def max(lst, args = None, key = None): """Return the largest item in an iterable or the largest of two or more arguments.""" return 0 def memoryview(obj): """Return a “memory view” object created from the given argument.""" return None def min(lst, default = None): """Return the smallest item in an iterable or the smallest of two or more arguments.""" return 0 def next(iterator, default = None): """Retrieve the next item from the iterator by calling its next() method.""" return iterator[0] def oct(x): """Convert an integer number (of any size) to an octal string.""" return 0o0 def open(filename, mode = None, bufsize = None): """Open a file, returning an object of the file type described in section File Objects.""" return _io_TextIOWrapper() def ord(c): """Given a string of length one, return an integer representing the Unicode code point of the character when the argument is a unicode object, or the value of the byte when the argument is an 8-bit string.""" return 0 def pow(x, y, z = 0): """Return x to the power y; if z is present, return x to the power y, modulo z.""" return 0.0 def property(fget = 0, fset = 0, fdel = 0, doc = 0): """Return a property attribute for new-style classes (classes that derive from object).""" return 0 def range(start = 0, stop = 0, step = 0): """This is a versatile function to create lists containing arithmetic progressions. It is most often used in for loops. The arguments must be plain integers.""" return [0] def raw_input(prompt = ""): """ The function then reads a line from input, converts it to a string (stripping a trailing newline), and returns that.""" return "" def reduce(function, iterable, init = None): """Apply function of two arguments cumulatively to the items of iterable, from left to right, so as to reduce the iterable to a single value.""" return None def reload(module): """Reload a previously imported module.""" return None def repr(object): """Return a string containing a printable representation of an object.""" return "" def reversed(seq): """Return a reverse iterator. ! returnContentEqualsContentOf ! 0""" return None def round(x, n=0): """Return the floating point value number rounded to ndigits digits after the decimal point.""" return 0.0 def setattr(obj, name, value): """This is the counterpart of getattr(). The arguments are an object, a string and an arbitrary value. The string may name an existing attribute or a new attribute. The function assigns the value to the attribute, provided the object allows it.""" return None def slice(start = 0, stop = 0, step = 0): """Return a slice object representing the set of indices specified by range(start, stop, step).""" return slice() def sorted(iterable, cmpre = None, key = None, reverse = False): """Return a new sorted list from the items in iterable. ! returnContentEqualsContentOf ! 0""" return [] def staticmethod(function): """Return a static method for function.""" return function def sum(iterable): """Sums start and the items of an iterable from left to right and returns the total.""" return 0.0 def super(_type, obj = None): """Return a proxy object that delegates method calls to a parent or sibling class of type.""" return None def tuple(iterable = None): """Return a tuple whose items are the same and in the same order as iterable‘s items.""" return () def type(object): """With one argument, return the type of an object.""" return object def unichr(i): """Return the Unicode string of one character whose Unicode code is the integer i.""" return "" def unicode(obj = None, encoding = None, errors = None): """Return the Unicode string version of object.""" return "" def vars(obj): """Return the __dict__ attribute for a module, class, instance, or any other object with a __dict__ attribute.""" return None def xrange(start = 0, stop = 0, step = 0): """This function is very similar to range(), but returns an xrange object instead of a list.""" return [0] def zip(iterable = None): """This function returns a list of tuples, where the i-th tuple contains the i-th element from each of the argument sequences or iterables.""" return [] def __import__(name, globa = None, loca = None, fromlist = None, level = 0): """This function is invoked by the import statement. It can be replaced (by importing the __builtin__ module and assigning to __builtin__.__import__) in order to change semantics of the import statement, but nowadays it is usually simpler to use import hooks (see PEP 302).""" return None def exit(status): return None __name__ = "none" __file__ = "none" __doc__ = "none" __package__ = "none" NotImplemented = None def print(obj, sep='', end='\n', file=open()): pass diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index 7d691772..509537fc 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1923 +1,1923 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/decorator.h" #include "duchain/declarations/functiondeclaration.h" #include "duchain/declarations/classdeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); qCDebug(KDEV_PYTHON_DUCHAIN) << "Building Declarations"; } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "building, but running pre-builder first"; DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); qCDebug(KDEV_PYTHON_DUCHAIN) << "pre-builder finished"; delete prebuilder; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "prebuilding"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, Ast* range, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = 0; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(range, range), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name, range); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. QList declaringContexts; declaringContexts << ExpressionAst::Store << ExpressionAst::Parameter << ExpressionAst::AugStore; if ( ! declaringContexts.contains(currentVariableDefinition->context) ) { return 0; } Identifier* id = currentVariableDefinition->identifier; return visitVariableDeclaration(id, currentVariableDefinition, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), nullptr, previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(0); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { QList existingDeclarations = currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); // append arguments context if ( m_mostRecentArgumentsContext ) { QList args = m_mostRecentArgumentsContext->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent ); existingDeclarations.append(args); } return existingDeclarations; } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = 0; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != 0 ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Ast* originalAst, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; Ast* rangeNode = originalAst ? originalAst : node; RangeInRevision range = editorFindRange(rangeNode, rangeNode); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = 0; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << hints->toString(); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, 0, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); visitVariableDeclaration(node->optionalVars, 0, v.lastType()); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = 0; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), 0, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return 0; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = 0; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(0); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(0); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void spoofNodePosition(Ast* node, const CursorInRevision& pos) { // Ridiculous hack, see next comment. node->startLine = node->endLine = pos.line; node->startCol = node->endCol = pos.column - 1; if (node->astType == Ast::TupleAstType) { // Recursion to bodge all declarations, e.g. // [x + y * z for x, (y, z) in foo] foreach(auto elem, static_cast(node)->elements) { spoofNodePosition(elem, pos); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); // make the declaration zero chars long; it must appear at the beginning of the context, // because it is actually used *before* its real declaration: [foo for foo in bar] // The DUChain doesn't like this, so for now, the declaration is at the opening bracket, // and both other occurrences are uses of that declaration. // TODO add a special case to the usebuilder to display the second occurrence as a declaration spoofNodePosition(node->target, currentContext()->range().start); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(0); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = 0; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = 0; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = 0; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, &temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return 0; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = 0; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return 0; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return 0; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, 0, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, 0); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(0), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { Python::AstDefaultVisitor::visitLambda(node); DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } if (node->arguments->vararg) { visitVariableDeclaration(node->arguments->vararg->argumentName); } if (node->arguments->kwarg) { visitVariableDeclaration(node->arguments->kwarg->argumentName); } closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { - const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0; + const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { - const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0; + const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker wlock; auto contentType = Helper::contentOfIterable(argVisitor.lastType(), topContext()); container->addContentType(contentType); v.lastDeclaration()->setType(container); } }; auto docstring = function->comment(); if ( ! docstring.isEmpty() ) { foreach ( const auto& key, items.keys() ) { if ( Helper::docstringContainsHint(docstring, key, &args) ) { items[key](); } } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer called) { DUChainReadLocker lock; auto funcInfo = Helper::functionForCalled(called.data()); auto function = funcInfo.declaration; if ( ! function ) { return; } if ( function->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } // Note: within this function: // - 'parameters' refers to the parameters of the function definition. // - 'arguments' refers to the arguments of the function call. DUContext* parameterContext = DUChainUtils::getArgumentContext(function); FunctionType::Ptr functionType = function->type(); if ( ! parameterContext || ! functionType ) { return; } QVector parameters = parameterContext->localDeclarations(); if ( parameters.isEmpty() ) { return; } const int specialParamsCount = (function->vararg() != -1) + (function->kwarg() != -1); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfParam = false; if ( ( function->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! function->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as parameters // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfParam = true; } } lock.unlock(); bool explicitSelfArgument = false; if ( hasSelfParam && ! function->isClassMethod() && node->function->astType == Ast::AttributeAstType ) { // Calling an attribute, e.g. `instance.foo(arg)` or `MyClass.foo(instance, arg)`. ExpressionVisitor valueVisitor(currentContext()); valueVisitor.visitNode(static_cast(node->function)->value); if ( valueVisitor.lastDeclaration().dynamicCast() && valueVisitor.isAlias() ) { // Function is attribute of a class _type_ (not instance), so first arg is used as `self`. explicitSelfArgument = true; } } int currentParamIndex = hasSelfParam; int currentArgumentIndex = explicitSelfArgument; int indexInVararg = -1; int paramsAvailable = qMin(functionType->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentArgumentIndex++ ) { atVararg = atVararg || currentParamIndex == function->vararg(); // Not >=, nonexistent vararg is -1. qCDebug(KDEV_PYTHON_DUCHAIN) << currentParamIndex << currentArgumentIndex << atVararg << function->vararg(); ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(function->vararg()); IndexedContainer::Ptr varargContainer = parameter->type(); qCDebug(KDEV_PYTHON_DUCHAIN) << "adding" << addType->toString() << "at position" << indexInVararg; if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functionType->removeArgument(currentParamIndex); functionType->addArgument(newType, currentParamIndex); function->setAbstractType(functionType.cast()); parameters.at(currentParamIndex)->setType(newType); currentParamIndex++; } } // **kwargs is always the last parameter MapType::Ptr kwargsDict; if ( function->kwarg() != -1 ) { kwargsDict = parameters.last()->abstractType().cast(); } lock.unlock(); DUChainWriteLocker wlock; foreach ( KeywordAst* keyword, node->keywords ) { wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); bool matchedNamedParam = false; HintedType::Ptr addType = HintedType::Ptr(new HintedType()); if ( keyword->argumentName ) { openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); for (int ip = currentParamIndex; ip < paramsAvailable; ++ip ) { if ( parameters.at(ip)->identifier().toString() != keyword->argumentName->value ) { continue; } matchedNamedParam = true; auto newType = Helper::mergeTypes(parameters.at(ip)->abstractType(), addType); functionType->removeArgument(ip); functionType->addArgument(newType, ip); parameters.at(ip)->setType(newType); } } else if ( auto unpackedDict = argumentVisitor.lastType().cast() ) { // 'keyword is actually an unpacked dict: `foo(**{'a': 12}). openType(addType); addType->setType(unpackedDict->contentType().abstractType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); } else { // Maybe the dict type wasn't loaded yet, or something else happened. continue; } if ( ! matchedNamedParam && kwargsDict ) { kwargsDict->addContentType(addType); parameters.last()->setAbstractType(kwargsDict); } } function->setAbstractType(functionType); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; Python::Identifier* identifier = target->identifier; AliasDeclaration* decl = eventuallyReopenDeclaration(identifier, target, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, 0, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(0); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreayOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } else qCWarning(KDEV_PYTHON_DUCHAIN) << "No declaration created for " << attrib->attribute << "as parent is not a class"; closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 || (starred == -1 && spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, NoTypeRequired); lock.unlock(); visitDecorators(node->decorators, dec); lock.lock(); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); auto docstring = getDocstring(node->body); dec->setComment(docstring); if ( ! docstring.isEmpty() ) { // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(docstring, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(docstring, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(docstring, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassType = type; // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); foreach ( Ast* _node, node->body ) { AstDefaultVisitor::visitNode(_node); } lock.lock(); closeContext(); closeType(); closeDeclaration(); } template void DeclarationBuilder::visitDecorators(QList< Python::ExpressionAst* > decorators, T* addTo) { foreach ( ExpressionAst* decorator, decorators ) { AstDefaultVisitor::visitNode(decorator); if ( decorator->astType == Ast::CallAstType ) { CallAst* call = static_cast(decorator); Decorator d; if ( call->function->astType != Ast::NameAstType ) { continue; } d.setName(*static_cast(call->function)->identifier); foreach ( ExpressionAst* arg, call->arguments ) { if ( arg->astType == Ast::NumberAstType ) { d.setAdditionalInformation(QString::number(static_cast(arg)->value)); } else if ( arg->astType == Ast::StringAstType ) { d.setAdditionalInformation(static_cast(arg)->value); } break; // we only need the first argument for documentation analysis } addTo->addDecorator(d); } else if ( decorator->astType == Ast::NameAstType ) { NameAst* name = static_cast(decorator); Decorator d; d.setName(*(name->identifier)); addTo->addDecorator(d); } } } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); visitDecorators(node->decorators, dec); const bool isStatic = Helper::findDecoratorByName(dec, "staticmethod"); const bool isClassMethod = Helper::findDecoratorByName(dec, "classmethod"); dec->setStatic(isStatic); dec->setClassMethod(isClassMethod); visitFunctionArguments(node); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } dec->setType(type); } if ( ! isStatic ) { DUContext* args = DUChainUtils::getArgumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! isClassMethod ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); qCDebug(KDEV_PYTHON_DUCHAIN) << "updated function return type to " << type->toString(); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); if ( node->value ) { if ( ! hasCurrentType() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } else { TypePtr t = currentType(); AbstractType::Ptr encountered = v.lastType(); DUChainWriteLocker lock; if ( t ) { // Update the containing function's return type t->setReturnType(Helper::mergeTypes(t->returnType(), encountered)); } } } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; qCDebug(KDEV_PYTHON_DUCHAIN) << "arguments:" << node->arguments.size(); foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } qCDebug(KDEV_PYTHON_DUCHAIN) << "visiting argument:" << arg->argumentName->value; // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && workingOnDeclaration->isClassMethod() ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, arg, AliasDeclarationType); if ( m_currentClassType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting declaration:" << m_currentClassType->declaration(topContext())->toString(); decl->setAliasedDeclaration(m_currentClassType->declaration(currentContext()->topContext())); } closeDeclaration(); paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } qCDebug(KDEV_PYTHON_DUCHAIN) << "is first:" << isFirst << hasCurrentDeclaration() << currentDeclaration(); if ( isFirst && ! workingOnDeclaration->isStatic() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassType.cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); if ( argumentType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "creating argument with type" << argumentType->toString(); } } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, 0, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, 0, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/expressionvisitor.cpp b/duchain/expressionvisitor.cpp index f048f351..726ad4bd 100644 --- a/duchain/expressionvisitor.cpp +++ b/duchain/expressionvisitor.cpp @@ -1,720 +1,720 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2010 Miquel Canes Gonzalez * * Copyright 2011-2013 by Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "expressionvisitor.h" #include "types/indexedcontainer.h" #include "declarations/functiondeclaration.h" #include "pythonduchainexport.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include #include using namespace KDevelop; using namespace Python; using namespace KTextEditor; namespace Python { QHash ExpressionVisitor::m_defaultTypes; AbstractType::Ptr ExpressionVisitor::encounterPreprocess(AbstractType::Ptr type) { return Helper::resolveAliasType(type); } ExpressionVisitor::ExpressionVisitor(const DUContext* ctx) : DynamicLanguageExpressionVisitor(ctx) { ENSURE_CHAIN_NOT_LOCKED if ( m_defaultTypes.isEmpty() ) { m_defaultTypes.insert(NameConstantAst::True, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::False, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::None, AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } Q_ASSERT(context()); Q_ASSERT(context()->topContext()); } ExpressionVisitor::ExpressionVisitor(ExpressionVisitor* parent, const DUContext* overrideContext) : DynamicLanguageExpressionVisitor(parent) , m_forceGlobalSearching(parent->m_forceGlobalSearching) , m_reportUnknownNames(parent->m_reportUnknownNames) , m_scanUntilCursor(parent->m_scanUntilCursor) { ENSURE_CHAIN_NOT_LOCKED if ( overrideContext ) { m_context = overrideContext; } Q_ASSERT(context()); } void ExpressionVisitor::encounter(AbstractType::Ptr type, DeclarationPointer declaration, bool alias) { setLastIsAlias(alias); DynamicLanguageExpressionVisitor::encounter(type, declaration); } void ExpressionVisitor::visitAttribute(AttributeAst* node) { ExpressionVisitor v(this); v.visitNode(node->value); setConfident(false); // Find a matching declaration which is made inside the type of the accessed object. // Like, for B.C where B is an instance of foo, find a property of foo called C. DUChainReadLocker lock; auto attribute = Helper::accessAttribute(v.lastType(), node->attribute->value, topContext()); if ( auto resolved = Helper::resolveAliasDeclaration(attribute) ) { encounter(attribute->abstractType(), DeclarationPointer(attribute)); setLastIsAlias(dynamic_cast(attribute) || resolved->isFunctionDeclaration() || dynamic_cast(resolved)); } else { encounterUnknown(); } } void ExpressionVisitor::visitCall(CallAst* node) { foreach ( ExpressionAst* c, node->arguments ) { AstDefaultVisitor::visitNode(c); } ExpressionVisitor v(this); v.visitNode(node->function); auto declaration = Helper::resolveAliasDeclaration(v.lastDeclaration().data()); if ( ! v.isAlias() && v.lastType() ) { if ( auto functionType = v.lastType().cast() ) { encounter(functionType->returnType()); return; } if ( auto classType = v.lastType().cast() ) { declaration = classType->declaration(topContext()); } } if ( ! declaration ) { encounterUnknown(); return; } ClassDeclaration* classDecl = dynamic_cast(declaration); DUChainReadLocker lock; auto function = Helper::functionForCalled(declaration, v.isAlias()); lock.unlock(); AbstractType::Ptr type; Declaration* decl; if ( function.isConstructor && classDecl ) { // Don't use return type from constructor. // It's wrong for builtins, or classes without their own __init__ methods(). type = classDecl->abstractType(); decl = classDecl; } else if ( function.declaration && function.declaration->type() ) { // But do use the return value of normal functions or __call__(). type = function.declaration->type()->returnType(); decl = function.declaration; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration is not a class or function declaration"; encounterUnknown(); return; } if ( function.declaration ) { auto docstring = function.declaration->comment(); if ( ! docstring.isEmpty() ) { // Our documentation data uses special docstrings that override the return type // of some functions (including constructors). type = docstringTypeOverride(node, type, docstring); } } encounter(type, DeclarationPointer(decl)); } AbstractType::Ptr ExpressionVisitor::docstringTypeOverride( CallAst* node, const AbstractType::Ptr normalType, const QString& docstring) { auto docstringType = normalType; auto listOfTuples = [&](AbstractType::Ptr key, AbstractType::Ptr value) { auto newType = typeObjectForIntegralType("list"); IndexedContainer::Ptr newContents = typeObjectForIntegralType("tuple"); if ( ! newType || ! newContents ) { return AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! key ) { key = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! value ) { value = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } newContents->addEntry(key); newContents->addEntry(value); newType->addContentType(AbstractType::Ptr::staticCast(newContents)); AbstractType::Ptr resultingType = AbstractType::Ptr::staticCast(newType); return resultingType; }; QHash< QString, std::function > knownDecoratorHints; qCDebug(KDEV_PYTHON_DUCHAIN) << "Got function declaration with decorators, checking for list content type..."; knownDecoratorHints["getsType"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found container, using type"; docstringType = t->contentType().abstractType(); return true; } return false; }; knownDecoratorHints["getsList"] = [&](QStringList /*arguments*/, QString currentHint) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainWriteLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got container:" << t->toString(); auto newType = typeObjectForIntegralType("list"); if ( ! newType ) { return false; } AbstractType::Ptr contentType; if ( currentHint == "getsList" ) { contentType = t->contentType().abstractType(); } else if ( auto map = MapType::Ptr::dynamicCast(t) ) { contentType = map->keyType().abstractType(); } newType->addContentType(contentType); docstringType = newType.cast(); return true; } return false; }; knownDecoratorHints["getListOfKeys"] = knownDecoratorHints["getsList"]; knownDecoratorHints["enumerate"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::NameAstType || node->arguments.size() < 1 ) { return false; } ExpressionVisitor enumeratedTypeVisitor(this); enumeratedTypeVisitor.visitNode(node->arguments.first()); DUChainWriteLocker lock; auto intType = typeObjectForIntegralType("int"); auto enumerated = enumeratedTypeVisitor.lastType(); docstringType = listOfTuples(intType, Helper::contentOfIterable(enumerated, topContext())); return true; }; knownDecoratorHints["getsListOfBoth"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got getsListOfBoth decorator, checking container"; if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainWriteLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got container:" << t->toString(); docstringType = listOfTuples(t->keyType().abstractType(), t->contentType().abstractType()); return true; } return false; }; knownDecoratorHints["returnContentEqualsContentOf"] = [&](QStringList arguments, QString /*currentHint*/) { - int argNum = ! arguments.isEmpty() ? arguments.at(0).toInt() : 0; + const int argNum = ! arguments.isEmpty() ? (int) arguments.at(0).toUInt() : 0; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found argument dependent decorator, checking argument type" << argNum; if ( argNum >= node->arguments.length() ) { return false; } ExpressionAst* relevantArgument = node->arguments.at(argNum); ExpressionVisitor v(this); v.visitNode(relevantArgument); if ( ! v.lastType() ) { return false; } AbstractType::Ptr newType; if ( auto targetContainer = ListType::Ptr::dynamicCast(normalType) ) { // Copy the return type, to set contents for this call only. docstringType = AbstractType::Ptr(targetContainer->clone()); // Add content type of the source. auto sourceContentType = Helper::contentOfIterable(v.lastType(), topContext()); ListType::Ptr::staticCast(docstringType)->addContentType(sourceContentType); } else if ( auto sourceContainer = ListType::Ptr::dynamicCast(v.lastType()) ) { // if the function does not force a return type, just copy the source (like for reversed()) docstringType = AbstractType::Ptr(sourceContainer->clone()); } else { return false; // No target container type } Q_ASSERT(docstringType); return true; }; foreach ( const QString& currentHint, knownDecoratorHints.keys() ) { QStringList arguments; if ( ! Helper::docstringContainsHint(docstring, currentHint, &arguments) ) { continue; } // If the hint word appears in the docstring, run the evaluation function. if ( knownDecoratorHints[currentHint](arguments, currentHint) ) { // We indeed found something, so we're done. return docstringType; } } // if none of the above decorator-finding methods worked, just use the ordinary return type. return docstringType; } void ExpressionVisitor::visitSubscript(SubscriptAst* node) { AstDefaultVisitor::visitNode(node->value); auto valueTypes = Helper::filterType(lastType(), [](AbstractType::Ptr) { return true; }); AbstractType::Ptr result(new IntegralType(IntegralType::TypeMixed)); foreach (const auto& type, valueTypes) { if ( (node->slice->astType != Ast::IndexAstType) && (type.cast() || type.cast()) ) { if ( type.cast() ) { continue; // Can't slice dicts. } // Assume that slicing (e.g. foo[3:5]) a tuple/list returns the same type. // TODO: we could do better for some tuple slices. result = Helper::mergeTypes(result, type); } else if ( const auto& indexed = type.cast() ) { IndexAst* sliceIndexAst = static_cast(node->slice); NumberAst* number = nullptr; bool invert = false; if ( sliceIndexAst->value->astType == Ast::UnaryOperationAstType ) { // might be -3 UnaryOperationAst* unary = static_cast(sliceIndexAst->value); if ( unary->type == Ast::UnaryOperatorSub && unary->operand->astType == Ast::NumberAstType ) { number = static_cast(unary->operand); invert = true; } } else if ( sliceIndexAst->value->astType == Ast::NumberAstType ) { number = static_cast(sliceIndexAst->value); } if ( number ) { int sliceIndex = number->value * ( invert ? -1 : 1 ); if ( sliceIndex < 0 && sliceIndex + indexed->typesCount() > 0 ) { sliceIndex += indexed->typesCount(); } if ( sliceIndex < indexed->typesCount() && sliceIndex >= 0 ) { result = Helper::mergeTypes(result, indexed->typeAt(sliceIndex).abstractType()); continue; } } result = Helper::mergeTypes(result, indexed->asUnsureType()); } else if ( const auto& listType = type.cast() ) { result = Helper::mergeTypes(result, listType->contentType().abstractType()); } else { // Type wasn't one with custom handling, so use return type of __getitem__(). DUChainReadLocker lock; static const IndexedIdentifier getitemIdentifier(KDevelop::Identifier("__getitem__")); auto function = Helper::accessAttribute(type, getitemIdentifier, topContext()); if ( function && function->isFunctionDeclaration() ) { if ( FunctionType::Ptr functionType = function->type() ) { result = Helper::mergeTypes(result, functionType->returnType()); } } } } encounter(result); } void ExpressionVisitor::visitLambda(LambdaAst* node) { AstDefaultVisitor::visitLambda(node); FunctionType::Ptr type(new FunctionType()); AbstractType::Ptr mixed(new IntegralType(IntegralType::TypeMixed)); for (int ii = 0; ii < node->arguments->arguments.length(); ++ii) { type->addArgument(mixed); } type->setReturnType(lastType()); encounter(type); } void ExpressionVisitor::visitList(ListAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); if ( content->astType == Ast::StarredAstType ) { auto contentType = Helper::contentOfIterable(contentVisitor.lastType(), topContext()); type->addContentType(contentType); } else { type->addContentType(contentVisitor.lastType()); } } } else { encounterUnknown(); qCWarning(KDEV_PYTHON_DUCHAIN) << " [ !!! ] did not get a typetrack container object when expecting one! Fix code / setup."; } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol)); lock.unlock(); Q_ASSERT(comprehensionContext); DUContext* ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->value); if ( v.lastType() ) { type->addContentType(v.lastType()); } ExpressionVisitor k(this, ctx); k.visitNode(node->key); if ( k.lastType() ) { type->addKeyType(k.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitSetComprehension(SetComprehensionAst* node) { Python::AstDefaultVisitor::visitSetComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); auto ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitListComprehension(ListComprehensionAst* node) { AstDefaultVisitor::visitListComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); if ( type && ! m_forceGlobalSearching ) { // TODO fixme DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); ExpressionVisitor v(this, comprehensionContext); Q_ASSERT(comprehensionContext); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitTuple(TupleAst* node) { DUChainReadLocker lock; IndexedContainer::Ptr type = typeObjectForIntegralType("tuple"); if ( type ) { lock.unlock(); foreach ( ExpressionAst* expr, node->elements ) { ExpressionVisitor v(this); v.visitNode(expr); if ( expr->astType == Ast::StarredAstType ) { // foo = a, *b, c if ( auto unpackedType = v.lastType().cast() ) { for ( int ii = 0; ii < unpackedType->typesCount(); ++ii ) { type->addEntry(unpackedType->typeAt(ii).abstractType()); } } // Unpacking something else, do nothing (i.e. assume it was empty). } else { type->addEntry(v.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "tuple type object is not available"; return encounterUnknown(); } } void ExpressionVisitor::visitIfExpression(IfExpressionAst* node) { AstDefaultVisitor::visitIfExpression(node); if ( node->body && node->orelse ) { ExpressionVisitor v(this); v.visitNode(node->body); AbstractType::Ptr first = v.lastType(); v.visitNode(node->orelse); AbstractType::Ptr second = v.lastType(); encounter(Helper::mergeTypes(first, second)); } } void ExpressionVisitor::visitSet(SetAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); if ( content->astType == Ast::StarredAstType ) { auto contentType = Helper::contentOfIterable(contentVisitor.lastType(), topContext()); type->addContentType(contentType); } else { type->addContentType(contentVisitor.lastType()); } } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDict(DictAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); lock.unlock(); ExpressionVisitor contentVisitor(this); ExpressionVisitor keyVisitor(this); if ( type ) { Q_ASSERT(node->keys.length() == node->values.length()); for ( int ii = 0; ii < node->values.length(); ++ii ) { contentVisitor.visitNode(node->values.at(ii)); if ( node->keys.at(ii) ) { type->addContentType(contentVisitor.lastType()); keyVisitor.visitNode(node->keys.at(ii)); type->addKeyType(keyVisitor.lastType()); } else if ( auto unpackedType = contentVisitor.lastType().cast() ) { // Key is null for `{**foo}` type->addContentType(unpackedType->contentType().abstractType()); type->addKeyType(unpackedType->keyType().abstractType()); } } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitNumber(Python::NumberAst* number) { AbstractType::Ptr type; DUChainReadLocker lock; if ( number->isInt ) { type = typeObjectForIntegralType("int"); } else { type = typeObjectForIntegralType("float"); } encounter(type); } void ExpressionVisitor::visitString(Python::StringAst* ) { DUChainReadLocker lock; StructureType::Ptr type = typeObjectForIntegralType("str"); encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitBytes(Python::BytesAst* ) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("bytes"); encounter(AbstractType::Ptr::staticCast(type)); } RangeInRevision nodeRange(Python::Ast* node) { return RangeInRevision(node->startLine, node->startCol, node->endLine,node->endCol); } void ExpressionVisitor::addUnknownName(const QString& name) { if ( m_parentVisitor ) { static_cast(m_parentVisitor)->addUnknownName(name); } else if ( ! m_unknownNames.contains(name) ) { m_unknownNames.insert(name); } } void ExpressionVisitor::visitNameConstant(NameConstantAst* node) { // handles "True", "False", "None" auto defId = m_defaultTypes.constFind(node->value); if ( defId != m_defaultTypes.constEnd() ) { return encounter(*defId); } } void ExpressionVisitor::visitName(Python::NameAst* node) { RangeInRevision range; if ( m_scanUntilCursor.isValid() ) { range = RangeInRevision(CursorInRevision(0, 0), m_scanUntilCursor); } else if ( m_forceGlobalSearching ) { range = RangeInRevision::invalid(); } else { range = RangeInRevision(0, 0, node->endLine, node->endCol); } DUChainReadLocker lock; Declaration* d = Helper::declarationForName(QualifiedIdentifier(node->identifier->value), range, DUChainPointer(context())); if ( d ) { bool isAlias = dynamic_cast(d) || d->isFunctionDeclaration() || dynamic_cast(d); return encounter(d->abstractType(), DeclarationPointer(d), isAlias); } else { if ( m_reportUnknownNames ) { addUnknownName(node->identifier->value); } return encounterUnknown(); } } void ExpressionVisitor::visitCompare(CompareAst* node) { Python::AstDefaultVisitor::visitCompare(node); encounter(AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); } AbstractType::Ptr ExpressionVisitor::fromBinaryOperator(AbstractType::Ptr lhs, AbstractType::Ptr rhs, const QString& op) { DUChainReadLocker lock; auto operatorReturnType = [&op, this](const AbstractType::Ptr& p) { StructureType::Ptr type = p.cast(); if ( ! type ) { return AbstractType::Ptr(); } auto func = Helper::accessAttribute(type, op, topContext()); if ( ! func ) { return AbstractType::Ptr(); } auto operatorFunctionType = func->type(); DUChainReadLocker lock; auto context = Helper::getDocumentationFileContext(); if ( context ) { auto object_decl = context->findDeclarations(QualifiedIdentifier("object")); if ( ! object_decl.isEmpty() && object_decl.first()->internalContext() == func->context() ) { // if the operator is only declared in object(), do not include its type (which is void). return AbstractType::Ptr(); } } return operatorFunctionType ? operatorFunctionType->returnType() : AbstractType::Ptr(); }; return Helper::mergeTypes(operatorReturnType(lhs), operatorReturnType(rhs)); } void ExpressionVisitor::visitBinaryOperation(Python::BinaryOperationAst* node) { ExpressionVisitor lhsVisitor(this); ExpressionVisitor rhsVisitor(this); AbstractType::Ptr result; lhsVisitor.visitNode(node->lhs); rhsVisitor.visitNode(node->rhs); if ( lhsVisitor.lastType() && lhsVisitor.lastType()->whichType() == AbstractType::TypeUnsure ) { KDevelop::UnsureType::Ptr unsure = lhsVisitor.lastType().cast(); const IndexedType* types = unsure->types(); for( uint i = 0; i < unsure->typesSize(); i++ ) { result = Helper::mergeTypes(result, fromBinaryOperator(types[i].abstractType(), rhsVisitor.lastType(), node->methodName())); } } else { result = fromBinaryOperator(lhsVisitor.lastType(), rhsVisitor.lastType(), node->methodName()); } if ( ! Helper::isUsefulType(result) ) { result = Helper::mergeTypes(lhsVisitor.lastType(), rhsVisitor.lastType()); } return encounter(result); } void ExpressionVisitor::visitUnaryOperation(Python::UnaryOperationAst* node) { // Only visit the value, and use that as the result. Unary operators usually // don't change the type of the object (i.e. -a has the same type as a) visitNode(node->operand); } void ExpressionVisitor::visitBooleanOperation(Python::BooleanOperationAst* node) { foreach (ExpressionAst* expression, node->values) { visitNode(expression); } encounter(AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); } } diff --git a/duchain/tests/pyduchaintest.cpp b/duchain/tests/pyduchaintest.cpp index 93813b6f..fd20cc81 100644 --- a/duchain/tests/pyduchaintest.cpp +++ b/duchain/tests/pyduchaintest.cpp @@ -1,1648 +1,1662 @@ /***************************************************************************** * Copyright 2010 (c) Miquel Canes Gonzalez * * Copyright 2012 (c) Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include "duchaindebug.h" #include "pyduchaintest.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" QTEST_MAIN(PyDUChainTest) using namespace KDevelop; using namespace Python; PyDUChainTest::PyDUChainTest(QObject* parent): QObject(parent) { assetsDir = QDir(DUCHAIN_PY_DATA_DIR); if (!assetsDir.cd("data")) { qFatal("Failed find data directory for test files. Aborting"); } testDir = QDir(testDirOwner.path()); qputenv("PYTHONPATH", assetsDir.absolutePath().toUtf8()); initShell(); } QList PyDUChainTest::FindPyFiles(QDir& rootDir) { QList foundfiles; rootDir.setFilter(QDir::Files | QDir::Dirs | QDir::NoDot | QDir::NoDotDot); rootDir.setNameFilters(QStringList() << "*.py"); // We only want .py files QDirIterator it(rootDir, QDirIterator::Subdirectories); while(it.hasNext()) { foundfiles.append(it.next()); } return foundfiles; } void PyDUChainTest::init() { QString currentTest = QString(QTest::currentTestFunction()); if (lastTest == currentTest) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Already prepared assets for " << currentTest << ", skipping"; return; } else { lastTest = currentTest; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Preparing assets for test " << currentTest; QDir assetModuleDir = QDir(assetsDir.absolutePath()); if (!assetModuleDir.cd(currentTest)) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Asset directory " << currentTest << " does not exist under " << assetModuleDir.absolutePath() << ". Skipping it."; return; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Searching for python files in " << assetModuleDir.absolutePath(); QList foundfiles = FindPyFiles(assetModuleDir); QString correctionFileDir = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files", QStandardPaths::LocateDirectory); auto correctionFileUrl = QUrl(QDir::cleanPath(correctionFileDir + "/testCorrectionFiles/example.py")); foundfiles.prepend(correctionFileUrl.path()); for ( int i = 0; i < 2; i++ ) { // Parse each file twice, to ensure no parsing-order related bugs appear. // Such bugs will need separate unit tests and should not influence these. foreach(const QString filename, foundfiles) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Parsing asset: " << filename; DUChain::self()->updateContextForUrl(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); } foreach(const QString filename, foundfiles) { DUChain::self()->waitForUpdate(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); } while ( ICore::self()->languageController()->backgroundParser()->queuedCount() > 0 ) { // make sure to wait for all parsejobs to finish QTest::qWait(10); } } } void PyDUChainTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); auto doc_url = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); qCDebug(KDEV_PYTHON_DUCHAIN) << doc_url; DUChain::self()->updateContextForUrl(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); DUChain::self()->waitForUpdate(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } ReferencedTopDUContext PyDUChainTest::parse(const QString& code) { TestFile* testfile = new TestFile(code + "\n", "py", 0, testDir.absolutePath().append("/")); createdFiles << testfile; testfile->parse((TopDUContext::Features) (TopDUContext::ForceUpdate | TopDUContext::AST) ); testfile->waitForParsed(2000); if ( testfile->isReady() ) { Q_ASSERT(testfile->topContext()); m_ast = static_cast(testfile->topContext()->ast().data())->ast; return testfile->topContext(); } else Q_ASSERT(false && "Timed out waiting for parser results, aborting all tests"); return 0; } PyDUChainTest::~PyDUChainTest() { foreach ( TestFile* f, createdFiles ) { delete f; } testDir.rmdir(testDir.absolutePath()); } void PyDUChainTest::testMultiFromImport() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList a = ctx->findDeclarations(QualifiedIdentifier("a")); QList b = ctx->findDeclarations(QualifiedIdentifier("b")); QVERIFY(! a.isEmpty()); QVERIFY(! b.isEmpty()); QVERIFY(a.first()->abstractType()->toString().endsWith("int")); QVERIFY(b.first()->abstractType()->toString().endsWith("int")); } void PyDUChainTest::testMultiFromImport_data() { QTest::addColumn("code"); QTest::newRow("multiimport") << "import testMultiFromImport.i.localvar1\n" "import testMultiFromImport.i.localvar2\n" "a = testMultiFromImport.i.localvar1\n" "b = testMultiFromImport.i.localvar2\n"; } void PyDUChainTest::testRelativeImport() { QFETCH(QString, code); QFETCH(QString, token); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList t = ctx->findDeclarations(QualifiedIdentifier(token)); QVERIFY(! t.isEmpty()); QVERIFY(t.first()->abstractType()->toString().endsWith(type)); } void PyDUChainTest::testRelativeImport_data() { QTest::addColumn("code"); QTest::addColumn("token"); QTest::addColumn("type"); QTest::newRow(".local") << "from testRelativeImport.m.sm1.go import i1" << "i1" << "int"; QTest::newRow(".init") << "from testRelativeImport.m.sm1.go import i2" << "i2" << "int"; QTest::newRow("..local") << "from testRelativeImport.m.sm1.go import i3" << "i3" << "int"; QTest::newRow("..init") << "from testRelativeImport.m.sm1.go import i4" << "i4" << "int"; QTest::newRow("..sub.local") << "from testRelativeImport.m.sm1.go import i5" << "i5" << "int"; QTest::newRow("..sub.init") << "from testRelativeImport.m.sm1.go import i6" << "i6" << "int"; } void PyDUChainTest::testImportFiles() { QString code = "import testImportFiles\nk = testImportFiles.fromInit()\np = testImportFiles.other.fromOther()"; ReferencedTopDUContext ctx = parse(code.toUtf8()); DUChainReadLocker lock; QVERIFY(ctx); auto k = ctx->findDeclarations(QualifiedIdentifier("k")); auto p = ctx->findDeclarations(QualifiedIdentifier("p")); QCOMPARE(k.size(), 1); QCOMPARE(p.size(), 1); QVERIFY(k.first()->abstractType()); QCOMPARE(k.first()->abstractType()->toString(), QString("fromInit")); QCOMPARE(p.first()->abstractType()->toString(), QString("fromOther")); } void PyDUChainTest::testCrashes() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); QVERIFY(! m_ast->body.isEmpty()); } void PyDUChainTest::testCrashes_data() { QTest::addColumn("code"); QTest::newRow("unicode_char") << "a = \"í\""; QTest::newRow("unicode escape char") << "print(\"\\xe9\")"; QTest::newRow("augassign") << "a = 3\na += 5"; QTest::newRow("delete") << "a = 3\ndel a"; QTest::newRow("double_comprehension") << "q = [[x for x in a] + [x for x in a] for y in b]"; QTest::newRow("for_else") << "for i in range(3): pass\nelse: pass"; QTest::newRow("for_while") << "while i < 4: pass\nelse: pass"; QTest::newRow("ellipsis") << "a[...]"; QTest::newRow("tuple_assign_unknown") << "foo = (unknown, unknown, unknown)"; QTest::newRow("for_assign_unknown") << "for foo, bar, baz in unknown: pass"; QTest::newRow("negative slice index") << "t = (1, 2, 3)\nd = t[-1]"; QTest::newRow("decorator_with_args") << "@foo('bar', 'baz')\ndef myfunc(): pass"; QTest::newRow("non_name_decorator") << "@foo.crazy_decorators\ndef myfunc(): pass"; QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass"; QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)"; QTest::newRow("whatever") << "for attr in updated:\n " " getattr.update"; QTest::newRow("return_outside_function") << "return 3"; QTest::newRow("return_context_outside_function") << "return [x for x in range(3)]"; QTest::newRow("paren_attrib_access") << "a = (xxx or yyy).zzz"; QTest::newRow("func_call") << "a = xxx.func(yyy.zzz)"; QTest::newRow("comprehension_attrib") << "a = [foo for foo in bar].baz"; QTest::newRow("comprehension_attrib2") << "a = [foo.bar for foo in bar]"; QTest::newRow("attrib") << "(sep or ' ').join(xxxx.capitalize() for xxxx in ssss.split(sep))"; QTest::newRow("attrib2") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))"; QTest::newRow("attrib3") << "known_threads = {line.strip()}"; QTest::newRow("attrib4") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}"; QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()"; QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n" ".format(type(meth))) from None"; QTest::newRow("newline_attrib") << "some_instance \\\n" ". attr1 \\\n" ".funcfunc(argarg, arg2arg) \\\n" ".foo"; QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n" "in ambiguous_nucleotide_values.iteritems() \\\n" "if set([codon[0] for codon in codons]).issuperset(set(meanings)))"; QTest::newRow("fancy class range") << "class SchemeLexer(RegexLexer):\n" " valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+'\n" "\n" " tokens = {\n" " 'root' : [\n" " # the comments - always starting with semicolon\n" " # and going to the end of the line\n" " (r';.*$', Comment.Single),\n" "\n" " # whitespaces - usually not relevant\n" " (r'\\s+', Text),\n" "\n" " # numbers\n" " (r'-?\\d+\\.\\d+', Number.Float),\n" " (r'-?\\d+', Number.Integer)\n" " ],\n" " }\n"; QTest::newRow("another fancy range") << "setup_args['data_files'] = [\n" " (os.path.dirname(os.path.join(install_base_dir, pattern)),\n" " [ f for f in glob.glob(pattern) ])\n" " for pattern in patterns\n" "]\n"; QTest::newRow("kwarg_empty_crash") << "def myfun(): return\ncheckme = myfun(kw=something)"; QTest::newRow("stacked_tuple_hang") << "tree = (1,(2,(3,(4,(5,'Foo')))))"; QTest::newRow("stacked_tuple_hang2") << "tree = (257," "(264," "(285," "(259," "(272," "(275," "(1, 'return')))))))"; QTest::newRow("very_large_tuple_hang") << "tree = " "(257," "(264," "(285," "(259," "(1, 'def')," "(1, 'f')," "(260, (7, '('), (8, ')'))," "(11, ':')," "(291," "(4, '')," "(5, '')," "(264," "(265," "(266," "(272," "(275," "(1, 'return')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302, (303, (304, (305, (2, '1'))))))))))))))))))," "(264," "(265," "(266," "(272," "(276," "(1, 'yield')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302," "(303, (304, (305, (2, '1'))))))))))))))))))," "(4, '')))," "(6, '')))))," "(4, '')," "(0, ''))))"; QTest::newRow("attribute_hang") << "s = \"123\"\n" "s = s.replace(u'ł', 'l').\\\n" "replace(u'ó', 'o').\\\n" "replace(u'ą', 'a').\\\n" "replace(u'ę', 'e').\\\n" "replace(u'ś', 's').\\\n" "replace(u'ż', 'z').\\\n" "replace(u'ź', 'z').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo')\n"; QTest::newRow("function context range crash") << "def myfunc(arg):\n foo = 3 + \\\n[x for x in range(20)]"; QTest::newRow("decorator comprehension crash") << "@implementer_only(interfaces.ISSLTransport,\n" " *[i for i in implementedBy(tcp.Client)\n" " if i != interfaces.ITLSTransport])\n" "class Client(tcp.Client):\n" " pass\n"; QTest::newRow("comprehension_as_default_crash") << "def foo(bar = [item for (_, item) in items()]):\n return"; QTest::newRow("try_except") << "try: pass\nexcept: pass"; QTest::newRow("try_except_type") << "try: pass\nexcept FooException: pass"; QTest::newRow("try_except_type_as") << "try: pass\nexcept FooException as bar: pass"; QTest::newRow("import_missing") << "from this_does_not_exist import nor_does_this"; QTest::newRow("list_append_missing") << "foo = []\nfoo.append(missing)"; QTest::newRow("list_append_missing_arg") << "foo = []\nfoo.append()"; QTest::newRow("list_extend_missing") << "foo = []\nfoo.extend(missing)"; QTest::newRow("list_extend_missing_arg") << "foo = []\nfoo.extend()"; QTest::newRow("method_of_call_with_list_arg") << "class MyClass:\n" " def bar(self): pass\n" "def foo(x):\n" " return MyClass()\n" "foo([0]).bar()"; QTest::newRow("unpacked_dict_kwarg") << "def foo(arg): pass\nfoo(**{'arg': 2})"; + QTest::newRow("negative_container_hints") << + "class Evil:\n" + " def aa(self, arg):\n" + " \"\"\"! addsTypeOfArgContent ! -1\"\"\"\n" + " def bb(self, arg):\n" + " \"\"\"! addsTypeOfArg ! -2\"\"\"\n" + " def cc(self, arg):\n" + " \"\"\"! returnContentEqualsContentOf ! -3\"\"\"\n" + "e = Evil()\n" + "z = [e.aa(1), e.bb(2), e.cc(3)]"; + QTest::newRow("comprehension_in_fstring") << + "def crash(): return f'expr={ {x: y for x, y in [(1, 2), ]}}'"; } void PyDUChainTest::testClassVariables() { ReferencedTopDUContext ctx = parse("class c():\n myvar = 3;\n def meth(self):\n print(myvar)"); QVERIFY(ctx.data()); DUChainWriteLocker lock(DUChain::lock()); CursorInRevision relevantPosition(3, 10); DUContext* c = ctx->findContextAt(relevantPosition); QVERIFY(c); int useIndex = c->findUseAt(relevantPosition); if ( useIndex != -1 ) { QVERIFY(useIndex < c->usesCount()); const Use* u = &(c->uses()[useIndex]); QVERIFY(!u->usedDeclaration(c->topContext())); } } void PyDUChainTest::testWarnNewNotCls() { QFETCH(QString, code); QFETCH(int, probs); ReferencedTopDUContext ctx = parse(code); DUChainReadLocker lock; QCOMPARE(ctx->problems().count(), probs); } void PyDUChainTest::testWarnNewNotCls_data() { QTest::addColumn("code"); QTest::addColumn("probs"); QTest::newRow("check_for_new_first_arg_cls") << "class c():\n def __new__(clf, other):\n pass" << 1; QTest::newRow("check_for_new_first_arg_cls_0") << "class c():\n def __new__(cls, other):\n pass" << 0; QTest::newRow("check_first_arg_class_self") << "class c():\n def test(seff, masik):\n pass" << 1; QTest::newRow("check_first_arg_class_self_0") << "class c():\n def test(self, masik):\n pass" << 0; } void PyDUChainTest::testBinaryOperatorsUnsure() { QFETCH(QString, code); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), type); } void PyDUChainTest::testBinaryOperatorsUnsure_data() { QTest::addColumn("code"); QTest::addColumn("type"); QTest::newRow("check_unsure_type_0") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3.5;\ny = 3;\ncheckme = x * y;" << "unsure (float, int)"; QTest::newRow("check_unsure_type_1") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_2") << "class c():\n pass;\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_3") << "class c():\n pass;\nclass d():\n pass;\nx = c();\nx = d();\ny = 3;\ncheckme = x * y;" << "int"; } void PyDUChainTest::testFlickering() { QFETCH(QStringList, code); QFETCH(int, before); QFETCH(int, after); TestFile f(code[0], "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx = f.topContext(); QVERIFY(ctx); DUChainWriteLocker lock(DUChain::lock()); int count = ctx->localDeclarations().size(); qDebug() << "Declaration count before: " << count; QVERIFY(count == before); lock.unlock(); f.setFileContents(code[1]); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ctx = f.topContext(); QVERIFY(ctx); lock.lock(); count = ctx->localDeclarations().size(); qDebug() << "Declaration count afterwards: " << count; QVERIFY(count == after); foreach(Declaration* dec, ctx->localDeclarations()) { qDebug() << dec->toString() << dec->range(); qDebug() << dec->uses().size(); } } void PyDUChainTest::testFlickering_data() { QTest::addColumn("code"); QTest::addColumn("before"); QTest::addColumn("after"); QTest::newRow("declaration_flicker") << ( QStringList() << "a=2\n" << "b=3\na=2\n" ) << 1 << 2; } void PyDUChainTest::testCannotOverwriteBuiltins() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testCannotOverwriteBuiltins_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("list_assign") << "class list(): pass\ncheckme = []\ncheckme.append(3)" << "list of int"; QTest::newRow("str_assign") << "str = 5; checkme = 'Foo'" << "str"; QTest::newRow("str_assign2") << "class Foo: pass\nstr = Foo; checkme = 'Foo'" << "str"; QTest::newRow("str_assign3") << "from testCannotOverwriteBuiltins.i import Foo as str\ncheckme = 'Foo'" << "str"; QTest::newRow("for") << "for str in [1, 2, 3]: pass\ncheckme = 'Foo'" << "str"; QTest::newRow("assert") << "assert isinstance(str, int)\ncheckme = 'Foo'" << "str"; QTest::newRow("assert2") << "assert isinstance(str, int)\ncheckme = 3" << "int"; QTest::newRow("can_have_custom") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open()" << "int"; QTest::newRow("can_have_custom2") << "from testCannotOverwriteBuiltins import mod\ncheckme = open().read()" << "str"; QTest::newRow("can_have_custom3") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open().read()" << "mixed"; } void PyDUChainTest::testVarKWArgs() { ReferencedTopDUContext ctx = parse("def myfun(arg, *vararg, **kwarg):\n pass\n pass"); DUChainWriteLocker lock; QVERIFY(ctx); DUContext* func = ctx->findContextAt(CursorInRevision(1, 0)); QVERIFY(func); QVERIFY(! func->findDeclarations(QualifiedIdentifier("arg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("vararg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("kwarg")).isEmpty()); QVERIFY(func->findDeclarations(QualifiedIdentifier("vararg")).first()->abstractType()->toString().startsWith("tuple")); QCOMPARE(func->findDeclarations(QualifiedIdentifier("kwarg")).first()->abstractType()->toString(), QString("dict of str : unknown")); } void PyDUChainTest::testSimple() { QFETCH(QString, code); QFETCH(int, decls); QFETCH(int, uses); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVector< Declaration* > declarations = ctx->localDeclarations(); QCOMPARE(declarations.size(), decls); int usesCount = 0; foreach(Declaration* d, declarations) { usesCount += d->uses().size(); QVERIFY(d->abstractType()); } QCOMPARE(usesCount, uses); } void PyDUChainTest::testSimple_data() { QTest::addColumn("code"); QTest::addColumn("decls"); QTest::addColumn("uses"); QTest::newRow("assign") << "b = 2;" << 1 << 0; QTest::newRow("assign_str") << "b = 'hola';" << 1 << 0; QTest::newRow("op") << "a = 3; b = a+2;" << 2 << 1; QTest::newRow("bool") << "a = True" << 1 << 0; QTest::newRow("op") << "a = True and True;" << 1 << 0; } class AttributeRangeTestVisitor : public AstDefaultVisitor { public: bool found; KTextEditor::Range searchingForRange; QString searchingForIdentifier; void visitAttribute(AttributeAst* node) override { auto r = KTextEditor::Range(0, node->startCol, 0, node->endCol); qDebug() << "Found attr: " << r << node->attribute->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->attribute->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitAttribute(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found func: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; qDebug() << node->arguments->vararg << node->arguments->kwarg; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } if ( node->arguments->vararg ) { auto r = KTextEditor::Range(0, node->arguments->vararg->startCol, 0, node->arguments->vararg->startCol+node->arguments->vararg->argumentName->value.length()); qDebug() << "Found vararg: " << node->arguments->vararg->argumentName->value << r; if ( r == searchingForRange && node->arguments->vararg->argumentName->value == searchingForIdentifier ) { found = true; return; } } if ( node->arguments->kwarg ) { auto r = KTextEditor::Range(0, node->arguments->kwarg->startCol, 0, node->arguments->kwarg->startCol+node->arguments->kwarg->argumentName->value.length()); qDebug() << "Found kwarg: " << node->arguments->kwarg->argumentName->value << r; if ( r == searchingForRange && node->arguments->kwarg->argumentName->value == searchingForIdentifier ) { found = true; return; } } AstDefaultVisitor::visitFunctionDefinition(node); } void visitClassDefinition(ClassDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found cls: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitClassDefinition(node); } void visitImport(ImportAst* node) override { foreach ( const AliasAst* name, node->names ) { if ( name->name ) { qDebug() << "found import" << name->name->value << name->name->range(); } if ( name->name && name->name->value == searchingForIdentifier && name->name->range() == searchingForRange ) { found = true; return; } if ( name->asName ) { qDebug() << "found import" << name->asName->value << name->asName->range(); } if ( name->asName && name->asName->value == searchingForIdentifier && name->asName->range() == searchingForRange ) { found = true; return; } } } }; void PyDUChainTest::testRanges() { QFETCH(QString, code); QFETCH(int, expected_amount_of_variables); Q_UNUSED(expected_amount_of_variables); QFETCH(QStringList, column_ranges); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); for ( int i = 0; i < column_ranges.length(); i++ ) { int scol = column_ranges.at(i).split(",")[0].toInt(); int ecol = column_ranges.at(i).split(",")[1].toInt(); QString identifier = column_ranges.at(i).split(",")[2]; auto r = KTextEditor::Range(0, scol, 0, ecol); AttributeRangeTestVisitor* visitor = new AttributeRangeTestVisitor(); visitor->searchingForRange = r; visitor->searchingForIdentifier = identifier; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("attr_dot_name_hash", "Insufficiently magic hack", Continue); QCOMPARE(visitor->found, true); delete visitor; } } void PyDUChainTest::testRanges_data() { QTest::addColumn("code"); QTest::addColumn("expected_amount_of_variables"); QTest::addColumn("column_ranges"); QTest::newRow("attr_two_attributes") << "base.attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_binary") << "base.attr + base.attr" << 4 << ( QStringList() << "5,8,attr" << "17,20,attr" ); QTest::newRow("attr_same") << "aaa.aaa.aaa + aaa.aaa.aaa" << 6 << ( QStringList() << "4,6,aaa" << "8,10,aaa" << "18,20,aaa" << "22,24,aaa" ); QTest::newRow("attr_three_attributes") << "base.attr.subattr" << 3 << ( QStringList() << "5,8,attr" << "10,16,subattr" ); QTest::newRow("attr_functionCall") << "base.attr().subattr" << 3 << ( QStringList() << "5,8,attr" << "12,18,subattr" ); QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" ); QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" ); QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" ); QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" << "25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr"); QTest::newRow("attr_two_in_call") << "func(inst.aaa, inst.bbbb)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,bbbb" ); QTest::newRow("attr_two_in_call_same") << "func(inst.aaa, inst.aaaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,aaaa" ); QTest::newRow("attr_two_in_call_same2") << "func(inst.aaa, inst.aaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,22,aaa" ); QTest::newRow("attr_of_string_slash") << "'/'.join(a)" << 1 << ( QStringList() << "4,7,join" ); QTest::newRow("attr_of_string_in_list") << "[\"*{0}*\".format(foo)]" << 1 << ( QStringList() << "9,14,format" ); QTest::newRow("attr_of_call_in_list") << "[foo().format(foo)]" << 1 << ( QStringList() << "7,12,format" ); QTest::newRow("attr_parentheses") << "(\"foo\" + \"foo\").capitalize()" << 1 << ( QStringList() << "16,25,capitalize" ); QTest::newRow("attr_commented_name") << "base.attr # attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_name_in_strings") << "'attr' + base['attr'].attr # attr" << 4 << ( QStringList() << "22,25,attr" ); QTest::newRow("attr_dot_hash_in_strings") << "'.foo#' + base['.#'].attr # attr" << 4 << ( QStringList() << "21,24,attr" ); QTest::newRow("attr_dot_name_hash") << "base['.attr#'].attr" << 4 << ( QStringList() << "15,18,attr" ); QTest::newRow("string_parentheses") << "(\"asdf\".join())" << 1 << ( QStringList() << "8,11,join" ); QTest::newRow("string_parentheses2") << "(\"asdf\").join()" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("string_parentheses3") << "(\"asdf\".join()).join()" << 2 << ( QStringList() << "8,11,join" << "16,19,join" ); QTest::newRow("string_parentheses4") << "(\"asdf\".join()+2).join()" << 2 << ( QStringList() << "8,11,join" << "18,21,join" ); QTest::newRow("string_parentheses_call") << "f(\"asdf\".join())" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" ); QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" ); QTest::newRow("classdef_range") << "class cls(): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance") << "class cls(parent1, parent2): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance_spaces") << "class cls( parent1, parent2 ):pass" << 1 << ( QStringList() << "12,14,cls" ); QTest::newRow("vararg_kwarg") << "def func(*vararg, **kwargs): pass" << 2 << ( QStringList() << "10,16,vararg" << "20,26,kwargs" ); QTest::newRow("import") << "import sys" << 1 << ( QStringList() << "7,10,sys" ); QTest::newRow("import2") << "import i.localvar1" << 1 << ( QStringList() << "7,18,i.localvar1" ); QTest::newRow("import3") << "import sys as a" << 1 << ( QStringList() << "13,14,a" ); } class TypeTestVisitor : public AstDefaultVisitor { public: QString searchingForType; TopDUContextPointer ctx; bool found; void visitName(NameAst* node) override { if ( node->identifier->value != "checkme" ) return; QList decls = ctx->findDeclarations(QualifiedIdentifier(node->identifier->value)); if ( ! decls.length() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration found for " << node->identifier->value; return; } Declaration* d = decls.last(); QVERIFY(d->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << "found: " << node->identifier->value << "is" << d->abstractType()->toString() << "should be" << searchingForType; if ( d->abstractType()->toString().replace("__kdevpythondocumentation_builtin_", "").startsWith(searchingForType) ) { found = true; return; } }; }; void PyDUChainTest::testTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("return_builtin_iterator", "fake builtin iter()", Continue); QEXPECT_FAIL("parent_constructor_arg_type", "Not enough passes?", Continue); QEXPECT_FAIL("init_class_no_decl", "aliasing info lost", Continue); QCOMPARE(visitor->found, true); } void PyDUChainTest::testTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("listtype") << "checkme = []" << "list"; QTest::newRow("listtype_func") << "checkme = list()" << "list"; QTest::newRow("listtype_with_contents") << "checkme = [1, 2, 3, 4, 5]" << "list of int"; QTest::newRow("listtype_extended") << "some_misc_var = []; checkme = some_misc_var" << "list"; QTest::newRow("dicttype") << "checkme = {}" << "dict"; QTest::newRow("dicttype_get") << "d = {0.4:5}; checkme = d.get(0)" << "int"; QTest::newRow("dicttype_func") << "checkme = dict()" << "dict"; QTest::newRow("dicttype_extended") << "some_misc_var = {}; checkme = some_misc_var" << "dict"; QTest::newRow("bool") << "checkme = True" << "bool"; QTest::newRow("float") << "checkme = 3.7" << "float"; QTest::newRow("int") << "checkme = 3" << "int"; QTest::newRow("str") << "checkme = \"foo\"" << "str"; QTest::newRow("bytes") << "checkme = b\"foo\"" << "bytes"; QTest::newRow("function_arg_scope") << "class Foo:\n" " a = 3\n" " def func(self, x=a):\n" " return x\n" "f = Foo()\n" "checkme = f.func()" << "int"; QTest::newRow("with") << "with open('foo') as f: checkme = f.read()" << "str"; QTest::newRow("arg_after_vararg") << "def func(x, y, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("arg_after_vararg_with_default") << "def func(x=5, y=3, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("class_scope_end_inside") << "a = str()\nclass M:\n" " a = 2\n foo = a\n" "checkme = M().foo" << "int"; QTest::newRow("class_scope_end_outside") << "a = str()\nclass M:\n a = 2\ncheckme = a" << "str"; QTest::newRow("list_access_right_open_slice") << "some_list = []; checkme = some_list[2:]" << "list"; QTest::newRow("list_access_left_open_slice") << "some_list = []; checkme = some_list[:2]" << "list"; QTest::newRow("list_access_closed_slice") << "some_list = []; checkme = some_list[2:17]" << "list"; QTest::newRow("list_access_step") << "some_list = []; checkme = some_list[::2]" << "list"; QTest::newRow("list_access_singleItem") << "some_list = []; checkme = some_list[42]" << "mixed"; QTest::newRow("funccall_number") << "def foo(): return 3; \ncheckme = foo();" << "int"; QTest::newRow("funccall_string") << "def foo(): return 'a'; \ncheckme = foo();" << "str"; QTest::newRow("funccall_list") << "def foo(): return []; \ncheckme = foo();" << "list"; QTest::newRow("funccall_dict") << "def foo(): return {}; \ncheckme = foo();" << "dict"; QTest::newRow("tuple1") << "checkme, foo = 3, \"str\"" << "int"; QTest::newRow("tuple2") << "foo, checkme = 3, \"str\"" << "str"; QTest::newRow("tuple2_negative_index") << "foo = (1, 2, 'foo')\ncheckme = foo[-1]" << "str"; QTest::newRow("tuple_type") << "checkme = 1, 2" << "tuple"; QTest::newRow("tuple_rhs_unpack") << "foo = 1, 2.5\nbar = 1, *foo, 2\ncheckme = bar[2]" << "float"; QTest::newRow("dict_iteritems") << "d = {1:2, 3:4}\nfor checkme, k in d.iteritems(): pass" << "int"; QTest::newRow("enumerate_key") << "d = [str(), str()]\nfor checkme, value in enumerate(d): pass" << "int"; QTest::newRow("enumerate_value") << "d = [str(), str()]\nfor key, checkme in enumerate(d): pass" << "str"; QTest::newRow("dict_enumerate") << "d = {1:2, 3:4}\nfor key, checkme in enumerate(d.values()): pass" << "int"; QTest::newRow("dict_assign_twice") << "d = dict(); d[''] = 0; d = dict(); d[''] = 0; checkme = d" << "unsure (dict of str : int, dict)"; QTest::newRow("class_method_import") << "class c:\n attr = \"foo\"\n def m():\n return attr;\n return 3;\ni=c()\ncheckme=i.m()" << "int"; QTest::newRow("getsListDecorator") << "foo = [1, 2, 3]\ncheckme = foo.reverse()" << "list of int"; QTest::newRow("str_iter") << "checkme = [char for char in 'Hello, world!']" << "list of str"; QTest::newRow("str_subscript") << "checkme = 'Hello, world!'[0]" << "str"; QTest::newRow("bytes_iter") << "checkme = [byte for byte in b'Hello, world!']" << "list of int"; QTest::newRow("bytes_subscript") << "checkme = b'Hello, world!'[0]" << "int"; QTest::newRow("fromAssertIsinstance") << "class c(): pass\ncheckme = mixed()\nassert isinstance(checkme, c)\n" << "c"; QTest::newRow("fromAssertIsinstanceInvalid") << "class c(): pass\ncheckme = mixed()\nassert isinstance(c, checkme)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid2") << "class c(): pass\ncheckme = mixed()\nassert isinstance(D, c)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid3") << "checkme = int()\nassert isinstance(checkme, X)\n" << "int"; QTest::newRow("fromAssertIsinstanceInvalid4") << "checkme = int()\nassert isinstance(checkme)\n" << "int"; QTest::newRow("fromAssertType") << "class c(): pass\ncheckme = mixed()\nassert type(checkme) == c\n" << "c"; QTest::newRow("fromIfType") << "class c(): pass\ncheckme = mixed()\nif type(checkme) == c: pass\n" << "c"; QTest::newRow("fromIfIsinstance") << "class c(): pass\ncheckme = mixed()\nif isinstance(checkme, c): pass\n" << "c"; QTest::newRow("diff_local_classattr") << "class c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; QTest::newRow("diff_local_classattr2") << "local=3\nclass c(): attr = 1\ninst=c()\ncheckme = c.local" << "mixed"; QTest::newRow("diff_local_classattr3") << "attr=3.5\nclass c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; // QTest::newRow("class_method_self") << "class c:\n def func(checkme, arg, arg2):\n pass\n" << "c"; // QTest::newRow("funccall_dict") << "def foo(): return foo; checkme = foo();" << (uint) IntegralType::TypeFunction; QTest::newRow("tuple_simple") << "mytuple = 3, 5.5\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_simple2") << "mytuple = 3, 5.5\nfoobar, checkme = mytuple" << "float"; QTest::newRow("tuple_simple3") << "mytuple = 3, 5.5, \"str\", 3, \"str\"\na, b, c, d, checkme = mytuple" << "str"; QTest::newRow("tuple_single") << "checkme = 4," << "tuple"; QTest::newRow("tuple_single2") << "checkme, = 4," << "int"; QTest::newRow("tuple_single3") << "mytuple = 4,\ncheckme, = mytuple" << "int"; QTest::newRow("tuple_ext_unpack") << "mytuple = 3, 5.5\nfoobar, *starred, checkme = mytuple" << "float"; QTest::newRow("tuple_ext_unpack2") << "mytuple = 3, 5.5\nfoobar, *checkme, another = mytuple" << "list"; QTest::newRow("tuple_ext_unpack3") << "mytuple = 3, 5.5\nfoobar, *checkme = mytuple" << "list of float"; QTest::newRow("tuple_ext_unpack4") << "mytuple = 3, 5.5\n*checkme, = mytuple" << "list of unsure (int, float)"; QTest::newRow("tuple_nested") << "mytuple = 3, ('foo', 5.5)\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_nested2") << "mytuple = 3, ('foo', 5.5)\nfoobar, (checkme, other) = mytuple" << "str"; QTest::newRow("tuple_nested3") << "mytuple = ((7, 'foo'), 5.5), 3\n((baz, checkme), other), foo = mytuple" << "str"; QTest::newRow("tuple_nested_ext") << "mytuple = (2, ('foo', 'bar', 6), 7)\na, (b, *checkme, c), *d = mytuple" << "list of str"; QTest::newRow("tuple_multi_assign") << "mytuple = 2, 'foo'\ncheckme = a = mytuple" << "tuple"; QTest::newRow("tuple_multi_assign2") << "mytuple = 2, 'foo'\ncheckme, a = b = mytuple" << "int"; QTest::newRow("list_unpack") << "mylist = [1, 2, 3]\ncheckme, b, c = mylist" << "int"; QTest::newRow("list_unpack2") << "mylist = [1, 'x', 3]\ncheckme, b, c = mylist" << "unsure (int, str)"; QTest::newRow("list_ext_unpack") << "mylist = [1, 2, 3]\n*checkme, foo = mylist" << "list of int"; QTest::newRow("list_ext_unpack2") << "mylist = [1, 'x', 3]\n*checkme, foo = mylist" << "list of unsure (int, str)"; QTest::newRow("if_expr_sure") << "checkme = 3 if 7 > 9 else 5" << "int"; QTest::newRow("unary_op") << "checkme = -42" << "int"; QTest::newRow("tuple_funcret") << "def myfun(): return 3, 5\ncheckme, a = myfun()" << "int"; QTest::newRow("tuple_funcret2") << "def myfun():\n t = 3, 5\n return t\ncheckme, a = myfun()" << "int"; QTest::newRow("yield") << "def myfun():\n yield 3\ncheckme = myfun()" << "list of int"; QTest::newRow("yield_twice") << "def myfun():\n yield 3\n yield 'foo'\ncheckme = myfun()" << "list of unsure (int, str)"; // this is mostly a check that it doesn't crash QTest::newRow("yield_return") << "def myfun():\n return 3\n yield 'foo'\ncheckme = myfun()" << "unsure (int, list of str)"; QTest::newRow("lambda") << "x = lambda t: 3\ncheckme = x()" << "int"; QTest::newRow("lambda_failure") << "x = lambda t: 3\ncheckme = t" << "mixed"; QTest::newRow("function_arg_tuple") << "def func(*arg):\n foo, bar = arg\n return bar\ncheckme = func(3, 5)" << "int"; QTest::newRow("function_arg_tuple2") << "def func(*arg):\n return arg[-1]\ncheckme = func(3, \"Foo\")" << "str"; QTest::newRow("tuple_indexaccess") << "t = 3, 5.5\ncheckme = t[0]" << "int"; QTest::newRow("tuple_indexaccess2") << "t = 3, 5.5\ncheckme = t[1]" << "float"; QTest::newRow("tuple_indexaccess3") << "t = 3, 4\ncheckme = t[1]" << "int"; QTest::newRow("dict_unsure") << "t = dict(); t = {3: str()}\ncheckme = t[1].capitalize()" << "str"; QTest::newRow("unsure_attr_access") << "d = str(); d = 3; checkme = d.capitalize()" << "str"; QTest::newRow("class_create_var") << "class c: pass\nd = c()\nd.foo = 3\ncheckme = d.foo" << "int"; QTest::newRow("tuple_loop") << "t = [(1, \"str\")]\nfor checkme, a in t: pass" << "int"; QTest::newRow("no_hints_type") << "def myfun(arg): arg = 3; return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("hints_type") << "def myfun(arg): return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("args_type") << "def myfun(*args): return args[0]\ncheckme = myfun(3)" << "int"; QTest::newRow("kwarg_type") << "def myfun(**kwargs): return kwargs['a']\ncheckme = myfun(a=3)" << "int"; QTest::newRow("dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\ncheckme = foo(**{'a': 12})" << "int"; QTest::newRow("dict_norm_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2)" << "unsure (int, float)"; QTest::newRow("multi_dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2, **{'c': ''})" << "unsure (int, float, str)"; QTest::newRow("named_arg_type") << "def myfun(arg): return arg\ncheckme = myfun(arg=3)" << "int"; QTest::newRow("arg_args_type") << "def myfun(arg, *args): return args[0]\n" "checkme = myfun(3, str())" << "str"; QTest::newRow("arg_kwargs_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(12, a=str())" << "str"; QTest::newRow("named_kwargs_type_1") << "def myfun(arg, **kwargs): return arg\n" "checkme = myfun(arg=12, a=str())" << "int"; QTest::newRow("named_kwargs_type_2") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(arg=12, a=str())" << "str"; QTest::newRow("kwargs_named_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(a=str(), arg=12)" << "str"; QTest::newRow("varied_args_type_1") << "def myfun(arg, *args, **kwargs): return arg\n" "checkme = myfun(1, 1.5, a=str())" << "int"; QTest::newRow("varied_args_type_2") << "def myfun(arg, *args, **kwargs): return args[0]\n" "checkme = myfun(1, 1.5, a=str())" << "float"; QTest::newRow("varied_args_type_3") << "def myfun(arg, *args, **kwargs): return kwargs['a']\n" "checkme = myfun(1, 1.5, a=str())" << "str"; QTest::newRow("method_args_type_1") << "class MyClass:\n" " def method(self, arg): return self\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("method_args_type_2") << "class MyClass:\n" " def method(self, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("clsmethod_args_type_1") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return cls\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("clsmethod_args_type_2") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_args_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_vararg_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg, *args): return args[0]\n" "checkme = MyClass().method(12, 2.5)" << "float"; QTest::newRow("method_explicit_self") << "class MyClass:\n" " def method(self, arg): return arg\n" "instance = MyClass()\n" "checkme = MyClass.method(instance, 12)" << "int"; QTest::newRow("method_vararg_explicit_self") << "class MyClass:\n" " def foo(self, arg, *args): return args[0]\n" "mc = MyClass()\n" "checkme = MyClass.foo(mc, 'str', 3, 4.5)" << "int"; QTest::newRow("clsmethod_explicit_self") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg1, arg2): return arg2\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "int"; QTest::newRow("staticmethod_explicit_self") << "class MyClass:\n" " @staticmethod\n" " def method(arg1, arg2): return arg1\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "str"; QTest::newRow("parent_constructor_arg_type") << "class Base:\n" // https://bugs.kde.org/show_bug.cgi?id=369364 " def __init__(self, foo):\n" " self.foo = foo\n" "class Derived(Base):\n" " def __init__(self, foo):\n" " Base.__init__(self, foo)\n" "instance = Derived('string')\n" "checkme = instance.foo" << "str"; QTest::newRow("tuple_unsure") << "q = (3, str())\nq=(str(), 3)\ncheckme, _ = q" << "unsure (int, str)"; QTest::newRow("custom_iterable") << "class Gen2:\n" " def __iter__(self): return self\n" " def __next__(self): return 'blah'\n" "for checkme in Gen2(): pass" << "str"; QTest::newRow("separate_iterator") << "class Foo:\n" " def __iter__(self): return Bar()\n" " def __next__(self): return 'blah'\n" // Not used (or shouldn't be!) "class Bar:\n" " def __next__(self): return {1}\n" "checkme = [a for a in Foo()]" << "list of set of int"; QTest::newRow("return_builtin_iterator") << "class Gen2:\n" " contents = [1, 2, 3]\n" " def __iter__(self): return iter(Gen2.contents)\n" "for checkme in Gen2(): pass" << "int"; QTest::newRow("init_class") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "checkme = Foo()\n" << "Foo"; QTest::newRow("init_class_no_decl") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "a = [Foo]\n" "checkme = a[0]()\n" << "Foo"; QTest::newRow("call_class") << "class Foo:\n" " def __call__(self):\n" " return 0\n" "f = Foo()\n" "checkme = f()\n" << "int"; QTest::newRow("call_class_no_decl") << "class Foo:\n" " def __call__(self): return 1.5\n" "a = [Foo()]\n" "checkme = a[0]()" << "float"; QTest::newRow("classmethod") << "class Foo:\n" " @classmethod\n" " def foo(cls):\n" " k = cls()\n" " return k\n" "f = Foo.foo()\n" "checkme = f\n" << "Foo"; QTest::newRow("tuple_listof") << "l = [(1, 2), (3, 4)]\ncheckme = l[1][0]" << "int"; QTest::newRow("getitem") << "class c:\n def __getitem__(self, slice): return 3.14\na = c()\ncheckme = a[2]" << "float"; QTest::newRow("constructor_type_deduction") << "class myclass:\n" "\tdef __init__(self, param): self.foo=param\n" "checkme = myclass(3).foo" << "int"; QTest::newRow("simpe_type_deduction") << "def myfunc(arg): return arg\n" "checkme = myfunc(3)" << "int"; QTest::newRow("functionCall_functionArg_part1") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "checkme = f1()" << "str"; QTest::newRow("functionCall_functionArg_part2") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = identity(getstr)\n" "checkme = f1()\n" << "str"; QTest::newRow("functionCall_functionArg_full") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "f2 = identity(getstr)\n" "a = getstr()\n" "b = f1()\n" "checkme = f2()\n" << "str"; QTest::newRow("vararg_before_other_args") << "def myfun(a, b, *z, x): return z[0]\n" "checkme = myfun(False, False, 1, x = False)" << "int"; QTest::newRow("vararg_before_other_args2") << "def myfun(a, b, *z, x): return z[3]\n" "checkme = myfun(False, False, 1, 2, 3, \"str\", x = False)" << "str"; QTest::newRow("vararg_constructor") << "class myclass():\n" " def __init__(self, *arg): self.prop = arg[0]\n" "obj = myclass(3, 5); checkme = obj.prop" << "int"; QTest::newRow("global_variable") << "a = 3\n" "def f1():\n" " global a\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_variable2") << "a = 3\n" "def f1():\n" " global a\n" " a = \"str\"\n" " return a\n" "checkme = f1()\n" << "str"; QTest::newRow("global_scope_variable") << "a = 3\n" "def f1():\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_no_toplevel_dec") << "def f1():\n" " global a\n a = 3\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("top_level_vs_class_member") << "var = 3\n" "class myclass:\n" " def __init__(self): self.var = \"str\"\n" " def f1(): return var\n" "checkme = myclass.f1()" << "int"; } typedef QPair pair; void PyDUChainTest::testImportDeclarations() { QFETCH(QString, code); QFETCH(QStringList, expectedDecls); QFETCH(bool, shouldBeAliased); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); foreach ( const QString& expected, expectedDecls ) { bool found = false; QString name = expected; QList decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); qCDebug(KDEV_PYTHON_DUCHAIN) << "FOUND DECLARATIONS:"; foreach ( const pair& current, decls ) { qCDebug(KDEV_PYTHON_DUCHAIN) << current.first->toString() << current.first->identifier().identifier().byteArray() << name; } foreach ( const pair& current, decls ) { if ( ! ( current.first->identifier().identifier().byteArray() == name ) ) continue; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found: " << current.first->toString() << " for " << name; AliasDeclaration* isAliased = dynamic_cast(current.first); if ( isAliased && shouldBeAliased ) { found = true; // TODO fixme } else if ( ! isAliased && ! shouldBeAliased ) { found = true; } } QVERIFY(found); } } void PyDUChainTest::testProblemCount() { QFETCH(QString, code); QFETCH(int, problemsCount); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QCOMPARE(ctx->problems().size(), problemsCount); } void PyDUChainTest::testProblemCount_data() { QTest::addColumn("code"); QTest::addColumn("problemsCount"); QTest::newRow("list_comp") << "[foo for foo in range(3)]" << 0; QTest::newRow("list_comp_wrong") << "[bar for foo in range(3)]" << 1; QTest::newRow("list_comp_staticmethod") << "class A:\n @staticmethod\n def func(cls):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_decorator") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_wrong") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("list_comp_staticmethod_wrong") << "class A:\n @staticmethod\n def func(cls):\n" " [x for a in [1, 2, 3]]" << 1; } void PyDUChainTest::testImportDeclarations_data() { QTest::addColumn("code"); QTest::addColumn("expectedDecls"); QTest::addColumn("shouldBeAliased"); QTest::newRow("from_import") << "from testImportDeclarations.i import checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("import") << "import testImportDeclarations.i" << ( QStringList() << "testImportDeclarations" ) << false; QTest::newRow("import_as") << "import testImportDeclarations.i as checkme" << ( QStringList() << "checkme" ) << false; QTest::newRow("from_import_as") << "from testImportDeclarations.i import checkme as checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("from_import_missing") << "from testImportDeclarations.i import missing as checkme" << ( QStringList() ) << true; } typedef QPair p; void PyDUChainTest::testAutocompletionFlickering() { TestFile f("foo = 3\nfoo2 = 2\nfo", "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx1 = f.topContext(); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx1); QList

decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); QList declIds; foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); f.setFileContents("foo = 3\nfoo2 = 2\nfoo"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx2 = f.topContext(); QVERIFY(ctx2); lock.lock(); QList

decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@1: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); qDebug() << "========================="; TestFile g("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfo", "py"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx1 = g.topContext(); lock.lock(); QVERIFY(ctx1); decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); declIds.clear(); foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); g.setFileContents("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfoo"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx2 = g.topContext(); QVERIFY(ctx2); lock.lock(); decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@2: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); } void PyDUChainTest::testFunctionHints() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testFunctionHints_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("func_return_type") << "def myfun(arg) -> int: pass\ncheckme = myfun(\"3\")" << "unsure (void, int)"; QTest::newRow("argument_type") << "def myfun(arg : int): return arg\ncheckme = myfun(foobar)" << "int"; QTest::newRow("argument_type_only_if_typeof") << "def myfun(arg : 3): return arg\ncheckme = myfun(foobar)" << "mixed"; } void PyDUChainTest::testHintedTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testHintedTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("simple_hint") << "def myfunc(x): return x\ncheckme = myfunc(3)" << "int"; QTest::newRow("hint_unsure") << "def myfunc(x): return x\nmyfunc(3.5)\ncheckme = myfunc(3)" << "unsure (float, int)"; QTest::newRow("unsure_attribute") << "def myfunc(x): return x.capitalize()\nmyfunc(3.5)\ncheckme = myfunc(str())" << "str"; } void PyDUChainTest::testDecorators() { QFETCH(QString, code); // QFETCH(int, amountOfDecorators); QFETCH(QStringList, names); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); foreach ( const QString& decoratorName, names ) { QVERIFY(Helper::findDecoratorByName(decl, decoratorName)); } } void PyDUChainTest::testDecorators_data() { QTest::addColumn("code"); QTest::addColumn("amountOfDecorators"); QTest::addColumn("names"); QTest::newRow("one_decorator") << "@foo\ndef func(): pass" << 1 << ( QStringList() << "foo" ); QTest::newRow("decorator_with_args") << "@foo(2, \"bar\")\ndef func(): pass" << 1 << ( QStringList() << "foo"); QTest::newRow("two_decorators") << "@foo\n@bar(17)\ndef func(): pass" << 2 << ( QStringList() << "foo" << "bar" ); } void PyDUChainTest::testOperators() { QFETCH(QString, code); QFETCH(QString, expectedType); code.prepend("from testOperators.example import *\n\n"); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QVERIFY(visitor->found); } void PyDUChainTest::testOperators_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("add") << "checkme = Example() + Example()" << "Add"; QTest::newRow("sub") << "checkme = Example() - Example()" << "Sub"; QTest::newRow("mul") << "checkme = Example() * Example()" << "Mul"; QTest::newRow("floordiv") << "checkme = Example() // Example()" << "Floordiv"; QTest::newRow("mod") << "checkme = Example() % Example()" << "Mod"; QTest::newRow("pow") << "checkme = Example() ** Example()" << "Pow"; QTest::newRow("lshift") << "checkme = Example() << Example()" << "Lshift"; QTest::newRow("rshift") << "checkme = Example() >> Example()" << "Rshift"; QTest::newRow("and") << "checkme = Example() & Example()" << "And"; QTest::newRow("xor") << "checkme = Example() ^ Example()" << "Xor"; QTest::newRow("or") << "checkme = Example() | Example()" << "Or"; } void PyDUChainTest::testFunctionArgs() { ReferencedTopDUContext ctx = parse("def ASDF(arg1, arg2):\n" " arg1 = arg2"); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVERIFY(m_ast); // dumpDUContext(ctx); QCOMPARE(ctx->childContexts().size(), 2); DUContext* funcArgCtx = ctx->childContexts().first(); QCOMPARE(funcArgCtx->type(), DUContext::Function); QCOMPARE(funcArgCtx->localDeclarations().size(), 2); QVERIFY(!funcArgCtx->owner()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); QCOMPARE(decl->type()->arguments().length(), 2); qDebug() << decl->type()->arguments().length() << 2; DUContext* funcBodyCtx = ctx->childContexts().last(); QCOMPARE(funcBodyCtx->type(), DUContext::Other); QVERIFY(funcBodyCtx->owner()); QVERIFY(funcBodyCtx->localDeclarations().isEmpty()); } void PyDUChainTest::testInheritance() { QFETCH(QString, code); QFETCH(int, expectedBaseClasses); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList

decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); bool found = false; bool classDeclFound = false; foreach ( const p& item, decls ) { if ( item.first->identifier().toString() == "B" ) { auto klass = dynamic_cast(item.first); QVERIFY(klass); QCOMPARE(klass->baseClassesSize(), static_cast(expectedBaseClasses)); classDeclFound = true; } if ( item.first->identifier().toString() == "checkme" ) { QCOMPARE(item.first->abstractType()->toString(), QString("int")); found = true; } } QVERIFY(found); QVERIFY(classDeclFound); } void PyDUChainTest::testInheritance_data() { QTest::addColumn("code"); QTest::addColumn("expectedBaseClasses"); QTest::newRow("simple") << "class A():\n\tattr = 3\n\nclass B(A):\n\tpass\n\ninst=B()\ncheckme = inst.attr" << 1; QTest::newRow("context_import_prereq") << "import testInheritance.i\ninst=testInheritance.i.testclass()\n" "checkme = inst.attr\nclass B(): pass" << 1; // 1 because object QTest::newRow("context_import") << "import testInheritance.i\n\nclass B(testInheritance.i.testclass):\n" "\ti = 4\n\ninst=B()\ncheckme = inst.attr" << 1; } void PyDUChainTest::testClassContextRanges() { QString code = "class my_class():\n pass\n \n \n \n \n"; ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; DUContext* classContext = ctx->findContextAt(CursorInRevision(5, 0)); QVERIFY(classContext); QVERIFY(classContext->type() == DUContext::Class); } void PyDUChainTest::testContainerTypes() { QFETCH(QString, code); QFETCH(QString, contenttype); QFETCH(bool, use_type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList decls = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(decls.length() > 0); QVERIFY(decls.first()->abstractType()); if ( ! use_type ) { auto type = ListType::Ptr::dynamicCast(decls.first()->abstractType()); QVERIFY(type); QVERIFY(type->contentType()); QCOMPARE(type->contentType().abstractType()->toString(), contenttype); } else { QVERIFY(decls.first()->abstractType()); QEXPECT_FAIL("dict_of_int_call", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("dict_from_tuples", "returnContentEqualsContentOf isn't suitable", Continue); QCOMPARE(decls.first()->abstractType()->toString(), contenttype); } } void PyDUChainTest::testContainerTypes_data() { QTest::addColumn("code"); QTest::addColumn("contenttype"); QTest::addColumn("use_type"); QTest::newRow("list_of_int") << "checkme = [1, 2, 3]" << "int" << false; QTest::newRow("list_from_unpacked") << "foo = [1.3]\ncheckme = [1, *foo, 3]" << "unsure (int, float)" << false; QTest::newRow("list_of_int_call") << "checkme = list([1, 2, 3])" << "int" << false; QTest::newRow("list_from_tuple") << "checkme = list((1, 2, 3))" << "int" << false; QTest::newRow("list_from_dict") << "checkme = list({'a':1, 'b':2})" << "str" << false; // Gets key type! QTest::newRow("list_from_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = list(MyClass())" << "float" << false; QTest::newRow("generator") << "checkme = [i for i in [1, 2, 3]]" << "int" << false; QTest::newRow("list_access") << "list = [1, 2, 3]\ncheckme = list[0]" << "int" << true; QTest::newRow("set_of_int") << "checkme = {1, 2, 3}" << "int" << false; QTest::newRow("set_from_unpacked") << "foo = [1.3]\ncheckme = {1, *foo, 3}" << "unsure (int, float)" << false; QTest::newRow("set_of_int_call") << "checkme = set({1, 2, 3})" << "int" << false; QTest::newRow("set_from_tuple") << "checkme = set((1, 2, 3))" << "int" << false; QTest::newRow("set_generator") << "checkme = {i for i in [1, 2, 3]}" << "int" << false; - QTest::newRow("dict_of_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; + QTest::newRow("dict_of_str_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; + QTest::newRow("frozenset_of_int_call") << "checkme = frozenset({1, 2, 3})" << "int" << false; + QTest::newRow("dict_of_int") << "checkme = {a:1, b:2, c:3}" << "int" << false; QTest::newRow("dict_from_unpacked") << "checkme = {**{'a': 1}}" << "dict of str : int" << true; QTest::newRow("dict_from_varied") << "checkme = {**{'a': 1}, 1: 1.5}" << "dict of unsure (str, int) : unsure (int, float)" << true; QTest::newRow("dict_of_int_call") << "checkme = dict({'a':1, 'b':2, 'c':3})" << "dict of str : int" << true; QTest::newRow("dict_from_tuples") << "checkme = dict([('a', 1), ('b', 2)])" << "dict of str : int" << true; QTest::newRow("dict_generator") << "checkme = {\"Foo\":i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_access") << "list = {'a':1, 'b':2, 'c':3}\ncheckme = list[0]" << "int" << true; QTest::newRow("generator_attribute") << "checkme = [item.capitalize() for item in ['foobar']]" << "str" << false; QTest::newRow("cannot_change_type") << "checkme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("cannot_change_type2") << "[1, 2, 3].append(5)\ncheckme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("list_append") << "d = []\nd.append(3)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend") << "d = []; q = [int()]\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_tuple") << "d = []; q = (1, 2)\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = []\ncheckme.extend(MyClass())" << "float" << false; QTest::newRow("for_loop") << "d = [3]\nfor item in d:\n checkme = item" << "int" << true; QTest::newRow("for_loop_unsure") << "d = [3, \"foo\"]\nfor item in d:\n checkme = item" << "unsure (int, str)" << true; QTest::newRow("for_loop_tuple_1") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = a" << "int" << true; QTest::newRow("for_loop_tuple_2") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = b" << "float" << true; QTest::newRow("for_loop_tuple_unsure") << "d = [(3, 3.5), (3.5, 3)]\nfor a, b in d:\n checkme = b" << "unsure (float, int)" << true; // Proposed by Nicolás Alvarez; why not? https://bugs.kde.org/show_bug.cgi?id=359915 QTest::newRow("comprehension_messy") << "users = {'a':19, 'b':42, 'c':35}\n" "sorted_list = sorted(users.items(), key=lambda kv: (-kv[1], kv[0]))\n" "checkme = [k for r,(k,v) in enumerate(sorted_list, 1)]" << "list of str" << true; QTest::newRow("comprehension_multiline") << "checkme = [a for\n a in \n (1, 2)]" << "list of int" << true; // From https://bugs.kde.org/show_bug.cgi?id=359912 QTest::newRow("subscript_multi") << "class Middle:\n def __getitem__(self, key):\n return str()\n" "class Outer:\n def __getitem__(self, key):\n return Middle()\n" "aaa = Outer()\ncheckme = aaa[0][0]" << "str" << true; QTest::newRow("subscript_func_call") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "def bar():\n return Foo()\n" "checkme = bar()[0]" << "str" << true; QTest::newRow("subscript_unknown_index") << "a = 1,str()\ncheckme = a[5-4]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure") << "a = 1,2\na=[str()]\ncheckme = a[0]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure_getitem") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "class Bar:\n def __getitem__(self, key):\n return float()\n" "a = Foo()\na=Bar()\na=[1,2]\ncheckme = a[1]" << "unsure (str, float, int)" << true; } void PyDUChainTest::testVariableCreation() { QFETCH(QString, code); QFETCH(QStringList, expected_local_declarations); QFETCH(QStringList, expected_types); ReferencedTopDUContext top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto localDecls = top->localDeclarations(); QVector localDeclNames; for ( const Declaration* d: localDecls ) { localDeclNames.append(d->identifier().toString()); } Q_ASSERT(expected_types.size() == expected_local_declarations.size()); int offset = 0; for ( const QString& expected : expected_local_declarations ) { int index = localDeclNames.indexOf(expected); QVERIFY(index != -1); QVERIFY(localDecls[index]->abstractType()); QCOMPARE(localDecls[index]->abstractType()->toString(), expected_types[offset]); offset++; } } void PyDUChainTest::testVariableCreation_data() { QTest::addColumn("code"); QTest::addColumn("expected_local_declarations"); QTest::addColumn("expected_types"); QTest::newRow("simple") << "a = 3" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("tuple_wrong") << "a, b = 3" << QStringList{"a", "b"} << QStringList{"mixed", "mixed"}; QTest::newRow("tuple_unpack_inplace") << "a, b = 3, 5.5" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_indirect") << "c = 3, 3.5\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_stacked_inplace") << "a, (b, c) = 1, (2, 3.5)" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("tuple_unpack_stacked_indirect") << "d = 3.5, (3, 1)\na, (b, c) = d" << QStringList{"a", "b", "c"} << QStringList{"float", "int", "int"}; QTest::newRow("unpack_from_list_inplace") << "a, b = [1, 2, 3]" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_from_list_indirect") << "c = [1, 2, 3]\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_custom_iterable") << "class Foo:\n" " def __iter__(self): return self\n" " def __next__(self): return 1.5\n" "a, *b = Foo()" << QStringList{"a", "b"} << QStringList {"float", "list of float"}; QTest::newRow("for_loop_simple") << "for i in range(3): pass" << QStringList{"i"} << QStringList{"int"}; QTest::newRow("for_loop_unpack") << "for a, b in [(3, 5.1)]: pass" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("for_loop_stacked") << "for a, (b, c) in [(1, (2, 3.5))]: pass" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("for_loop_tuple") << "for a in 1, 2: pass" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("for_loop_dict") << "for a in {'foo': 1}: pass" << QStringList{"a"} << QStringList{"str"}; } void PyDUChainTest::testCleanupMultiplePasses() { for ( int j = 0; j < 20; j++ ) { ReferencedTopDUContext top = parse("from testCleanupMultiplePasses import foo\ndef fonc(): return 3+2j\nfoo.foo.func = fonc"); } } void PyDUChainTest::testManyDeclarations() { ReferencedTopDUContext top = parse("from testManyDeclarations import test\nk=test.Foo()"); } void PyDUChainTest::testComments() { QFETCH(QString, code); auto top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto decls = top->findDeclarations(QualifiedIdentifier("a")); QCOMPARE(decls.size(), 1); auto a = decls.first(); QCOMPARE(a->comment(), QByteArray("comment")); decls = top->findDeclarations(QualifiedIdentifier("b")); if ( decls.isEmpty() ) { decls = top->childContexts().last()->findDeclarations(QualifiedIdentifier("b")); } auto b = decls.first(); QCOMPARE(b->comment(), QByteArray()); } void PyDUChainTest::testComments_data() { QTest::addColumn("code"); QTest::newRow("variable") << "b=5\n\"\"\"comment\"\"\"\na=5\nb=5"; QTest::newRow("function") << "def a():\n \"\"\"comment\"\"\"\n b=5"; QTest::newRow("class") << "class a:\n \"\"\"comment\"\"\"\n b=5"; } diff --git a/parser/astdefaultvisitor.cpp b/parser/astdefaultvisitor.cpp index 0ac0e50e..0df6e6ee 100644 --- a/parser/astdefaultvisitor.cpp +++ b/parser/astdefaultvisitor.cpp @@ -1,446 +1,448 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astdefaultvisitor.h" #include "ast.h" namespace Python { void free_ast_recursive(CodeAst *node) { AstFreeVisitor v; v.visitCode(node); } AstDefaultVisitor::AstDefaultVisitor() { } AstDefaultVisitor::~AstDefaultVisitor() { } // The Ast "ends" here, those dont have child nodes // note that Identifier is not a node in this Ast void AstDefaultVisitor::visitNameConstant(NameConstantAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitPass(PassAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNonlocal(NonlocalAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBreak(BreakAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitContinue(ContinueAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitEllipsis(EllipsisAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNumber(NumberAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitString(StringAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBytes(BytesAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitIdentifier(Identifier* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitJoinedString(JoinedStringAst* node) { - foreach (Ast* value, node->values) { - visitNode(value); - } +//TODO: Fix range/context/??? bugs, then re-enable this. + Q_UNUSED(node); +// foreach (Ast* value, node->values) { +// visitNode(value); +// } } void AstDefaultVisitor::visitFormattedValue(FormattedValueAst* node) { visitNode(node->value); visitNode(node->formatSpec); } void AstDefaultVisitor::visitStarred(StarredAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitArg(ArgAst* node) { visitNode(node->annotation); visitNode(node->argumentName); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitAlias(AliasAst* node) { visitIdentifier(node->name); visitIdentifier(node->asName); } void AstDefaultVisitor::visitName(NameAst* node) { visitIdentifier(node->identifier); } void AstDefaultVisitor::visitGlobal(GlobalAst* node) { foreach (Identifier* identifier, node->names) { visitIdentifier(identifier); } } void AstDefaultVisitor::visitCode(CodeAst* node) { foreach (Ast* statement, node->body) { visitNode(statement); } visitIdentifier(node->name); } void AstDefaultVisitor::visitExpression(ExpressionAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitYieldFrom(YieldFromAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAssertion(AssertionAst* node) { visitNode(node->condition); visitNode(node->message); } void AstDefaultVisitor::visitDelete(DeleteAst* node) { foreach (ExpressionAst* expression, node->targets) { visitNode(expression); } } void AstDefaultVisitor::visitExtendedSlice(ExtendedSliceAst* node) { foreach (SliceAst* slice, node->dims) { visitNode(slice); } } void AstDefaultVisitor::visitFor(ForAst* node) { visitNode(node->target); visitNode(node->iterator); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitGeneratorExpression(GeneratorExpressionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitIf(IfAst* node) { visitNode(node->condition); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitIfExpression(IfExpressionAst* node) { visitNode(node->condition); visitNode(node->body); visitNode(node->orelse); } void AstDefaultVisitor::visitImport(ImportAst* node) { foreach (AliasAst* alias, node->names) { visitNode(alias); } } void AstDefaultVisitor::visitImportFrom(ImportFromAst* node) { foreach (AliasAst* alias, node->names) { visitNode(alias); } visitIdentifier(node->module); } void AstDefaultVisitor::visitIndex(IndexAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitLambda(LambdaAst* node) { visitNode(node->arguments); visitNode(node->body); } void AstDefaultVisitor::visitRaise(RaiseAst* node) { visitNode(node->type); } void AstDefaultVisitor::visitReturn(ReturnAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAwait(AwaitAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitSet(SetAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitSetComprehension(SetComprehensionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitSlice(SliceAst* node) { visitNode(node->lower); visitNode(node->upper); visitNode(node->step); } void AstDefaultVisitor::visitSubscript(SubscriptAst* node) { visitNode(node->value); visitNode(node->slice); } void AstDefaultVisitor::visitTry(TryAst* node) { foreach (Ast* statement, node->body) { visitNode(statement); } foreach (ExceptionHandlerAst* handler, node->handlers) { visitNode(handler); } foreach (Ast* statement, node->orelse) { visitNode(statement); } foreach (Ast* statement, node->finally) { visitNode(statement); } } void AstDefaultVisitor::visitTuple(TupleAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitUnaryOperation(UnaryOperationAst* node) { visitNode(node->operand); } void AstDefaultVisitor::visitWhile(WhileAst* node) { visitNode(node->condition); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitWith(WithAst* node) { foreach (Ast* item, node->items) { visitNode(item); } foreach (Ast* statement, node->body) { visitNode(statement); } } void AstDefaultVisitor::visitWithItem(WithItemAst* node) { visitNode(node->contextExpression); visitNode(node->optionalVars); } void AstDefaultVisitor::visitYield(YieldAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitList(ListAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitListComprehension(ListComprehensionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitExceptionHandler(ExceptionHandlerAst* node) { visitNode(node->type); visitNode(node->name); foreach (Ast* statement, node->body) { visitNode(statement); } } void AstDefaultVisitor::visitDict(DictAst* node) { foreach (ExpressionAst* expression, node->keys) { visitNode(expression); } foreach (ExpressionAst* expression, node->values) { visitNode(expression); } } void AstDefaultVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { visitNode(node->key); visitNode(node->value); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitAugmentedAssignment(AugmentedAssignmentAst* node) { visitNode(node->target); visitNode(node->value); } void AstDefaultVisitor::visitAnnotationAssignment(AnnotationAssignmentAst* node) { visitNode(node->target); visitNode(node->annotation); visitNode(node->value); } void AstDefaultVisitor::visitBinaryOperation(BinaryOperationAst* node) { visitNode(node->lhs); visitNode(node->rhs); } void AstDefaultVisitor::visitBooleanOperation(BooleanOperationAst* node) { foreach (ExpressionAst* expression, node->values) { visitNode(expression); } } void AstDefaultVisitor::visitClassDefinition(ClassDefinitionAst* node) { foreach (ExpressionAst* expression, node->baseClasses) { visitNode(expression); } foreach (Ast* statement, node->body) { visitNode(statement); } foreach (ExpressionAst* expression, node->decorators) { visitNode(expression); } visitIdentifier(node->name); } void AstDefaultVisitor::visitCompare(CompareAst* node) { visitNode(node->leftmostElement); foreach (ExpressionAst* expression, node->comparands) { visitNode(expression); } } void AstDefaultVisitor::visitComprehension(ComprehensionAst* node) { visitNode(node->target); visitNode(node->iterator); foreach (ExpressionAst* expression, node->conditions) { visitNode(expression); } } void AstDefaultVisitor::visitAssignment(AssignmentAst* node) { foreach (ExpressionAst* expression, node->targets) { visitNode(expression); }; visitNode(node->value); } void AstDefaultVisitor::visitCall(CallAst* node) { visitNode(node->function); foreach (ExpressionAst* argument, node->arguments) { visitNode(argument); } foreach (KeywordAst* kwd, node->keywords) { visitNode(kwd); } } void AstDefaultVisitor::visitFunctionDefinition(FunctionDefinitionAst* node) { foreach (ExpressionAst* decorator, node->decorators) { visitNode(decorator); } visitNode(node->arguments); visitNode(node->returns); foreach (Ast* stmt, node->body) { visitNode(stmt); } visitIdentifier(node->name); } void AstDefaultVisitor::visitAttribute(AttributeAst* node) { visitNode(node->value); visitIdentifier(node->attribute); } void AstDefaultVisitor::visitKeyword(KeywordAst* node) { visitNode(node->value); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitArguments(ArgumentsAst* node) { foreach (ArgAst* expression, node->arguments) { visitNode(expression); } foreach (ExpressionAst* expression, node->defaultValues ) { visitNode(expression); } } }