diff --git a/codecompletion/context.cpp b/codecompletion/context.cpp index 3d84c80c..b08ed217 100644 --- a/codecompletion/context.cpp +++ b/codecompletion/context.cpp @@ -1,1305 +1,1305 @@ /***************************************************************************** * Copyright (c) 2011-2012 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "context.h" #include "items/keyword.h" #include "items/importfile.h" #include "items/functiondeclaration.h" #include "items/implementfunction.h" #include "items/missingincludeitem.h" #include "items/replacementvariable.h" #include "worker.h" #include "helpers.h" #include "duchain/pythoneditorintegrator.h" #include "duchain/expressionvisitor.h" #include "duchain/declarationbuilder.h" #include "duchain/helpers.h" #include "duchain/types/unsuretype.h" #include "duchain/navigation/navigationwidget.h" #include "parser/astbuilder.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "codecompletiondebug.h" using namespace KTextEditor; using namespace KDevelop; namespace Python { PythonCodeCompletionContext::ItemTypeHint PythonCodeCompletionContext::itemTypeHint() { return m_itemTypeHint; } PythonCodeCompletionContext::CompletionContextType PythonCodeCompletionContext::completionContextType() { return m_operation; } std::unique_ptr visitorForString(QString str, DUContext* context, CursorInRevision scanUntil = CursorInRevision::invalid()) { ENSURE_CHAIN_NOT_LOCKED AstBuilder builder; CodeAst::Ptr tmpAst = builder.parse({}, str); if ( ! tmpAst ) { return std::unique_ptr(nullptr); } ExpressionVisitor* v = new ExpressionVisitor(context); v->enableGlobalSearching(); if ( scanUntil.isValid() ) { v->scanUntil(scanUntil); v->enableUnknownNameReporting(); } v->visitCode(tmpAst.data()); return std::unique_ptr(v); } void PythonCodeCompletionContext::eventuallyAddGroup(QString name, int priority, QList items) { if ( items.isEmpty() ) { return; } KDevelop::CompletionCustomGroupNode* node = new KDevelop::CompletionCustomGroupNode(name, priority); node->appendChildren(items); m_storedGroups << CompletionTreeElementPointer(node); } QList< CompletionTreeElementPointer > PythonCodeCompletionContext::ungroupedElements() { return m_storedGroups; } static QList setOmitParentheses(QList items) { for ( auto current: items ) { if ( auto func = dynamic_cast(current.data()) ) { func->setDoNotCall(true); } } return items; }; PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::shebangItems() { KeywordItem::Flags f = (KeywordItem::Flags) ( KeywordItem::ForceLineBeginning | KeywordItem::ImportantItem ); QList shebangGroup; if ( m_position.line == 0 && ( m_text.startsWith('#') || m_text.isEmpty() ) ) { QString i18ndescr = i18n("insert Shebang line"); shebangGroup << CompletionTreeItemPointer(new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), "#!/usr/bin/env python\n", i18ndescr, f)); shebangGroup << CompletionTreeItemPointer(new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), "#!/usr/bin/env python3\n", i18ndescr, f)); } else if ( m_position.line <= 1 && m_text.endsWith('#') ) { shebangGroup << CompletionTreeItemPointer(new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), "# -*- coding:utf-8 -*-\n\n", i18n("specify document encoding"), f)); } eventuallyAddGroup(i18n("Add file header"), 1000, shebangGroup); return ItemList(); } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::functionCallItems() { ItemList resultingItems; // gather additional items to show above the real ones (for parameters, and stuff) FunctionDeclaration* functionCalled = 0; auto v = visitorForString(m_guessTypeOfExpression, m_duContext.data()); DUChainReadLocker lock; if ( ! v || ! v->lastDeclaration() ) { qCWarning(KDEV_PYTHON_CODECOMPLETION) << "Did not receive a function declaration from expression visitor! Not offering call tips."; qCWarning(KDEV_PYTHON_CODECOMPLETION) << "Tried: " << m_guessTypeOfExpression; return resultingItems; } - functionCalled = Helper::functionDeclarationForCalledDeclaration(v->lastDeclaration()).first.data(); + functionCalled = Helper::functionForCalled(v->lastDeclaration().data()).declaration; auto current = Helper::resolveAliasDeclaration(functionCalled); QList calltips; if ( current && current->isFunctionDeclaration() ) { calltips << current; } auto calltipItems = declarationListToItemList(calltips); foreach ( CompletionTreeItemPointer current, calltipItems ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Adding calltip item, at argument:" << m_alreadyGivenParametersCount+1; FunctionDeclarationCompletionItem* item = static_cast(current.data()); item->setAtArgument(m_alreadyGivenParametersCount + 1); item->setDepth(depth()); } resultingItems.append(calltipItems); // If this is the top-level calltip, add additional items for the default-parameters of the function, // but only if all non-default arguments (the mandatory ones) already have been provided. // TODO fancy feature: Filter out already provided default-parameters if ( depth() != 1 || ! functionCalled ) { return resultingItems; } if ( DUContext* args = DUChainUtils::getArgumentContext(functionCalled) ) { int normalParameters = args->localDeclarations().count() - functionCalled->defaultParametersSize(); if ( normalParameters > m_alreadyGivenParametersCount ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Not at default arguments yet"; return resultingItems; } for ( unsigned int i = 0; i < functionCalled->defaultParametersSize(); i++ ) { QString paramName = args->localDeclarations().at(normalParameters + i)->identifier().toString(); resultingItems << CompletionTreeItemPointer(new KeywordItem(CodeCompletionContext::Ptr(m_child), paramName + "=", i18n("specify default parameter"), KeywordItem::ImportantItem)); } qCDebug(KDEV_PYTHON_CODECOMPLETION) << "adding " << functionCalled->defaultParametersSize() << "default args"; } return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::defineItems() { DUChainReadLocker lock; ItemList resultingItems; // Find all base classes of the current class context if ( m_duContext->type() != DUContext::Class ) { qCWarning(KDEV_PYTHON_CODECOMPLETION) << "current context is not a class context, not offering define completion"; return resultingItems; } ClassDeclaration* klass = dynamic_cast(m_duContext->owner()); if ( ! klass ) { return resultingItems; } auto baseClassContexts = Helper::internalContextsForClass( klass->type(), m_duContext->topContext() ); // This class' context is put first in the list, so all functions existing here // can be skipped. baseClassContexts.removeAll(m_duContext.data()); baseClassContexts.prepend(m_duContext.data()); Q_ASSERT(baseClassContexts.size() >= 1); QList existingIdentifiers; bool isOwnContext = true; foreach ( DUContext* c, baseClassContexts ) { QList declarations = c->allDeclarations( CursorInRevision::invalid(), m_duContext->topContext(), false ); foreach ( const DeclarationDepthPair& d, declarations ) { if ( FunctionDeclaration* funcDecl = dynamic_cast(d.first) ) { // python does not have overloads or similar, so comparing the function names is enough. const IndexedString identifier = funcDecl->identifier().identifier(); if ( isOwnContext ) { existingIdentifiers << identifier; } if ( existingIdentifiers.contains(identifier) ) { continue; } existingIdentifiers << identifier; QStringList argumentNames; DUContext* argumentsContext = DUChainUtils::getArgumentContext(funcDecl); if ( argumentsContext ) { foreach ( Declaration* argument, argumentsContext->localDeclarations() ) { argumentNames << argument->identifier().toString(); } resultingItems << CompletionTreeItemPointer(new ImplementFunctionCompletionItem( funcDecl->identifier().toString(), argumentNames, m_indent) ); } } } isOwnContext = false; } return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::raiseItems() { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Finding items for raise statement"; DUChainReadLocker lock; ItemList resultingItems; ReferencedTopDUContext ctx = Helper::getDocumentationFileContext(); QList< Declaration* > declarations = ctx->findDeclarations(QualifiedIdentifier("BaseException")); if ( declarations.isEmpty() || ! declarations.first()->abstractType() ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "No valid exception classes found, aborting"; return resultingItems; } Declaration* base = declarations.first(); IndexedType baseType = base->abstractType()->indexed(); QList validDeclarations; ClassDeclaration* current = 0; StructureType::Ptr type; auto decls = m_duContext->topContext()->allDeclarations(CursorInRevision::invalid(), m_duContext->topContext()); foreach ( const DeclarationDepthPair d, decls ) { current = dynamic_cast(d.first); if ( ! current || ! current->baseClassesSize() ) { continue; } FOREACH_FUNCTION( const BaseClassInstance& base, current->baseClasses ) { if ( base.baseClass == baseType ) { validDeclarations << d; } } } auto items = declarationListToItemList(validDeclarations); if ( m_itemTypeHint == ClassTypeRequested ) { // used for except , we don't want the parentheses there items = setOmitParentheses(items); } resultingItems.append(items); return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::importFileItems() { DUChainReadLocker lock; ItemList resultingItems; qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Preparing to do autocompletion for import..."; m_maxFolderScanDepth = 1; resultingItems << includeItemsForSubmodule(""); return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::inheritanceItems() { ItemList resultingItems; DUChainReadLocker lock; qCDebug(KDEV_PYTHON_CODECOMPLETION) << "InheritanceCompletion"; QList declarations; if ( ! m_guessTypeOfExpression.isEmpty() ) { // The class completion is a member access lock.unlock(); auto v = visitorForString(m_guessTypeOfExpression, m_duContext.data()); lock.lock(); if ( v ) { TypePtr cls = StructureType::Ptr::dynamicCast(v->lastType()); if ( cls && cls->declaration(m_duContext->topContext()) ) { if ( DUContext* internal = cls->declaration(m_duContext->topContext())->internalContext() ) { declarations = internal->allDeclarations(m_position, m_duContext->topContext(), false); } } } } else { declarations = m_duContext->allDeclarations(m_position, m_duContext->topContext()); } QList remainingDeclarations; foreach ( const DeclarationDepthPair& d, declarations ) { Declaration* r = Helper::resolveAliasDeclaration(d.first); if ( r && r->topContext() == Helper::getDocumentationFileContext() ) { continue; } if ( r && dynamic_cast(r) ) { remainingDeclarations << d; } } resultingItems.append(setOmitParentheses(declarationListToItemList(remainingDeclarations))); return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::memberAccessItems() { ItemList resultingItems; auto v = visitorForString(m_guessTypeOfExpression, m_duContext.data()); DUChainReadLocker lock; if ( v ) { if ( v->lastType() ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << v->lastType()->toString(); resultingItems << getCompletionItemsForType(v->lastType()); } else { qCWarning(KDEV_PYTHON_CODECOMPLETION) << "Did not receive a type from expression visitor! Not offering autocompletion."; } } else { qCWarning(KDEV_PYTHON_CODECOMPLETION) << "Completion requested for syntactically invalid expression, not offering anything"; } // append eventually stripped postfix, for e.g. os.chdir| bool needDot = true; foreach ( const QChar& c, m_followingText ) { if ( needDot ) { m_guessTypeOfExpression.append('.'); needDot = false; } if ( c.isLetterOrNumber() || c == '_' ) { m_guessTypeOfExpression.append(c); } } if ( resultingItems.isEmpty() && m_fullCompletion ) { resultingItems << getMissingIncludeItems(m_guessTypeOfExpression); } return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::stringFormattingItems() { if ( ! m_fullCompletion ) { return ItemList(); } DUChainReadLocker lock; ItemList resultingItems; int cursorPosition; StringFormatter stringFormatter(CodeHelpers::extractStringUnderCursor(m_text, m_duContext->range().castToSimpleRange(), m_position.castToSimpleCursor(), &cursorPosition)); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Next identifier id: " << stringFormatter.nextIdentifierId(); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Cursor position in string: " << cursorPosition; bool insideReplacementVariable = stringFormatter.isInsideReplacementVariable(cursorPosition); RangeInString variablePosition = stringFormatter.getVariablePosition(cursorPosition); bool onVariableBoundary = (cursorPosition == variablePosition.beginIndex || cursorPosition == variablePosition.endIndex); if ( ! insideReplacementVariable || onVariableBoundary ) { resultingItems << CompletionTreeItemPointer(new ReplacementVariableItem( ReplacementVariable(QString::number(stringFormatter.nextIdentifierId())), i18n("Insert next positional variable"), false) ); resultingItems << CompletionTreeItemPointer(new ReplacementVariableItem( ReplacementVariable("${argument}"), i18n("Insert named variable"), true) ); } if ( ! insideReplacementVariable ) { return resultingItems; } const ReplacementVariable *variable = stringFormatter.getReplacementVariable(cursorPosition); // Convert the range relative to the beginning of the string to the absolute position // in the document. We can safely assume that the replacement variable is on one line, // because the regex does not allow newlines inside replacement variables. KTextEditor::Range range; range.setStart({m_position.line, m_position.column - (cursorPosition - variablePosition.beginIndex)}); range.setEnd({m_position.line, m_position.column + (variablePosition.endIndex - cursorPosition)}); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Variable under cursor: " << variable->toString(); bool hasNumericOnlyOption = variable->hasPrecision() || (variable->hasType() && variable->type() != 's') || variable->align() == '='; auto makeFormattingItem = [&variable, &range](const QChar& conversion, const QString& spec, const QString& description, bool useTemplateEngine) { return CompletionTreeItemPointer( new ReplacementVariableItem(ReplacementVariable(variable->identifier(), conversion, spec), description, useTemplateEngine, range) ); }; if ( ! variable->hasConversion() && ! hasNumericOnlyOption ) { auto addConversionItem = [&](const QChar& conversion, const QString& title) { resultingItems.append(makeFormattingItem(conversion, variable->formatSpec(), title, false)); }; addConversionItem('s', i18n("Format using str()")); addConversionItem('r', i18n("Format using repr()")); } if ( ! variable->hasFormatSpec() ) { auto addFormatSpec = [&](const QString& format, const QString& title, bool useTemplateEngine) { resultingItems.append(makeFormattingItem(variable->conversion(), format, title, useTemplateEngine)); }; addFormatSpec("<${width}", i18n("Format as left-aligned"), true); addFormatSpec(">${width}", i18n("Format as right-aligned"), true); addFormatSpec("^${width}", i18n("Format as centered"), true); // These options don't make sense if we've set conversion using str() or repr() if ( ! variable->hasConversion() ) { addFormatSpec(".${precision}", i18n("Specify precision"), true); addFormatSpec("%", i18n("Format as percentage"), false); addFormatSpec("c", i18n("Format as character"), false); addFormatSpec("b", i18n("Format as binary number"), false); addFormatSpec("o", i18n("Format as octal number"), false); addFormatSpec("x", i18n("Format as hexadecimal number"), false); addFormatSpec("e", i18n("Format in scientific (exponent) notation"), false); addFormatSpec("f", i18n("Format as fixed point number"), false); } } qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Resulting items size: " << resultingItems.size(); return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::keywordItems() { ItemList resultingItems; QStringList keywordItems; keywordItems << "def" << "class" << "lambda" << "global" << "import" << "from" << "while" << "for" << "yield" << "return"; foreach ( const QString& current, keywordItems ) { KeywordItem* k = new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), current + " ", ""); resultingItems << CompletionTreeItemPointer(k); } return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::classMemberInitItems() { DUChainReadLocker lock; ItemList resultingItems; Declaration* decl = duContext()->owner(); if ( ! decl ) { return resultingItems; } DUContext* args = DUChainUtils::getArgumentContext(duContext()->owner()); if ( ! args ) { return resultingItems; } if ( ! decl->isFunctionDeclaration() || decl->identifier() != KDevelop::Identifier("__init__") ) { return resultingItems; } // the current context actually belongs to a constructor foreach ( const Declaration* argument, args->localDeclarations() ) { const QString argName = argument->identifier().toString(); // Do not suggest "self.self = self" if ( argName == "self" ) { continue; } bool usedAlready = false; // Do not suggest arguments which already have a use in the context // This is uesful because you can then do { Ctrl+Space Enter Enter } while ( 1 ) // to initialize all available class variables, without using arrow keys. for ( int i = 0; i < duContext()->usesCount(); i++ ) { if ( duContext()->uses()[i].usedDeclaration(duContext()->topContext()) == argument ) { usedAlready = true; break; } } if ( usedAlready ) { continue; } const QString value = "self." + argName + " = " + argName; KeywordItem* item = new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), value, i18n("Initialize property"), KeywordItem::ImportantItem); resultingItems.append(CompletionTreeItemPointer(item)); } return resultingItems; } PythonCodeCompletionContext::ItemList PythonCodeCompletionContext::generatorItems() { ItemList resultingItems; QList items; auto v = visitorForString(m_guessTypeOfExpression, m_duContext.data(), m_position); DUChainReadLocker lock; if ( ! v || v->unknownNames().isEmpty() ) { return resultingItems; } if ( v->unknownNames().size() >= 2 ) { // we only take the first two, and only two. It gets too much items otherwise. QStringList combinations; auto names = v->unknownNames().toList(); combinations << names.at(0) + ", " + names.at(1); combinations << names.at(1) + ", " + names.at(0); foreach ( const QString& c, combinations ) { items << new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), "" + c + " in ", ""); } } foreach ( const QString& n, v->unknownNames() ) { items << new KeywordItem(KDevelop::CodeCompletionContext::Ptr(this), "" + n + " in ", ""); } foreach ( KeywordItem* item, items ) { resultingItems << CompletionTreeItemPointer(item); } return resultingItems; } QList PythonCodeCompletionContext::completionItems(bool& abort, bool fullCompletion) { m_fullCompletion = fullCompletion; ItemList resultingItems; qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Line: " << m_position.line; qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Completion type:" << m_operation; if ( m_operation != FunctionCallCompletion ) { resultingItems.append(shebangItems()); } // Find all calltips recursively if ( parentContext() ) { resultingItems.append(parentContext()->completionItems(abort, fullCompletion)); } if ( m_operation == PythonCodeCompletionContext::NoCompletion ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "no code completion"; } else if ( m_operation == PythonCodeCompletionContext::GeneratorVariableCompletion ) { resultingItems.append(generatorItems()); } else if ( m_operation == PythonCodeCompletionContext::FunctionCallCompletion ) { resultingItems.append(functionCallItems()); } else if ( m_operation == PythonCodeCompletionContext::DefineCompletion ) { resultingItems.append(defineItems()); } else if ( m_operation == PythonCodeCompletionContext::RaiseExceptionCompletion ) { resultingItems.append(raiseItems()); } else if ( m_operation == PythonCodeCompletionContext::ImportFileCompletion ) { resultingItems.append(importFileItems()); } else if ( m_operation == PythonCodeCompletionContext::ImportSubCompletion ) { DUChainReadLocker lock; resultingItems.append(includeItemsForSubmodule(m_searchImportItemsInModule)); } else if ( m_operation == PythonCodeCompletionContext::InheritanceCompletion ) { resultingItems.append(inheritanceItems()); } else if ( m_operation == PythonCodeCompletionContext::MemberAccessCompletion ) { resultingItems.append(memberAccessItems()); } else if ( m_operation == PythonCodeCompletionContext::StringFormattingCompletion ) { resultingItems.append(stringFormattingItems()); } else { // it's stupid to display a 3-letter completion item on manually invoked code completion and makes everything look crowded if ( m_operation == PythonCodeCompletionContext::NewStatementCompletion && ! fullCompletion ) { resultingItems.append(keywordItems()); } if ( m_operation == PythonCodeCompletionContext::NewStatementCompletion ) { // Eventually suggest initializing class members from constructor arguments resultingItems.append(classMemberInitItems()); } if ( abort ) { return ItemList(); } DUChainReadLocker lock; QList declarations = m_duContext->allDeclarations(m_position, m_duContext->topContext()); foreach ( const DeclarationDepthPair& d, declarations ) { if ( d.first && d.first->context()->type() == DUContext::Class ) { declarations.removeAll(d); } } resultingItems.append(declarationListToItemList(declarations)); } m_searchingForModule.clear(); m_searchImportItemsInModule.clear(); return resultingItems; } QList PythonCodeCompletionContext::getMissingIncludeItems(QString forString) { QList items; // Find all the non-empty name components (mainly, remove the last empty one for "sys." or similar) QStringList components = forString.split('.'); components.removeAll(QString()); // Check all components are alphanumeric QRegExp alnum("\\w*"); foreach ( const QString& component, components ) { if ( ! alnum.exactMatch(component) ) return items; } if ( components.isEmpty() ) { return items; } Declaration* existing = Helper::declarationForName(QualifiedIdentifier(components.first()), RangeInRevision(m_position, m_position), DUChainPointer(m_duContext.data())); if ( existing ) { // There's already a declaration for the first component; no need to suggest it return items; } // See if there's a module called like that. auto found = ContextBuilder::findModulePath(components.join("."), m_workingOnDocument); // Check if anything was found if ( found.first.isValid() ) { // Add items for the "from" and the plain import if ( components.size() > 1 && found.second.isEmpty() ) { // There's something left for X in "from foo import X", // and it's not a declaration inside the module so offer that const QString module = QStringList(components.mid(0, components.size() - 1)).join("."); const QString text = QString("from %1 import %2").arg(module, components.last()); MissingIncludeItem* item = new MissingIncludeItem(text, components.last(), forString); items << CompletionTreeItemPointer(item); } const QString module = QStringList(components.mid(0, components.size() - found.second.size())).join("."); const QString text = QString("import %1").arg(module); MissingIncludeItem* item = new MissingIncludeItem(text, components.last()); items << CompletionTreeItemPointer(item); } return items; } QList PythonCodeCompletionContext::declarationListToItemList(QList declarations, int maxDepth) { QList items; DeclarationPointer currentDeclaration; Declaration* checkDeclaration = 0; int count = declarations.length(); for ( int i = 0; i < count; i++ ) { if ( maxDepth && maxDepth > declarations.at(i).second ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Skipped completion item because of its depth"; continue; } currentDeclaration = DeclarationPointer(declarations.at(i).first); PythonDeclarationCompletionItem* item = 0; checkDeclaration = Helper::resolveAliasDeclaration(currentDeclaration.data()); if ( ! checkDeclaration ) { continue; } if ( checkDeclaration->isFunctionDeclaration() || (checkDeclaration->internalContext() && checkDeclaration->internalContext()->type() == DUContext::Class) ) { item = new FunctionDeclarationCompletionItem(currentDeclaration, KDevelop::CodeCompletionContext::Ptr(this)); } else { item = new PythonDeclarationCompletionItem(currentDeclaration, KDevelop::CodeCompletionContext::Ptr(this)); } if ( ! m_matchAgainst.isEmpty() ) { item->addMatchQuality(identifierMatchQuality(m_matchAgainst, checkDeclaration->identifier().toString())); } items << CompletionTreeItemPointer(item); } return items; } QList< CompletionTreeItemPointer > PythonCodeCompletionContext::declarationListToItemList(QList< Declaration* > declarations) { QList fakeItems; foreach ( Declaration* d, declarations ) { fakeItems << DeclarationDepthPair(d, 0); } return declarationListToItemList(fakeItems); } QList< CompletionTreeItemPointer > PythonCodeCompletionContext::getCompletionItemsForType(AbstractType::Ptr type) { type = Helper::resolveAliasType(type); if ( type->whichType() != AbstractType::TypeUnsure ) { return getCompletionItemsForOneType(type); } QList result; UnsureType::Ptr unsure = type.cast(); int count = unsure->typesSize(); for ( int i = 0; i < count; i++ ) { result.append(getCompletionItemsForOneType(unsure->types()[i].abstractType())); } // Do some weighting: the more often an entry appears, the better the entry. // That way, entries which are in all of the types this object could have will // be sorted higher up. QStringList itemTitles; QList remove; for ( int i = 0; i < result.size(); i++ ) { DeclarationPointer decl = result.at(i)->declaration(); if ( ! decl ) { itemTitles.append(QString()); continue; } const QString& title = decl->identifier().toString(); if ( itemTitles.contains(title) ) { // there's already an item with that title, increase match quality int item = itemTitles.indexOf(title); PythonDeclarationCompletionItem* declItem = dynamic_cast(result[item].data()); if ( ! m_fullCompletion ) { remove.append(result.at(i)); } if ( declItem ) { // Add 1 to the match quality of the first item in the list. declItem->addMatchQuality(1); } } itemTitles.append(title); } foreach ( const CompletionTreeItemPointer& ptr, remove ) { result.removeOne(ptr); } return result; } QList PythonCodeCompletionContext::getCompletionItemsForOneType(AbstractType::Ptr type) { type = Helper::resolveAliasType(type); ReferencedTopDUContext builtinTopContext = Helper::getDocumentationFileContext(); if ( type->whichType() != AbstractType::TypeStructure ) { return ItemList(); } // find properties of class declaration TypePtr cls = StructureType::Ptr::dynamicCast(type); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Finding completion items for class type"; if ( ! cls || ! cls->internalContext(m_duContext->topContext()) ) { qCWarning(KDEV_PYTHON_CODECOMPLETION) << "No class type available, no completion offered"; return QList(); } // the PublicOnly will filter out non-explictly defined __get__ etc. functions inherited from object auto searchContexts = Helper::internalContextsForClass(cls, m_duContext->topContext(), Helper::PublicOnly); QList keepDeclarations; foreach ( const DUContext* currentlySearchedContext, searchContexts ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "searching context " << currentlySearchedContext->scopeIdentifier() << "for autocompletion items"; QList declarations = currentlySearchedContext->allDeclarations(CursorInRevision::invalid(), m_duContext->topContext(), false); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "found" << declarations.length() << "declarations"; // filter out those which are builtin functions, and those which were imported; we don't want those here // also, discard all magic functions from autocompletion // TODO rework this, it's maybe not the most elegant solution possible // TODO rework the magic functions thing, I want them sorted at the end of the list but KTE doesn't seem to allow that foreach ( const DeclarationDepthPair& current, declarations ) { if ( current.first->context() != builtinTopContext && ! current.first->identifier().identifier().str().startsWith("__") ) { keepDeclarations.append(current); } else { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Discarding declaration " << current.first->toString(); } } } return declarationListToItemList(keepDeclarations); } QList PythonCodeCompletionContext::findIncludeItems(IncludeSearchTarget item) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "TARGET:" << item.directory.path() << item.remainingIdentifiers; QDir currentDirectory(item.directory.path()); QFileInfoList contents = currentDirectory.entryInfoList(QStringList(), QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot); bool atBottom = item.remainingIdentifiers.isEmpty(); QList items; QString sourceFile; if ( item.remainingIdentifiers.isEmpty() ) { // check for the __init__ file QFileInfo initFile(item.directory.path(), "__init__.py"); if ( initFile.exists() ) { IncludeItem init; init.basePath = item.directory; init.isDirectory = true; init.name = ""; if ( ! item.directory.fileName().contains('-') ) { // Do not include items which contain "-", those are not valid // modules but instead often e.g. .egg directories ImportFileItem* importfile = new ImportFileItem(init); importfile->moduleName = item.directory.fileName(); items << CompletionTreeItemPointer(importfile); sourceFile = initFile.filePath(); } } } else { QFileInfo file(item.directory.path(), item.remainingIdentifiers.first() + ".py"); item.remainingIdentifiers.removeFirst(); qCDebug(KDEV_PYTHON_CODECOMPLETION) << " CHECK:" << file.absoluteFilePath(); if ( file.exists() ) { sourceFile = file.absoluteFilePath(); } } if ( ! sourceFile.isEmpty() ) { IndexedString filename(sourceFile); TopDUContext* top = DUChain::self()->chainForDocument(filename); qCDebug(KDEV_PYTHON_CODECOMPLETION) << top; DUContext* c = internalContextForDeclaration(top, item.remainingIdentifiers); qCDebug(KDEV_PYTHON_CODECOMPLETION) << " GOT:" << c; if ( c ) { // tell function declaration items not to add brackets items << setOmitParentheses(declarationListToItemList(c->localDeclarations().toList())); } else { // do better next time DUChain::self()->updateContextForUrl(filename, TopDUContext::AllDeclarationsAndContexts); } } if ( atBottom ) { // append all python files in the directory foreach ( QFileInfo file, contents ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << " > CONTENT:" << file.absolutePath() << file.fileName(); if ( file.isFile() ) { if ( file.fileName().endsWith(".py") || file.fileName().endsWith(".so") ) { IncludeItem fileInclude; fileInclude.basePath = item.directory; fileInclude.isDirectory = false; fileInclude.name = file.fileName().mid(0, file.fileName().length() - 3); // remove ".py" ImportFileItem* import = new ImportFileItem(fileInclude); import->moduleName = fileInclude.name; items << CompletionTreeItemPointer(import); } } else if ( ! file.fileName().contains('-') ) { IncludeItem dirInclude; dirInclude.basePath = item.directory; dirInclude.isDirectory = true; dirInclude.name = file.fileName(); ImportFileItem* import = new ImportFileItem(dirInclude); import->moduleName = dirInclude.name; items << CompletionTreeItemPointer(import); } } } return items; } QList PythonCodeCompletionContext::findIncludeItems(QList< Python::IncludeSearchTarget > items) { QList results; foreach ( const IncludeSearchTarget& item, items ) { results << findIncludeItems(item); } return results; } DUContext* PythonCodeCompletionContext::internalContextForDeclaration(TopDUContext* topContext, QStringList remainingIdentifiers) { Declaration* d = 0; DUContext* c = topContext; if ( ! topContext ) { return 0; } if ( remainingIdentifiers.isEmpty() ) { return topContext; } do { QList< Declaration* > decls = c->findDeclarations(QualifiedIdentifier(remainingIdentifiers.first())); remainingIdentifiers.removeFirst(); if ( decls.isEmpty() ) { return 0; } d = decls.first(); if ( (c = d->internalContext()) ) { if ( remainingIdentifiers.isEmpty() ) { return c; } } else return 0; } while ( d && ! remainingIdentifiers.isEmpty() ); return 0; } QList PythonCodeCompletionContext::includeItemsForSubmodule(QString submodule) { auto searchPaths = Helper::getSearchPaths(m_workingOnDocument); QStringList subdirs; if ( ! submodule.isEmpty() ) { subdirs = submodule.split("."); } Q_ASSERT(! subdirs.contains("")); QList foundPaths; // this is a bit tricky. We need to find every path formed like /.../foo/bar for // a query string ("submodule" variable) like foo.bar // we also need paths like /foo.py, because then bar is probably a module in that file. // Thus, we first generate a list of possible paths, then match them against those which actually exist // and then gather all the items in those paths. foreach ( QUrl currentPath, searchPaths ) { auto d = QDir(currentPath.path()); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Searching: " << currentPath << subdirs; int identifiersUsed = 0; foreach ( const QString& subdir, subdirs ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "changing into subdir" << subdir; if ( ! d.cd(subdir) ) { break; } qCDebug(KDEV_PYTHON_CODECOMPLETION) << d.absolutePath() << d.exists(); identifiersUsed++; } QStringList remainingIdentifiers = subdirs.mid(identifiersUsed, -1); foundPaths.append(IncludeSearchTarget(QUrl::fromLocalFile(d.absolutePath()), remainingIdentifiers)); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Found path:" << d.absolutePath() << remainingIdentifiers << subdirs; } return findIncludeItems(foundPaths); } PythonCodeCompletionContext::PythonCodeCompletionContext(DUContextPointer context, const QString& remainingText, QString calledFunction, int depth, int alreadyGivenParameters, CodeCompletionContext* child) : CodeCompletionContext(context, remainingText, CursorInRevision::invalid(), depth) , m_operation(FunctionCallCompletion) , m_itemTypeHint(NoHint) , m_child(child) , m_guessTypeOfExpression(calledFunction) , m_alreadyGivenParametersCount(alreadyGivenParameters) , m_fullCompletion(false) { ExpressionParser p(remainingText); summonParentForEventualCall(p.popAll(), remainingText); } void PythonCodeCompletionContext::summonParentForEventualCall(TokenList allExpressions, const QString& text) { DUChainReadLocker lock; int offset = 0; while ( true ) { QPair nextCall = allExpressions.nextIndexOfStatus(ExpressionParser::EventualCallFound, offset); qCDebug(KDEV_PYTHON_CODECOMPLETION) << "next call:" << nextCall; qCDebug(KDEV_PYTHON_CODECOMPLETION) << allExpressions.toString(); if ( nextCall.first == -1 ) { // no more eventual calls break; } offset = nextCall.first; allExpressions.reset(offset); TokenListEntry eventualFunction = allExpressions.weakPop(); qCDebug(KDEV_PYTHON_CODECOMPLETION) << eventualFunction.expression << eventualFunction.status; // it's only a call if a "(" bracket is followed (<- direction) by an expression. if ( eventualFunction.status != ExpressionParser::ExpressionFound ) { continue; // not a call, try the next opening "(" bracket } qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Call found! Creating parent-context."; // determine the amount of "free" commas in between allExpressions.reset(); int atParameter = 0; for ( int i = 0; i < offset-1; i++ ) { TokenListEntry entry = allExpressions.weakPop(); if ( entry.status == ExpressionParser::CommaFound ) { atParameter += 1; } // clear the param count for something like "f([1, 2, 3" (it will be cleared when the "[" is read) if ( entry.status == ExpressionParser::InitializerFound || entry.status == ExpressionParser::EventualCallFound ) { atParameter = 0; } } m_parentContext = new PythonCodeCompletionContext(m_duContext, text.mid(0, eventualFunction.charOffset), eventualFunction.expression, depth() + 1, atParameter, this ); break; } allExpressions.reset(1); } // decide what kind of completion will be offered based on the code before the current cursor position PythonCodeCompletionContext::PythonCodeCompletionContext(DUContextPointer context, const QString& text, const QString& followingText, const KDevelop::CursorInRevision& position, int depth, const PythonCodeCompletionWorker* /*parent*/) : CodeCompletionContext(context, text, position, depth) , m_operation(PythonCodeCompletionContext::DefaultCompletion) , m_itemTypeHint(NoHint) , m_child(0) , m_followingText(followingText) , m_position(position) { m_workingOnDocument = context->topContext()->url().toUrl(); QString textWithoutStrings = CodeHelpers::killStrings(text); qCDebug(KDEV_PYTHON_CODECOMPLETION) << text << position << context->localScopeIdentifier().toString() << context->range(); QPair beforeAndAfterCursor = CodeHelpers::splitCodeByCursor(text, context->range().castToSimpleRange(), position.castToSimpleCursor()); // check if the current position is inside a multi-line comment -> no completion if this is the case CodeHelpers::EndLocation location = CodeHelpers::endsInside(beforeAndAfterCursor.first); if ( location == CodeHelpers::Comment ) { m_operation = PythonCodeCompletionContext::NoCompletion; return; } else if ( location == CodeHelpers::String ) { m_operation = PythonCodeCompletionContext::StringFormattingCompletion; return; } // The expression parser used to determine the type of completion required. ExpressionParser parser(textWithoutStrings); TokenList allExpressions = parser.popAll(); allExpressions.reset(1); ExpressionParser::Status firstStatus = allExpressions.last().status; // TODO reuse already calculated information FileIndentInformation indents(text); DUContext* currentlyChecked = context.data(); // This will set the line to use for the completion to the beginning of the expression. // In reality, the line we're in might mismatch the beginning of the current expression, // for example in multi-line list initializers. int currentlyCheckedLine = position.line - text.mid(text.length() - allExpressions.first().charOffset).count('\n'); // The following code will check whether the DUContext directly at the cursor should be used, or a previous one. // The latter might be the case if there's code like this: /* class foo(): * ..pass * . * ..# completion requested here; note that there's less indent in the previous line! * */ // In that case, the DUContext of "foo" would end at line 2, but should still be used for completion. { DUChainReadLocker lock(DUChain::lock()); while ( currentlyChecked == context.data() && currentlyCheckedLine >= 0 ) { currentlyChecked = context->topContext()->findContextAt(CursorInRevision(currentlyCheckedLine, 0), true); currentlyCheckedLine -= 1; } while ( currentlyChecked && context->parentContextOf(currentlyChecked) ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "checking:" << currentlyChecked->range() << currentlyChecked->type(); // FIXME: "<=" is not really good, it must be exactly one indent-level less int offset = position.line-currentlyChecked->range().start.line; // If the check leaves the current context, abort. if ( offset >= indents.linesCount() ) { break; } if ( indents.indentForLine(indents.linesCount()-1-offset) <= indents.indentForLine(indents.linesCount()-1) ) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "changing context to" << currentlyChecked->range() << ( currentlyChecked->type() == DUContext::Class ); context = currentlyChecked; break; } currentlyChecked = currentlyChecked->parentContext(); } } m_duContext = context; QList defKeywords; defKeywords << ExpressionParser::DefFound << ExpressionParser::ClassFound; if ( allExpressions.nextIndexOfStatus(ExpressionParser::EventualCallFound).first != -1 ) { // 3 is always the case for "def foo(" or class foo(": one names the function, the other is the keyword if ( allExpressions.length() >= 4 && allExpressions.at(1).status == ExpressionParser::DefFound ) { // The next thing the user probably wants to type are parameters for his function. // We cannot offer completion for this. m_operation = NoCompletion; return; } if ( allExpressions.length() >= 4 ) { // TODO: optimally, filter out classes we already inherit from. That's a bonus, tough. if ( allExpressions.at(1).status == ExpressionParser::ClassFound ) { // show only items of type "class" for completion if ( allExpressions.at(allExpressions.length()-1).status == ExpressionParser::MemberAccessFound ) { m_guessTypeOfExpression = allExpressions.at(allExpressions.length()-2).expression; } m_operation = InheritanceCompletion; return; } } } // For something like "func1(3, 5, func2(7, ", we want to show all calltips recursively summonParentForEventualCall(allExpressions, textWithoutStrings); if ( firstStatus == ExpressionParser::MeaninglessKeywordFound ) { m_operation = DefaultCompletion; return; } if ( firstStatus == ExpressionParser::InFound ) { m_operation = DefaultCompletion; m_itemTypeHint = IterableRequested; return; } if ( firstStatus == ExpressionParser::NoCompletionKeywordFound ) { m_operation = NoCompletion; return; } if ( firstStatus == ExpressionParser::RaiseFound || firstStatus == ExpressionParser::ExceptFound ) { m_operation = RaiseExceptionCompletion; if ( firstStatus == ExpressionParser::ExceptFound ) { m_itemTypeHint = ClassTypeRequested; } return; } if ( firstStatus == ExpressionParser::NothingFound ) { m_operation = NewStatementCompletion; return; } if ( firstStatus == ExpressionParser::DefFound ) { if ( context->type() == DUContext::Class ) { m_indent = QString(" ").repeated(indents.indentForLine(indents.linesCount()-1)); m_operation = DefineCompletion; } else { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "def outside class context"; m_operation = NoCompletion; } return; } // The "def in class context" case is handled above already if ( defKeywords.contains(firstStatus) ) { m_operation = NoCompletion; return; } if ( firstStatus == ExpressionParser::ForFound ) { int offset = allExpressions.length() - 2; // one for the "for", and one for the general off-by-one thing QPair nextInitializer = allExpressions.nextIndexOfStatus(ExpressionParser::InitializerFound); if ( nextInitializer.first == -1 ) { // no opening bracket, so no generator completion. m_operation = NoCompletion; return; } // check that all statements in between are part of a generator initializer list bool ok = true; QStringList exprs; int currentOffset = 0; while ( ok && offset > currentOffset ) { ok = allExpressions.at(offset).status == ExpressionParser::ExpressionFound; if ( ! ok ) break; exprs.prepend(allExpressions.at(offset).expression); offset -= 1; ok = allExpressions.at(offset).status == ExpressionParser::CommaFound; // the last expression must *not* have a comma currentOffset = allExpressions.length() - nextInitializer.first; if ( ! ok && currentOffset == offset ) { ok = true; } offset -= 1; } if ( ok ) { m_guessTypeOfExpression = exprs.join(","); m_operation = GeneratorVariableCompletion; return; } else { m_operation = NoCompletion; return; } } QPair import = allExpressions.nextIndexOfStatus(ExpressionParser::ImportFound); QPair from = allExpressions.nextIndexOfStatus(ExpressionParser::FromFound); int importIndex = import.first; int fromIndex = from.first; if ( importIndex != -1 && fromIndex != -1 ) { // it's either "import ... from" or "from ... import" // we treat both in exactly the same way. This is not quite correct, as python // forbids some combinations. TODO fix this. // There's two relevant pieces of text, the one between the "from...import" (or "import...from"), // and the one behind the "import" or the "from" (at the end of the line). // Both need to be extracted and passed to the completion computer. QString firstPiece, secondPiece; if ( fromIndex > importIndex ) { // import ... from if ( fromIndex == allExpressions.length() ) { // The "from" is the last item in the chain m_operation = ImportFileCompletion; return; } firstPiece = allExpressions.at(allExpressions.length() - importIndex - 1).expression; } else { firstPiece = allExpressions.at(allExpressions.length() - fromIndex - 1).expression; } // might be "from foo import bar as baz, bang as foobar, ..." if ( allExpressions.length() > 4 && firstStatus == ExpressionParser::MemberAccessFound ) { secondPiece = allExpressions.at(allExpressions.length() - 2).expression; } if ( fromIndex < importIndex ) { // if it's "from ... import", swap the two pieces. qSwap(firstPiece, secondPiece); } if ( firstPiece.isEmpty() ) { m_searchImportItemsInModule = secondPiece; } else if ( secondPiece.isEmpty() ) { m_searchImportItemsInModule = firstPiece; } else { m_searchImportItemsInModule = firstPiece + "." + secondPiece; } qCDebug(KDEV_PYTHON_CODECOMPLETION) << firstPiece << secondPiece; qCDebug(KDEV_PYTHON_CODECOMPLETION) << "Got submodule to search:" << m_searchImportItemsInModule << "from text" << textWithoutStrings; m_operation = ImportSubCompletion; return; } if ( firstStatus == ExpressionParser::FromFound || firstStatus == ExpressionParser::ImportFound ) { // it's either "from ..." or "import ...", which both come down to the same completion offered m_operation = ImportFileCompletion; return; } if ( fromIndex != -1 || importIndex != -1 ) { if ( firstStatus == ExpressionParser::CommaFound ) { // "import foo.bar, " m_operation = ImportFileCompletion; return; } if ( firstStatus == ExpressionParser::MemberAccessFound ) { m_operation = ImportSubCompletion; m_searchImportItemsInModule = allExpressions.at(allExpressions.length() - 2).expression; return; } // "from foo ..." m_operation = NoCompletion; return; } if ( firstStatus == ExpressionParser::MemberAccessFound ) { TokenListEntry item = allExpressions.weakPop(); if ( item.status == ExpressionParser::ExpressionFound ) { m_guessTypeOfExpression = item.expression; m_operation = MemberAccessCompletion; } else { m_operation = NoCompletion; } return; } if ( firstStatus == ExpressionParser::EqualsFound && allExpressions.length() >= 2 ) { m_operation = DefaultCompletion; m_matchAgainst = allExpressions.at(allExpressions.length() - 2).expression; } } } diff --git a/codecompletion/items/functiondeclaration.cpp b/codecompletion/items/functiondeclaration.cpp index b807a2fb..1301735c 100644 --- a/codecompletion/items/functiondeclaration.cpp +++ b/codecompletion/items/functiondeclaration.cpp @@ -1,185 +1,185 @@ /***************************************************************************** * Copyright (c) 2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "functiondeclaration.h" #include #include #include #include #include #include #include #include #include #include "duchain/navigation/navigationwidget.h" #include "codecompletion/helpers.h" #include "declaration.h" #include "declarations/functiondeclaration.h" #include "duchain/helpers.h" #include #include "../codecompletiondebug.h" using namespace KDevelop; using namespace KTextEditor; namespace Python { FunctionDeclarationCompletionItem::FunctionDeclarationCompletionItem(DeclarationPointer decl, CodeCompletionContext::Ptr context) : PythonDeclarationCompletionItem(decl, context) , m_atArgument(-1) , m_depth(0) , m_doNotCall(false) { } int FunctionDeclarationCompletionItem::atArgument() const { return m_atArgument; } void FunctionDeclarationCompletionItem::setDepth(int d) { m_depth = d; } void FunctionDeclarationCompletionItem::setAtArgument(int d) { m_atArgument = d; } int FunctionDeclarationCompletionItem::argumentHintDepth() const { return m_depth; } QVariant FunctionDeclarationCompletionItem::data(const QModelIndex& index, int role, const KDevelop::CodeCompletionModel* model) const { DUChainReadLocker lock; FunctionDeclaration* dec = dynamic_cast(m_declaration.data()); switch ( role ) { case Qt::DisplayRole: { if ( ! dec ) { break; // use the default } if ( index.column() == KDevelop::CodeCompletionModel::Arguments ) { if (FunctionType::Ptr functionType = dec->type()) { QString ret; createArgumentList(dec, ret, 0, 0, false); return ret; } } if ( index.column() == KDevelop::CodeCompletionModel::Prefix ) { FunctionType::Ptr type = dec->type(); if ( type && type->returnType() ) { return i18n("function") + " -> " + type->returnType()->toString(); } } break; } case KDevelop::CodeCompletionModel::HighlightingMethod: { if ( index.column() == KDevelop::CodeCompletionModel::Arguments ) return QVariant(KDevelop::CodeCompletionModel::CustomHighlighting); break; } case KDevelop::CodeCompletionModel::CustomHighlight: { if ( index.column() == KDevelop::CodeCompletionModel::Arguments ) { if ( ! dec ) return QVariant(); QString ret; QList highlight; if ( atArgument() ) { createArgumentList(dec, ret, &highlight, atArgument(), false); } else { createArgumentList(dec, ret, 0, false); } return QVariant(highlight); } } case KDevelop::CodeCompletionModel::MatchQuality: { if ( m_typeHint == PythonCodeCompletionContext::IterableRequested && dec && dec->type() && dynamic_cast(dec->type()->returnType().data()) ) { return 2 + PythonDeclarationCompletionItem::data(index, role, model).toInt(); } return PythonDeclarationCompletionItem::data(index, role, model); } } return Python::PythonDeclarationCompletionItem::data(index, role, model); } void FunctionDeclarationCompletionItem::setDoNotCall(bool doNotCall) { m_doNotCall = doNotCall; } void FunctionDeclarationCompletionItem::executed(KTextEditor::View* view, const KTextEditor::Range& word) { qCDebug(KDEV_PYTHON_CODECOMPLETION) << "FunctionDeclarationCompletionItem executed"; KTextEditor::Document* document = view->document(); - DeclarationPointer resolvedDecl(Helper::resolveAliasDeclaration(declaration().data())); + auto resolvedDecl = Helper::resolveAliasDeclaration(declaration().data()); DUChainReadLocker lock; - QPair fdecl = Helper::functionDeclarationForCalledDeclaration(resolvedDecl); + auto functionDecl = Helper::functionForCalled(resolvedDecl).declaration; lock.unlock(); - if ( ! fdecl.first && (! resolvedDecl || ! resolvedDecl->abstractType() + if ( ! functionDecl && (! resolvedDecl || ! resolvedDecl->abstractType() || resolvedDecl->abstractType()->whichType() != AbstractType::TypeStructure) ) { qCritical(KDEV_PYTHON_CODECOMPLETION) << "ERROR: could not get declaration data, not executing completion item!"; return; } QString suffix = "()"; KTextEditor::Range checkPrefix(word.start().line(), 0, word.start().line(), word.start().column()); KTextEditor::Range checkSuffix(word.end().line(), word.end().column(), word.end().line(), document->lineLength(word.end().line())); if ( m_doNotCall || document->text(checkSuffix).trimmed().startsWith('(') || document->text(checkPrefix).trimmed().endsWith('@') - || (fdecl.first && Helper::findDecoratorByName(fdecl.first.data(), QLatin1String("property"))) ) + || (functionDecl && Helper::findDecoratorByName(functionDecl, QLatin1String("property"))) ) { // don't insert brackets if they're already there, // the item is a decorator, or if it's an import item. suffix.clear(); } // place cursor behind bracktes by default int skip = 2; - if ( fdecl.first ) { + if ( functionDecl ) { bool needsArguments = false; - int argumentCount = fdecl.first->type()->arguments().length(); - if ( fdecl.first->context()->type() == KDevelop::DUContext::Class ) { + int argumentCount = functionDecl->type()->arguments().length(); + if ( functionDecl->context()->type() == KDevelop::DUContext::Class ) { // it's a member function, so it has the implicit self // TODO static methods needsArguments = argumentCount > 1; } else { // it's a free function needsArguments = argumentCount > 0; } if ( needsArguments ) { // place cursor in brackets if there's parameters skip = 1; } } document->replaceText(word, declaration()->identifier().toString() + suffix); view->setCursorPosition( Cursor(word.end().line(), word.end().column() + skip) ); } FunctionDeclarationCompletionItem::~FunctionDeclarationCompletionItem() { } } diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index eb51fcb2..681c0889 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1902 +1,1901 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/decorator.h" #include "duchain/declarations/functiondeclaration.h" #include "duchain/declarations/classdeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); qCDebug(KDEV_PYTHON_DUCHAIN) << "Building Declarations"; } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "building, but running pre-builder first"; DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); qCDebug(KDEV_PYTHON_DUCHAIN) << "pre-builder finished"; delete prebuilder; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "prebuilding"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, Ast* range, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = 0; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(range, range), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name, range); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. QList declaringContexts; declaringContexts << ExpressionAst::Store << ExpressionAst::Parameter << ExpressionAst::AugStore; if ( ! declaringContexts.contains(currentVariableDefinition->context) ) { return 0; } Identifier* id = currentVariableDefinition->identifier; return visitVariableDeclaration(id, currentVariableDefinition, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), nullptr, previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(0); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { QList existingDeclarations = currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); // append arguments context if ( m_mostRecentArgumentsContext ) { QList args = m_mostRecentArgumentsContext->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent ); existingDeclarations.append(args); } return existingDeclarations; } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = 0; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d && d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != 0 ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Ast* originalAst, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; Ast* rangeNode = originalAst ? originalAst : node; RangeInRevision range = editorFindRange(rangeNode, rangeNode); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = 0; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << hints->toString(); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, 0, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); visitVariableDeclaration(node->optionalVars, 0, v.lastType()); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = 0; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), 0, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return 0; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = 0; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(0); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(0); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void spoofNodePosition(Ast* node, int column) { // Ridiculous hack, see next comment. node->startCol = node-> endCol = column; if (node->astType == Ast::TupleAstType) { // Recursion to bodge all declarations, e.g. // [x + y * z for x, (y, z) in foo] foreach(auto elem, static_cast(node)->elements) { spoofNodePosition(elem, column); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); // make the declaration zero chars long; it must appear at the beginning of the context, // because it is actually used *before* its real declaration: [foo for foo in bar] // The DUChain doesn't like this, so for now, the declaration is at the opening bracket, // and both other occurrences are uses of that declaration. // TODO add a special case to the usebuilder to display the second occurrence as a declaration spoofNodePosition(node->target, currentContext()->range().start.column - 1); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(0); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = 0; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = 0; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = 0; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, &temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return 0; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = 0; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return 0; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return 0; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, 0, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, 0); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(0), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { Python::AstDefaultVisitor::visitLambda(node); DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { const int offset = ! args.isEmpty() ? args.at(0).toInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); DUChainWriteLocker wlock; if ( ! argVisitor.lastType() ) { return; } auto sources = Helper::filterType( argVisitor.lastType(), [](AbstractType::Ptr type) { return type.cast(); } ); for ( auto sourceContainer : sources ) { if ( ! sourceContainer->contentType() ) { continue; } container->addContentType(sourceContainer->contentType().abstractType()); v.lastDeclaration()->setType(container); } }; foreach ( const QString& key, items.keys() ) { if ( Helper::docstringContainsHint(function.data(), key, &args) ) { items[key](); } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer function) { DUChainReadLocker lock; - QPair called = Helper::functionDeclarationForCalledDeclaration(function); - FunctionDeclaration::Ptr lastFunctionDeclaration = called.first; - bool isConstructor = called.second; + auto funcInfo = Helper::functionForCalled(function.data()); + auto lastFunctionDeclaration = funcInfo.declaration; if ( ! lastFunctionDeclaration ) { return; } if ( lastFunctionDeclaration->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } - DUContext* args = DUChainUtils::getArgumentContext(lastFunctionDeclaration.data()); + DUContext* args = DUChainUtils::getArgumentContext(lastFunctionDeclaration); FunctionType::Ptr functiontype = lastFunctionDeclaration->type(); if ( ! args || ! functiontype ) { return; } // The declaration which was found is a function declaration, and has a valid arguments list assigned. QVector parameters = args->localDeclarations(); const int specialParamsCount = (lastFunctionDeclaration->vararg() > 0) + (lastFunctionDeclaration->kwarg() > 0); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfArgument = false; - if ( ( lastFunctionDeclaration->context()->type() == DUContext::Class || isConstructor ) + if ( ( lastFunctionDeclaration->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! parameters.isEmpty() && ! lastFunctionDeclaration->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as arguments // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfArgument = true; } } int currentParamIndex = hasSelfArgument; int currentArgumentIndex = 0; int indexInVararg = -1; int paramsAvailable = qMin(functiontype->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; lock.unlock(); // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentParamIndex++, currentArgumentIndex++ ) { if ( ! atVararg && currentArgumentIndex == lastFunctionDeclaration->vararg() ) { atVararg = true; } qCDebug(KDEV_PYTHON_DUCHAIN) << currentParamIndex << currentArgumentIndex << atVararg << lastFunctionDeclaration->vararg(); ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(lastFunctionDeclaration->vararg()+hasSelfArgument); IndexedContainer::Ptr varargContainer = parameter->type(); qCDebug(KDEV_PYTHON_DUCHAIN) << "adding" << addType->toString() << "at position" << indexInVararg; if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functiontype->removeArgument(currentArgumentIndex + hasSelfArgument); functiontype->addArgument(newType, currentArgumentIndex + hasSelfArgument); lastFunctionDeclaration->setAbstractType(functiontype.cast()); parameters.at(currentParamIndex)->setType(newType); } } lock.unlock(); DUChainWriteLocker wlock; if ( lastFunctionDeclaration->kwarg() < 0 || parameters.isEmpty() ) { // no kwarg, stop here. return; } foreach ( KeywordAst* keyword, node->keywords ) { AbstractType::Ptr param = parameters.last()->abstractType(); auto list = param.cast(); if ( ! list ) { continue; } wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); list->addContentType(addType.cast()); parameters.last()->setAbstractType(list.cast()); } } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; Python::Identifier* identifier = target->identifier; AliasDeclaration* decl = eventuallyReopenDeclaration(identifier, target, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, 0, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(0); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreayOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } else qCWarning(KDEV_PYTHON_DUCHAIN) << "No declaration created for " << attrib->attribute << "as parent is not a class"; closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 or (starred == -1 and spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, NoTypeRequired); lock.unlock(); visitDecorators(node->decorators, dec); lock.lock(); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); dec->setComment(getDocstring(node->body)); // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(dec, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(dec, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(dec, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassType = type; // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); foreach ( Ast* _node, node->body ) { AstDefaultVisitor::visitNode(_node); } lock.lock(); closeContext(); closeType(); closeDeclaration(); } template void DeclarationBuilder::visitDecorators(QList< Python::ExpressionAst* > decorators, T* addTo) { foreach ( ExpressionAst* decorator, decorators ) { AstDefaultVisitor::visitNode(decorator); if ( decorator->astType == Ast::CallAstType ) { CallAst* call = static_cast(decorator); Decorator d; if ( call->function->astType != Ast::NameAstType ) { continue; } d.setName(*static_cast(call->function)->identifier); foreach ( ExpressionAst* arg, call->arguments ) { if ( arg->astType == Ast::NumberAstType ) { d.setAdditionalInformation(QString::number(static_cast(arg)->value)); } else if ( arg->astType == Ast::StringAstType ) { d.setAdditionalInformation(static_cast(arg)->value); } break; // we only need the first argument for documentation analysis } addTo->addDecorator(d); } else if ( decorator->astType == Ast::NameAstType ) { NameAst* name = static_cast(decorator); Decorator d; d.setName(*(name->identifier)); addTo->addDecorator(d); } } } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); visitDecorators(node->decorators, dec); visitFunctionArguments(node); lock.lock(); const bool isStatic = Helper::findDecoratorByName(dec, "staticmethod"); const bool isClassMethod = Helper::findDecoratorByName(dec, "classmethod"); dec->setStatic(isStatic); // If this is a member function, set the type of the first argument (the "self") to be // an instance of the class. // this must be done here, because the type of self must be known when parsing the body if ( eventualParentDeclaration && currentType()->arguments().length() && currentContext()->type() == DUContext::Class && ! isStatic ) { if ( dec->vararg() != -1 ) { dec->setVararg(dec->vararg() - 1); } if ( dec->kwarg() != -1 ) { dec->setKwarg(dec->kwarg() - 1); } } lock.unlock(); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } dec->setType(type); } if ( ! isStatic ) { DUContext* args = DUChainUtils::getArgumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! isClassMethod ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); qCDebug(KDEV_PYTHON_DUCHAIN) << "updated function return type to " << type->toString(); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); if ( node->value ) { if ( ! hasCurrentType() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } else { TypePtr t = currentType(); AbstractType::Ptr encountered = v.lastType(); DUChainWriteLocker lock; if ( t ) { // Update the containing function's return type t->setReturnType(Helper::mergeTypes(t->returnType(), encountered)); } } } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); const bool isClassMethod = Helper::findDecoratorByName(workingOnDeclaration, "classmethod"); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; qCDebug(KDEV_PYTHON_DUCHAIN) << "arguments:" << node->arguments.size(); foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } qCDebug(KDEV_PYTHON_DUCHAIN) << "visiting argument:" << arg->argumentName->value; // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && isClassMethod ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, arg, AliasDeclarationType); if ( m_currentClassType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting declaration:" << m_currentClassType->declaration(topContext())->toString(); decl->setAliasedDeclaration(m_currentClassType->declaration(currentContext()->topContext())); } paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } qCDebug(KDEV_PYTHON_DUCHAIN) << "is first:" << isFirst << hasCurrentDeclaration() << currentDeclaration(); if ( isFirst && hasCurrentDeclaration() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassType.cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); if ( argumentType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "creating argument with type" << argumentType->toString(); } } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, 0, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, 0, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/expressionvisitor.cpp b/duchain/expressionvisitor.cpp index b4821276..9c377edf 100644 --- a/duchain/expressionvisitor.cpp +++ b/duchain/expressionvisitor.cpp @@ -1,706 +1,703 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2010 Miquel Canes Gonzalez * * Copyright 2011-2013 by Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "expressionvisitor.h" #include "types/indexedcontainer.h" #include "declarations/functiondeclaration.h" #include "pythonduchainexport.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include #include using namespace KDevelop; using namespace Python; using namespace KTextEditor; namespace Python { QHash ExpressionVisitor::m_defaultTypes; AbstractType::Ptr ExpressionVisitor::encounterPreprocess(AbstractType::Ptr type) { return Helper::resolveAliasType(type); } ExpressionVisitor::ExpressionVisitor(const DUContext* ctx) : DynamicLanguageExpressionVisitor(ctx) { ENSURE_CHAIN_NOT_LOCKED if ( m_defaultTypes.isEmpty() ) { m_defaultTypes.insert(NameConstantAst::True, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::False, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::None, AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } Q_ASSERT(context()); Q_ASSERT(context()->topContext()); } ExpressionVisitor::ExpressionVisitor(ExpressionVisitor* parent, const DUContext* overrideContext) : DynamicLanguageExpressionVisitor(parent) , m_forceGlobalSearching(parent->m_forceGlobalSearching) , m_reportUnknownNames(parent->m_reportUnknownNames) , m_scanUntilCursor(parent->m_scanUntilCursor) { ENSURE_CHAIN_NOT_LOCKED if ( overrideContext ) { m_context = overrideContext; } Q_ASSERT(context()); } void ExpressionVisitor::encounter(AbstractType::Ptr type, DeclarationPointer declaration, bool alias) { setLastIsAlias(alias); DynamicLanguageExpressionVisitor::encounter(type, declaration); } void ExpressionVisitor::visitAttribute(AttributeAst* node) { ExpressionVisitor v(this); v.visitNode(node->value); setConfident(false); // Find a matching declaration which is made inside the type of the accessed object. // Like, for B.C where B is an instance of foo, find a property of foo called C. DUChainReadLocker lock; auto attribute = Helper::accessAttribute(v.lastType(), node->attribute->value, topContext()); if ( auto resolved = Helper::resolveAliasDeclaration(attribute) ) { encounter(attribute->abstractType(), DeclarationPointer(attribute)); setLastIsAlias(dynamic_cast(attribute) || resolved->isFunctionDeclaration() || dynamic_cast(resolved)); } else { encounterUnknown(); } } void ExpressionVisitor::visitCall(CallAst* node) { foreach ( ExpressionAst* c, node->arguments ) { AstDefaultVisitor::visitNode(c); } ExpressionVisitor v(this); v.visitNode(node->function); Declaration* actualDeclaration = 0; FunctionType::Ptr unidentifiedFunctionType; if ( ! v.m_isAlias && v.lastType() && v.lastType()->whichType() == AbstractType::TypeFunction ) { unidentifiedFunctionType = v.lastType().cast(); } else if ( ! v.m_isAlias && v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { // use __call__ DUChainReadLocker lock; auto c = v.lastType().cast()->internalContext(topContext()); if ( c ) { auto decls = c->findDeclarations(QualifiedIdentifier("__call__")); if ( ! decls.isEmpty() ) { auto decl = dynamic_cast(decls.first()); if ( decl ) { unidentifiedFunctionType = decl->abstractType().cast(); } } } } else { actualDeclaration = v.lastDeclaration().data(); } if ( unidentifiedFunctionType ) { encounter(unidentifiedFunctionType->returnType()); return; } else if ( !actualDeclaration ) { setConfident(false); return encounterUnknown(); } DUChainReadLocker lock; actualDeclaration = Helper::resolveAliasDeclaration(actualDeclaration); ClassDeclaration* classDecl = dynamic_cast(actualDeclaration); - QPair d = Helper::functionDeclarationForCalledDeclaration( - DeclarationPointer(actualDeclaration)); - FunctionDeclaration* funcDecl = d.first.data(); - bool isConstructor = d.second; + auto function = Helper::functionForCalled(actualDeclaration); lock.unlock(); - if ( funcDecl && funcDecl->type() ) { + if ( function.declaration && function.declaration->type() ) { // try to deduce type from a decorator - checkForDecorators(node, funcDecl, classDecl, isConstructor); + checkForDecorators(node, function.declaration, classDecl, function.isConstructor); } else if ( classDecl ) { return encounter(classDecl->abstractType(), DeclarationPointer(classDecl)); } else { if ( actualDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaraton is not a class or function declaration"; } return encounterUnknown(); } } void ExpressionVisitor::checkForDecorators(CallAst* node, FunctionDeclaration* funcDecl, ClassDeclaration* classDecl, bool isConstructor) { AbstractType::Ptr type; Declaration* useDeclaration = nullptr; if ( isConstructor && classDecl ) { type = classDecl->abstractType(); useDeclaration = classDecl; } else { type = funcDecl->type()->returnType(); useDeclaration = funcDecl; } auto listOfTuples = [&](AbstractType::Ptr key, AbstractType::Ptr value) { auto newType = typeObjectForIntegralType("list"); IndexedContainer::Ptr newContents = typeObjectForIntegralType("tuple"); if ( ! newType || ! newContents ) { return AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! key ) { key = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! value ) { value = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } newContents->addEntry(key); newContents->addEntry(value); newType->addContentType(AbstractType::Ptr::staticCast(newContents)); AbstractType::Ptr resultingType = AbstractType::Ptr::staticCast(newType); return resultingType; }; QHash< QString, std::function > knownDecoratorHints; qCDebug(KDEV_PYTHON_DUCHAIN) << "Got function declaration with decorators, checking for list content type..."; knownDecoratorHints["getsType"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found container, using type"; AbstractType::Ptr newType = t->contentType().abstractType(); encounter(newType, DeclarationPointer(useDeclaration)); return true; } return false; }; knownDecoratorHints["getsList"] = [&](QStringList /*arguments*/, QString currentHint) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainWriteLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got container:" << t->toString(); auto newType = typeObjectForIntegralType("list"); if ( ! newType ) { return false; } AbstractType::Ptr contentType; if ( currentHint == "getsList" ) { contentType = t->contentType().abstractType(); } else if ( auto map = MapType::Ptr::dynamicCast(t) ) { contentType = map->keyType().abstractType(); } newType->addContentType(contentType); AbstractType::Ptr resultingType = newType.cast(); encounter(resultingType, DeclarationPointer(useDeclaration)); return true; } return false; }; knownDecoratorHints["getListOfKeys"] = knownDecoratorHints["getsList"]; knownDecoratorHints["enumerate"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::NameAstType || node->arguments.size() < 1 ) { return false; } ExpressionVisitor enumeratedTypeVisitor(this); enumeratedTypeVisitor.visitNode(node->arguments.first()); DUChainWriteLocker lock; auto intType = typeObjectForIntegralType("int"); auto enumerated = enumeratedTypeVisitor.lastType(); auto result = listOfTuples(intType, Helper::contentOfIterable(enumerated, topContext())); encounter(result, DeclarationPointer(useDeclaration)); return true; }; knownDecoratorHints["getsListOfBoth"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got getsListOfBoth decorator, checking container"; if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainWriteLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got container:" << t->toString(); auto resultingType = listOfTuples(t->keyType().abstractType(), t->contentType().abstractType()); encounter(resultingType, DeclarationPointer(useDeclaration)); return true; } return false; }; knownDecoratorHints["returnContentEqualsContentOf"] = [&](QStringList arguments, QString /*currentHint*/) { int argNum = ! arguments.isEmpty() ? arguments.at(0).toInt() : 0; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found argument dependent decorator, checking argument type" << argNum; if ( argNum >= node->arguments.length() ) { return false; } ExpressionAst* relevantArgument = node->arguments.at(argNum); ExpressionVisitor v(this); v.visitNode(relevantArgument); if ( ! v.lastType() ) { return false; } ListType::Ptr realTarget; if ( auto target = ListType::Ptr::dynamicCast(type) ) { realTarget = target; } if ( auto source = ListType::Ptr::dynamicCast(v.lastType()) ) { if ( ! realTarget ) { // if the function does not force a return type, just copy the source (like for reversed()) realTarget = source; } auto newType = ListType::Ptr::staticCast(AbstractType::Ptr(realTarget->clone())); Q_ASSERT(newType); newType->addContentType(source->contentType().abstractType()); encounter(AbstractType::Ptr::staticCast(newType), DeclarationPointer(useDeclaration)); return true; } return false; }; foreach ( const QString& currentHint, knownDecoratorHints.keys() ) { QStringList arguments; if ( ! Helper::docstringContainsHint(funcDecl, currentHint, &arguments) ) { continue; } // If the hint word appears in the docstring, run the evaluation function. if ( knownDecoratorHints[currentHint](arguments, currentHint) ) { // We indeed found something, so we're done. return; } } // if none of the above decorator-finding methods worked, just use the ordinary return type. return encounter(type, DeclarationPointer(useDeclaration)); } void ExpressionVisitor::visitSubscript(SubscriptAst* node) { AstDefaultVisitor::visitNode(node->value); auto valueTypes = Helper::filterType(lastType(), [](AbstractType::Ptr) { return true; }); AbstractType::Ptr result(new IntegralType(IntegralType::TypeMixed)); foreach (const auto& type, valueTypes) { if ( (node->slice && node->slice->astType != Ast::IndexAstType) && (type.cast() || type.cast()) ) { if ( type.cast() ) { continue; // Can't slice dicts. } // Assume that slicing (e.g. foo[3:5]) a tuple/list returns the same type. // TODO: we could do better for some tuple slices. result = Helper::mergeTypes(result, type); } else if ( const auto& indexed = type.cast() ) { IndexAst* sliceIndexAst = static_cast(node->slice); NumberAst* number = nullptr; bool invert = false; if ( sliceIndexAst->value && sliceIndexAst->value->astType == Ast::UnaryOperationAstType ) { // might be -3 UnaryOperationAst* unary = static_cast(sliceIndexAst->value); if ( unary->type == Ast::UnaryOperatorSub && unary->operand->astType == Ast::NumberAstType ) { number = static_cast(unary->operand); invert = true; } } else if ( sliceIndexAst->value->astType == Ast::NumberAstType ) { number = static_cast(sliceIndexAst->value); } if ( number ) { int sliceIndex = number->value * ( invert ? -1 : 1 ); if ( sliceIndex < 0 && sliceIndex + indexed->typesCount() > 0 ) { sliceIndex += indexed->typesCount(); } if ( sliceIndex < indexed->typesCount() && sliceIndex >= 0 ) { result = Helper::mergeTypes(result, indexed->typeAt(sliceIndex).abstractType()); continue; } } result = Helper::mergeTypes(result, indexed->asUnsureType()); } else if ( const auto& listType = type.cast() ) { result = Helper::mergeTypes(result, listType->contentType().abstractType()); } else { // Type wasn't one with custom handling, so use return type of __getitem__(). DUChainReadLocker lock; static const IndexedIdentifier getitemIdentifier(KDevelop::Identifier("__getitem__")); auto function = Helper::accessAttribute(type, getitemIdentifier, topContext()); if ( function && function->isFunctionDeclaration() ) { if ( FunctionType::Ptr functionType = function->type() ) { result = Helper::mergeTypes(result, functionType->returnType()); } } } } encounter(result); } void ExpressionVisitor::visitList(ListAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); type->addContentType(contentVisitor.lastType()); } } else { encounterUnknown(); qCWarning(KDEV_PYTHON_DUCHAIN) << " [ !!! ] did not get a typetrack container object when expecting one! Fix code / setup."; } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol)); lock.unlock(); Q_ASSERT(comprehensionContext); DUContext* ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->value); if ( v.lastType() ) { type->addContentType(v.lastType()); } ExpressionVisitor k(this, ctx); k.visitNode(node->key); if ( k.lastType() ) { type->addKeyType(k.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitSetComprehension(SetComprehensionAst* node) { Python::AstDefaultVisitor::visitSetComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); auto ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitListComprehension(ListComprehensionAst* node) { AstDefaultVisitor::visitListComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); if ( type && ! m_forceGlobalSearching ) { // TODO fixme DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); ExpressionVisitor v(this, comprehensionContext); Q_ASSERT(comprehensionContext); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitTuple(TupleAst* node) { DUChainReadLocker lock; IndexedContainer::Ptr type = typeObjectForIntegralType("tuple"); if ( type ) { lock.unlock(); foreach ( ExpressionAst* expr, node->elements ) { ExpressionVisitor v(this); v.visitNode(expr); if ( v.lastType() ) { type->addEntry(v.lastType()); } else { type->addEntry(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); } } encounter(AbstractType::Ptr::staticCast(type)); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "tuple type object is not available"; return encounterUnknown(); } } void ExpressionVisitor::visitIfExpression(IfExpressionAst* node) { AstDefaultVisitor::visitIfExpression(node); if ( node->body && node->orelse ) { ExpressionVisitor v(this); v.visitNode(node->body); AbstractType::Ptr first = v.lastType(); v.visitNode(node->orelse); AbstractType::Ptr second = v.lastType(); encounter(Helper::mergeTypes(first, second)); } } void ExpressionVisitor::visitSet(SetAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); type->addContentType(contentVisitor.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDict(DictAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); lock.unlock(); ExpressionVisitor contentVisitor(this); ExpressionVisitor keyVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->values ) { contentVisitor.visitNode(content); type->addContentType(contentVisitor.lastType()); } foreach ( ExpressionAst* key, node->keys ) { keyVisitor.visitNode(key); type->addKeyType(keyVisitor.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitNumber(Python::NumberAst* number) { AbstractType::Ptr type; DUChainReadLocker lock; if ( number->isInt ) { type = typeObjectForIntegralType("int"); } else { type = typeObjectForIntegralType("float"); } encounter(type); } void ExpressionVisitor::visitString(Python::StringAst* ) { DUChainReadLocker lock; StructureType::Ptr type = typeObjectForIntegralType("str"); encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitBytes(Python::BytesAst* ) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("bytes"); encounter(AbstractType::Ptr::staticCast(type)); } RangeInRevision nodeRange(Python::Ast* node) { return RangeInRevision(node->startLine, node->startCol, node->endLine,node->endCol); } void ExpressionVisitor::addUnknownName(const QString& name) { if ( m_parentVisitor ) { static_cast(m_parentVisitor)->addUnknownName(name); } else if ( ! m_unknownNames.contains(name) ) { m_unknownNames.insert(name); } } void ExpressionVisitor::visitNameConstant(NameConstantAst* node) { // handles "True", "False", "None" auto defId = m_defaultTypes.constFind(node->value); if ( defId != m_defaultTypes.constEnd() ) { return encounter(*defId); } } void ExpressionVisitor::visitName(Python::NameAst* node) { RangeInRevision range; if ( m_scanUntilCursor.isValid() ) { range = RangeInRevision(CursorInRevision(0, 0), m_scanUntilCursor); } else if ( m_forceGlobalSearching ) { range = RangeInRevision::invalid(); } else { range = RangeInRevision(0, 0, node->endLine, node->endCol); } DUChainReadLocker lock; Declaration* d = Helper::declarationForName(QualifiedIdentifier(node->identifier->value), range, DUChainPointer(context())); if ( d ) { bool isAlias = dynamic_cast(d) || d->isFunctionDeclaration() || dynamic_cast(d); return encounter(d->abstractType(), DeclarationPointer(d), isAlias); } else { if ( m_reportUnknownNames ) { addUnknownName(node->identifier->value); } return encounterUnknown(); } } void ExpressionVisitor::visitCompare(CompareAst* node) { Python::AstDefaultVisitor::visitCompare(node); encounter(AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); } AbstractType::Ptr ExpressionVisitor::fromBinaryOperator(AbstractType::Ptr lhs, AbstractType::Ptr rhs, const QString& op) { DUChainReadLocker lock; auto operatorReturnType = [&op, this](const AbstractType::Ptr& p) { StructureType::Ptr type = p.cast(); if ( ! type ) { return AbstractType::Ptr(); } auto func = Helper::accessAttribute(type, op, topContext()); if ( ! func ) { return AbstractType::Ptr(); } auto operatorFunctionType = func->type(); DUChainReadLocker lock; auto context = Helper::getDocumentationFileContext(); if ( context ) { auto object_decl = context->findDeclarations(QualifiedIdentifier("object")); if ( ! object_decl.isEmpty() && object_decl.first()->internalContext() == func->context() ) { // if the operator is only declared in object(), do not include its type (which is void). return AbstractType::Ptr(); } } return operatorFunctionType ? operatorFunctionType->returnType() : AbstractType::Ptr(); }; return Helper::mergeTypes(operatorReturnType(lhs), operatorReturnType(rhs)); } void ExpressionVisitor::visitBinaryOperation(Python::BinaryOperationAst* node) { ExpressionVisitor lhsVisitor(this); ExpressionVisitor rhsVisitor(this); AbstractType::Ptr result; lhsVisitor.visitNode(node->lhs); rhsVisitor.visitNode(node->rhs); if ( lhsVisitor.lastType() && lhsVisitor.lastType()->whichType() == AbstractType::TypeUnsure ) { KDevelop::UnsureType::Ptr unsure = lhsVisitor.lastType().cast(); const IndexedType* types = unsure->types(); for( uint i = 0; i < unsure->typesSize(); i++ ) { result = Helper::mergeTypes(result, fromBinaryOperator(types[i].abstractType(), rhsVisitor.lastType(), node->methodName())); } } else { result = fromBinaryOperator(lhsVisitor.lastType(), rhsVisitor.lastType(), node->methodName()); } if ( ! Helper::isUsefulType(result) ) { result = Helper::mergeTypes(lhsVisitor.lastType(), rhsVisitor.lastType()); } return encounter(result); } void ExpressionVisitor::visitUnaryOperation(Python::UnaryOperationAst* node) { // Only visit the value, and use that as the result. Unary operators usually // don't change the type of the object (i.e. -a has the same type as a) visitNode(node->operand); } void ExpressionVisitor::visitBooleanOperation(Python::BooleanOperationAst* node) { foreach (ExpressionAst* expression, node->values) { visitNode(expression); } encounter(AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); } } diff --git a/duchain/helpers.cpp b/duchain/helpers.cpp index 7ee30e55..35d13e1a 100644 --- a/duchain/helpers.cpp +++ b/duchain/helpers.cpp @@ -1,534 +1,534 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2011-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "helpers.h" #include #include #include #include #include #include "duchaindebug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ast.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/indexedcontainer.h" #include "kdevpythonversion.h" #include "expressionvisitor.h" using namespace KDevelop; namespace Python { QMap> Helper::cachedCustomIncludes; QMap> Helper::cachedSearchPaths; QVector Helper::projectSearchPaths; QStringList Helper::dataDirs; QString Helper::documentationFile; DUChainPointer Helper::documentationFileContext = DUChainPointer(0); QStringList Helper::correctionFileDirs; QString Helper::localCorrectionFileDir; QMutex Helper::cacheMutex; QMutex Helper::projectPathLock; void Helper::scheduleDependency(const IndexedString& dependency, int betterThanPriority) { BackgroundParser* bgparser = KDevelop::ICore::self()->languageController()->backgroundParser(); bool needsReschedule = true; if ( bgparser->isQueued(dependency) ) { const auto priority= bgparser->priorityForDocument(dependency); if ( priority > betterThanPriority - 1 ) { bgparser->removeDocument(dependency); } else { needsReschedule = false; } } if ( needsReschedule ) { bgparser->addDocument(dependency, TopDUContext::ForceUpdate, betterThanPriority - 1, 0, ParseJob::FullSequentialProcessing); } } IndexedDeclaration Helper::declarationUnderCursor(bool allowUse) { KDevelop::IDocument* doc = ICore::self()->documentController()->activeDocument(); const auto view = static_cast(ICore::self()->partController())->activeView(); if ( doc && doc->textDocument() && view ) { DUChainReadLocker lock; const auto cursor = view->cursorPosition(); if ( allowUse ) { return IndexedDeclaration(DUChainUtils::itemUnderCursor(doc->url(), cursor).declaration); } else { return DUChainUtils::declarationInLine(cursor, DUChainUtils::standardContextForUrl(doc->url())); } } return KDevelop::IndexedDeclaration(); } Declaration* Helper::accessAttribute(const AbstractType::Ptr accessed, const IndexedIdentifier& attribute, const TopDUContext* topContext) { if ( ! accessed ) { return 0; } // if the type is unsure, search all the possibilities (but return the first match) auto structureTypes = Helper::filterType(accessed, [](AbstractType::Ptr toFilter) { auto type = Helper::resolveAliasType(toFilter); return type && type->whichType() == AbstractType::TypeStructure; }, [](AbstractType::Ptr toMap) { return StructureType::Ptr::staticCast(Helper::resolveAliasType(toMap)); } ); auto docFileContext = Helper::getDocumentationFileContext(); for ( const auto& type: structureTypes ) { auto searchContexts = Helper::internalContextsForClass(type, topContext); for ( const auto ctx: searchContexts ) { auto found = ctx->findDeclarations(attribute, CursorInRevision::invalid(), topContext, DUContext::DontSearchInParent); if ( !found.isEmpty() && ( found.last()->topContext() != docFileContext || ctx->topContext() == docFileContext) ) { // never consider decls from the builtins return found.last(); } } } return nullptr; } AbstractType::Ptr Helper::resolveAliasType(const AbstractType::Ptr eventualAlias) { return TypeUtils::resolveAliasType(eventualAlias); } AbstractType::Ptr Helper::extractTypeHints(AbstractType::Ptr type) { return Helper::foldTypes(Helper::filterType(type, [](AbstractType::Ptr t) -> bool { auto hint = t.cast(); return !hint || hint->isValid(); })); } -Helper::FuncInfo Helper::functionDeclarationForCalledDeclaration(DeclarationPointer ptr) +Helper::FuncInfo Helper::functionForCalled(Declaration* called ) { - if ( ! ptr ) { - return FuncInfo(); + if ( ! called ) { + return { nullptr, false }; } - else if ( auto functionDecl = FunctionDeclarationPointer(ptr) ) { - return FuncInfo(functionDecl, false); + else if ( called->isFunctionDeclaration() ) { + return { static_cast( called ), false }; } else { // not a function -- try looking for a constructor static const IndexedIdentifier initIdentifier(KDevelop::Identifier("__init__")); - auto attr = accessAttribute(ptr->abstractType(), initIdentifier, ptr->topContext()); - return FuncInfo(FunctionDeclarationPointer(attr), true); + auto attr = accessAttribute( called->abstractType(), initIdentifier, called->topContext()); + return { dynamic_cast(attr), true }; } } Declaration* Helper::declarationForName(const QualifiedIdentifier& identifier, const RangeInRevision& nodeRange, KDevelop::DUChainPointer context) { QList declarations; QList localDeclarations; QList importedLocalDeclarations; { DUChainReadLocker lock(DUChain::lock()); if ( context.data() == context->topContext() && nodeRange.isValid() ) { declarations = context->topContext()->findDeclarations(identifier, nodeRange.end); } else { declarations = context->topContext()->findDeclarations(identifier, CursorInRevision::invalid()); } localDeclarations = context->findLocalDeclarations(identifier.last(), nodeRange.end, 0, AbstractType::Ptr(0), DUContext::DontResolveAliases); importedLocalDeclarations = context->findDeclarations(identifier.last(), nodeRange.end); } Declaration* declaration = 0; if ( localDeclarations.length() ) { declaration = localDeclarations.last(); } else if ( importedLocalDeclarations.length() ) { // don't use declarations from class decls, they must be referenced through "self." do { declaration = importedLocalDeclarations.last(); importedLocalDeclarations.pop_back(); if ( !declaration || (declaration->context()->type() == DUContext::Class && context->type() != DUContext::Function) ) { declaration = 0; } if ( importedLocalDeclarations.isEmpty() ) { break; } } while ( ! importedLocalDeclarations.isEmpty() ); } if ( !declaration && declarations.length() ) { declaration = declarations.last(); } return declaration; } QVector Helper::internalContextsForClass(const StructureType::Ptr classType, const TopDUContext* context, ContextSearchFlags flags, int depth) { QVector searchContexts; if ( ! classType ) { return searchContexts; } if ( auto c = classType->internalContext(context) ) { searchContexts << c; } auto decl = Helper::resolveAliasDeclaration(classType->declaration(context)); if ( auto classDecl = dynamic_cast(decl) ) { FOREACH_FUNCTION ( const auto& base, classDecl->baseClasses ) { if ( flags == PublicOnly && base.access == KDevelop::Declaration::Private ) { continue; } auto baseClassType = base.baseClass.type(); // recursive call, because the base class will have more base classes eventually if ( depth < 10 ) { searchContexts.append(Helper::internalContextsForClass(baseClassType, context, flags, depth + 1)); } } } return searchContexts; } Declaration* Helper::resolveAliasDeclaration(Declaration* decl) { AliasDeclaration* alias = dynamic_cast(decl); if ( alias ) { DUChainReadLocker lock; return alias->aliasedDeclaration().data(); } else return decl; } QStringList Helper::getDataDirs() { if ( Helper::dataDirs.isEmpty() ) { Helper::dataDirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files", QStandardPaths::LocateDirectory); } return Helper::dataDirs; } QString Helper::getDocumentationFile() { if ( Helper::documentationFile.isNull() ) { Helper::documentationFile = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); } return Helper::documentationFile; } ReferencedTopDUContext Helper::getDocumentationFileContext() { if ( Helper::documentationFileContext ) { return ReferencedTopDUContext(Helper::documentationFileContext.data()); } else { DUChainReadLocker lock; auto file = IndexedString(Helper::getDocumentationFile()); ReferencedTopDUContext ctx = ReferencedTopDUContext(DUChain::self()->chainForDocument(file)); Helper::documentationFileContext = DUChainPointer(ctx.data()); return ctx; } } // stolen from KUrl. duh. static QString _relativePath(const QString &base_dir, const QString &path) { QString _base_dir(QDir::cleanPath(base_dir)); QString _path(QDir::cleanPath(path.isEmpty() || QDir::isRelativePath(path) ? _base_dir+QLatin1Char('/')+path : path)); if (_base_dir.isEmpty()) return _path; if (_base_dir[_base_dir.length()-1] != QLatin1Char('/')) _base_dir.append(QLatin1Char('/') ); const QStringList list1 = _base_dir.split(QLatin1Char('/'), QString::SkipEmptyParts); const QStringList list2 = _path.split(QLatin1Char('/'), QString::SkipEmptyParts); // Find where they meet int level = 0; int maxLevel = qMin(list1.count(), list2.count()); while((level < maxLevel) && (list1[level] == list2[level])) level++; QString result; // Need to go down out of the first path to the common branch. for(int i = level; i < list1.count(); i++) result.append(QLatin1String("../")); // Now up up from the common branch to the second path. for(int i = level; i < list2.count(); i++) result.append(list2[i]).append(QLatin1Char('/')); if ((level < list2.count()) && (path[path.length()-1] != QLatin1Char('/'))) result.truncate(result.length()-1); return result; } QUrl Helper::getCorrectionFile(const QUrl& document) { if ( Helper::correctionFileDirs.isEmpty() ) { Helper::correctionFileDirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files/", QStandardPaths::LocateDirectory); } foreach (QString correctionFileDir, correctionFileDirs) { foreach ( const QUrl& basePath, Helper::getSearchPaths(QUrl()) ) { if ( ! basePath.isParentOf(document) ) { continue; } auto base = basePath.path(); auto doc = document.path(); auto relative = _relativePath(base, doc); auto fullPath = correctionFileDir + "/" + relative; if ( QFile::exists(fullPath) ) { return QUrl::fromLocalFile(fullPath).adjusted(QUrl::NormalizePathSegments); } } } return {}; } QUrl Helper::getLocalCorrectionFile(const QUrl& document) { if ( Helper::localCorrectionFileDir.isNull() ) { Helper::localCorrectionFileDir = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + QLatin1Char('/') + "kdevpythonsupport/correction_files/"; } auto absolutePath = QUrl(); foreach ( const auto& basePath, Helper::getSearchPaths(QUrl()) ) { if ( ! basePath.isParentOf(document) ) { continue; } auto path = QDir(basePath.path()).relativeFilePath(document.path()); absolutePath = QUrl::fromLocalFile(Helper::localCorrectionFileDir + path); break; } return absolutePath; } QString getPythonExecutablePath(IProject* project) { if ( project ) { auto interpreter = project->projectConfiguration()->group("pythonsupport").readEntry("interpreter"); if ( !interpreter.isEmpty() ) { // we have a user-configured interpreter, try using it QFile f(interpreter); if ( f.exists() ) { return interpreter; } qCWarning(KDEV_PYTHON_DUCHAIN) << "Custom python interpreter" << interpreter << "configured for project" << project->name() << "is invalid, using default"; } } // Find python 3 (https://www.python.org/dev/peps/pep-0394/) auto result = QStandardPaths::findExecutable("python" PYTHON_VERSION_MAJOR "." PYTHON_VERSION_MINOR); if ( ! result.isEmpty() ) { return result; } result = QStandardPaths::findExecutable("python" PYTHON_VERSION_MAJOR); if ( ! result.isEmpty() ) { return result; } result = QStandardPaths::findExecutable("python"); if ( ! result.isEmpty() ) { return result; } #ifdef Q_OS_WIN QStringList extraPaths; // Check for default CPython installation path, because the // installer does not add the path to $PATH. QStringList keys = { "HKEY_LOCAL_MACHINE\\Software\\Python\\PythonCore\\PYTHON_VERSION\\InstallPath", "HKEY_LOCAL_MACHINE\\Software\\Python\\PythonCore\\PYTHON_VERSION-32\\InstallPath", "HKEY_CURRENT_USER\\Software\\Python\\PythonCore\\PYTHON_VERSION\\InstallPath", "HKEY_CURRENT_USER\\Software\\Python\\PythonCore\\PYTHON_VERSION-32\\InstallPath" }; auto version = QString(PYTHON_VERSION_MAJOR) + "." + PYTHON_VERSION_MINOR; foreach ( QString key, keys ) { key.replace("PYTHON_VERSION", version); QSettings base(key.left(key.indexOf("Python")), QSettings::NativeFormat); if ( ! base.childGroups().contains("Python") ) { continue; } QSettings keySettings(key, QSettings::NativeFormat); auto path = keySettings.value("Default").toString(); if ( ! path.isEmpty() ) { extraPaths << path; break; } } result = QStandardPaths::findExecutable("python", extraPaths); if ( ! result.isEmpty() ) { return result; } #endif // fallback return PYTHON_EXECUTABLE; } QVector Helper::getSearchPaths(const QUrl& workingOnDocument) { QMutexLocker lock(&Helper::cacheMutex); QVector searchPaths; // search in the projects, as they're packages and likely to be installed or added to PYTHONPATH later // and also add custom include paths that are defined in the projects auto project = ICore::self()->projectController()->findProjectForUrl(workingOnDocument); { QMutexLocker lock(&Helper::projectPathLock); searchPaths << Helper::projectSearchPaths; searchPaths << Helper::cachedCustomIncludes.value(project); } foreach ( const QString& path, getDataDirs() ) { searchPaths.append(QUrl::fromLocalFile(path)); } if ( !cachedSearchPaths.contains(project) ) { QVector cachedForProject; qCDebug(KDEV_PYTHON_DUCHAIN) << "*** Collecting search paths..."; QStringList getpath; getpath << "-c" << "import sys; sys.stdout.write('$|$'.join(sys.path))"; QProcess python; python.start(getPythonExecutablePath(project), getpath); python.waitForFinished(1000); QString pythonpath = QString::fromUtf8(python.readAllStandardOutput()); auto paths = pythonpath.split("$|$"); paths.removeAll(""); if ( ! pythonpath.isEmpty() ) { foreach ( const QString& path, paths ) { cachedForProject.append(QUrl::fromLocalFile(path)); } } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "Could not get search paths! Defaulting to stupid stuff."; searchPaths.append(QUrl::fromLocalFile("/usr/lib/python3.5")); searchPaths.append(QUrl::fromLocalFile("/usr/lib/python3.5/site-packages")); QString path = qgetenv("PYTHONPATH"); QStringList paths = path.split(':'); foreach ( const QString& path, paths ) { cachedForProject.append(QUrl::fromLocalFile(path)); } } qCDebug(KDEV_PYTHON_DUCHAIN) << " *** Done. Got search paths: " << cachedSearchPaths; cachedSearchPaths.insert(project, cachedForProject); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << " --- Search paths from cache: " << cachedSearchPaths; } searchPaths.append(cachedSearchPaths.value(project)); auto dir = workingOnDocument.adjusted(QUrl::RemoveFilename); if ( ! dir.isEmpty() ) { // search in the current packages searchPaths.append(dir); } return searchPaths; } bool Helper::isUsefulType(AbstractType::Ptr type) { return TypeUtils::isUsefulType(type); } AbstractType::Ptr Helper::contentOfIterable(const AbstractType::Ptr iterable, const TopDUContext* topContext) { auto types = filterType(iterable, [](AbstractType::Ptr t) { return t->whichType() == AbstractType::TypeStructure; } ); static const IndexedIdentifier iterId(KDevelop::Identifier("__iter__")); static const IndexedIdentifier nextId(KDevelop::Identifier("__next__")); AbstractType::Ptr content(new IntegralType(IntegralType::TypeMixed)); for ( const auto& type: types ) { if ( auto map = type.cast() ) { // Iterating over dicts gets keys, not values content = mergeTypes(content, map->keyType().abstractType()); continue; } else if ( auto list = type.cast() ) { content = mergeTypes(content, list->contentType().abstractType()); continue; } else if ( auto indexed = type.cast() ) { content = mergeTypes(content, indexed->asUnsureType()); continue; } DUChainReadLocker lock; // Content of an iterable object is iterable.__iter__().__next__(). if ( auto iterFunc = dynamic_cast(accessAttribute(type, iterId, topContext)) ) { if ( auto iterator = iterFunc->type()->returnType().cast() ) { if ( auto nextFunc = dynamic_cast(accessAttribute(iterator, nextId, topContext)) ) { content = mergeTypes(content, nextFunc->type()->returnType()); } } } } return content; } AbstractType::Ptr Helper::mergeTypes(AbstractType::Ptr type, const AbstractType::Ptr newType) { UnsureType::Ptr ret; return TypeUtils::mergeTypes(type, newType); } } diff --git a/duchain/helpers.h b/duchain/helpers.h index fa46fb1c..4cbd6b17 100644 --- a/duchain/helpers.h +++ b/duchain/helpers.h @@ -1,233 +1,233 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2011-2012 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef GLOBALHELPERS_H #define GLOBALHELPERS_H #include "declarations/classdeclaration.h" #include "pythonduchainexport.h" #include "types/unsuretype.h" #include "ast.h" #include #include #include #include #include #include #include #include #include #include #include "pythonduchainexport.h" #include "types/unsuretype.h" #include "declarations/functiondeclaration.h" #include "ast.h" using namespace KDevelop; namespace Python { class KDEVPYTHONDUCHAIN_EXPORT Helper { public: /** get search paths for python files **/ static QVector getSearchPaths(const QUrl& workingOnDocument); static QStringList dataDirs; static QString documentationFile; static QStringList correctionFileDirs; static QString localCorrectionFileDir; static DUChainPointer documentationFileContext; static QStringList getDataDirs(); static QString getDocumentationFile(); static ReferencedTopDUContext getDocumentationFileContext(); static QUrl getCorrectionFile(const QUrl& document); static QUrl getLocalCorrectionFile(const QUrl& document); static QMutex cacheMutex; static QMap> cachedCustomIncludes; static QMap> cachedSearchPaths; static QMutex projectPathLock; static QVector projectSearchPaths; static AbstractType::Ptr extractTypeHints(AbstractType::Ptr type); /** * @brief Get the declaration of 'accessed.attribute', or return null. * * @param accessed Type (Structure or Unsure) that should have this attribute. * @param attribute Which attribute to look for. * @param topContext Top context (for this file?) * @return Declaration* of the attribute, or null. * If UnsureType with >1 matching attributes, returns an arbitrary choice. **/ static KDevelop::Declaration* accessAttribute(const KDevelop::AbstractType::Ptr accessed, const KDevelop::IndexedIdentifier& attribute, const KDevelop::TopDUContext* topContext); static KDevelop::Declaration* accessAttribute(const KDevelop::AbstractType::Ptr accessed, const QString& attribute, const KDevelop::TopDUContext* topContext) { return accessAttribute(accessed, IndexedIdentifier(KDevelop::Identifier(attribute)), topContext); } static AbstractType::Ptr resolveAliasType(const AbstractType::Ptr eventualAlias); /** * @brief Get the content type(s) of something that is an iterable. * * @param iterable Type to get the contents of. Can be an unsure. * @return KDevelop::AbstractType::Ptr Content type. Might be an unsure. */ static AbstractType::Ptr contentOfIterable(const AbstractType::Ptr iterable, const TopDUContext* topContext); /** * @brief Get a list of types inside the passed type which match the specified filter. * The filter will be matched against the type only if it is not an unsure type, * or else against all types inside that unsure type. * @param type The type to search * @param accept Filter function, return true if you want the type. * @return QList< KDevelop::AbstractType::Ptr > list of types accepted by the filter. */ template static QList filterType(AbstractType::Ptr type, std::function accept, std::function map = std::function()) { QList types; if ( ! type ) { return types; } if ( type->whichType() == KDevelop::AbstractType::TypeUnsure ) { UnsureType::Ptr unsure(type.cast()); for ( unsigned int i = 0; i < unsure->typesSize(); i++ ) { AbstractType::Ptr t = unsure->types()[i].abstractType(); if ( accept(t) ) { types << ( map ? map(t) : t.cast() ); } } } else if ( accept(type) ) { types << ( map ? map(type) : type.cast() ); } return types; } static void scheduleDependency(const IndexedString& dependency, int betterThanPriority); static KDevelop::IndexedDeclaration declarationUnderCursor(bool allowUse = true); - using FuncInfo = QPair; + struct FuncInfo { FunctionDeclaration* declaration; bool isConstructor; }; /** * @brief Finds whether the specified called declaration is a function declaration, and if not, * checks for a class declaration; then returns the constructor * - * @param ptr the declaration to check + * @param called the declaration to check * @return the function pointer which was found, or an invalid pointer, and a bool * which is true when it is a constructor **/ - static FuncInfo functionDeclarationForCalledDeclaration(DeclarationPointer ptr); + static FuncInfo functionForCalled(Declaration* called); template static const Decorator* findDecoratorByName(T* inDeclaration, const QString& name) { const int count = inDeclaration->decoratorsSize(); const IndexedString indexedName = IndexedString(name); for ( int i = 0; i < count; i++ ) { if ( inDeclaration->decorators()[i].fastName() == indexedName ) return &(inDeclaration->decorators()[i]); } return 0; }; static bool docstringContainsHint(Declaration* declaration, const QString& hintName, QStringList* args = 0) { // TODO cache types! this is horribly inefficient const QString& comment = declaration->comment(); const QString search = "! " + hintName + " !"; int index = comment.indexOf(search); if ( index >= 0 ) { if ( args ) { int eol = comment.indexOf('\n', index); int start = index+search.size()+1; QString decl = comment.mid(start, eol-start); *args = decl.split(' '); } return true; } return false; } /** * @copydoc TypeUtils::mergeTypes */ static AbstractType::Ptr mergeTypes(AbstractType::Ptr type, const AbstractType::Ptr newType); /** * @brief Like mergeTypes(), but merges a list of types into a newly allocated type. * Returns mixed if the list is empty. * @return KDevelop::AbstractType::Ptr an unsure type consisting of all types in the list. */ template static AbstractType::Ptr foldTypes(QList types, std::function transform = std::function()) { AbstractType::Ptr result(new IntegralType(IntegralType::TypeMixed)); for ( T type : types ) { result = Helper::mergeTypes(result, transform ? transform(type) : AbstractType::Ptr::staticCast(type)); } return result; }; /** check whether the argument is a null, mixed, or none integral type **/ static bool isUsefulType(AbstractType::Ptr type); enum ContextSearchFlags { NoFlags, PublicOnly }; /** * @brief Find all internal contexts for this class and its base classes recursively * * @param classType Type object for the class to search contexts * @param context TopContext for finding the declarations for types * @return list of contexts which were found **/ static QVector internalContextsForClass(const KDevelop::StructureType::Ptr classType, const TopDUContext* context, ContextSearchFlags flags = NoFlags, int depth = 0); /** * @brief Resolve the given declaration if it is an alias declaration. * * @param decl the declaration to resolve * @return :Declaration* decl if not an alias declaration, decl->aliasedDeclaration().data otherwise * DUChain must be read locked **/ static Declaration* resolveAliasDeclaration(Declaration* decl); static Declaration* declarationForName(const QualifiedIdentifier& identifier, const RangeInRevision& nodeRange, DUChainPointer context); }; } #endif diff --git a/duchain/usebuilder.cpp b/duchain/usebuilder.cpp index 9f6da41c..0d5cd9b2 100644 --- a/duchain/usebuilder.cpp +++ b/duchain/usebuilder.cpp @@ -1,152 +1,150 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2010-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "usebuilder.h" #include #include "duchaindebug.h" #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "ast.h" #include "expressionvisitor.h" #include "helpers.h" using namespace KTextEditor; using namespace KDevelop; namespace Python { UseBuilder::UseBuilder(PythonEditorIntegrator* editor, QVector ignoreVariables) : UseBuilderBase() , m_errorReportingEnabled(true) , m_ignoreVariables(ignoreVariables) { setEditor(editor); } DUContext* UseBuilder::contextAtOrCurrent(const CursorInRevision& pos) { DUContext* context = 0; { DUChainReadLocker lock; context = topContext()->findContextAt(pos, true); } if ( ! context ) { context = currentContext(); } return context; } void UseBuilder::visitName(NameAst* node) { DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node)); Declaration* declaration = Helper::declarationForName(identifierForNode(node->identifier), editorFindRange(node, node), DUChainPointer(context)); static QStringList keywords; if ( keywords.isEmpty() ) { keywords << "None" << "True" << "False" << "print"; } Q_ASSERT(node->identifier); RangeInRevision useRange = rangeForNode(node->identifier, true); if ( declaration && declaration->range() == useRange ) return; if ( ! declaration && ! keywords.contains(node->identifier->value) && m_errorReportingEnabled ) { if ( ! m_ignoreVariables.contains(IndexedString(node->identifier->value)) ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Hint); p->setDescription(i18n("Undefined variable: %1", node->identifier->value)); { DUChainWriteLocker wlock(DUChain::lock()); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( declaration && declaration->abstractType() && declaration->abstractType()->whichType() == AbstractType::TypeStructure ) { if ( node->belongsToCall ) { DUChainReadLocker lock; - QPair< Python::FunctionDeclarationPointer, bool > constructor = Helper:: - functionDeclarationForCalledDeclaration(DeclarationPointer(declaration)); + auto constructor = Helper::functionForCalled(declaration); lock.unlock(); - bool isConstructor = constructor.second; - if ( isConstructor ) { + if ( constructor.isConstructor ) { RangeInRevision constructorRange; // TODO fixme! this does not necessarily use the opening bracket as it should constructorRange.start = CursorInRevision(node->endLine, node->endCol + 1); constructorRange.end = CursorInRevision(node->endLine, node->endCol + 2); - UseBuilderBase::newUse(node, constructorRange, DeclarationPointer(constructor.first)); + UseBuilderBase::newUse(node, constructorRange, DeclarationPointer(constructor.declaration)); } } } UseBuilderBase::newUse(node, useRange, DeclarationPointer(declaration)); } void UseBuilder::visitAttribute(AttributeAst* node) { qCDebug(KDEV_PYTHON_DUCHAIN) << "VisitAttribute start"; UseBuilderBase::visitAttribute(node); qCDebug(KDEV_PYTHON_DUCHAIN) << "Visit Attribute base end"; DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node)); ExpressionVisitor v(context); v.visitNode(node); RangeInRevision useRange(node->attribute->startLine, node->attribute->startCol, node->attribute->endLine, node->attribute->endCol + 1); DeclarationPointer declaration = v.lastDeclaration(); DUChainWriteLocker wlock; if ( declaration && declaration->range() == useRange ) { // this is the declaration, don't build a use for it return; } if ( ! declaration && v.isConfident() && ( ! v.lastType() || Helper::isUsefulType(v.lastType()) ) ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Hint); p->setDescription(i18n("Attribute \"%1\" not found on accessed object", node->attribute->value)); ProblemPointer ptr(p); topContext()->addProblem(ptr); } UseBuilderBase::newUse(node, useRange, declaration); } ParseSession *UseBuilder::parseSession() const { return m_session; } } // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on