diff --git a/CMakeLists.txt b/CMakeLists.txt index bc408b82..b03b9f6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,114 +1,114 @@ -PROJECT(kdevpython) +cmake_minimum_required(VERSION 2.8.12) + +project(kdevpython) # write the plugin version to a file set(KDEVPYTHON_VERSION_MAJOR 5) set(KDEVPYTHON_VERSION_MINOR 0) set(KDEVPYTHON_VERSION_PATCH 80) # KDevplatform dependency version set( KDEVPLATFORM_VERSION "${KDEVPYTHON_VERSION_MAJOR}.${KDEVPYTHON_VERSION_MINOR}" ) find_package(ECM 0.0.9 REQUIRED NO_MODULE) set(CMAKE_MODULE_PATH ${kdevpython_SOURCE_DIR}/cmake/modules ${CMAKE_MODULE_PATH} ${ECM_MODULE_PATH}) set(REQUIRED QT_VERSION 5.2.0) -cmake_minimum_required(VERSION 2.8.12) - include(KDECompilerSettings NO_POLICY_SCOPE) include(GenerateExportHeader) include(CMakePackageConfigHelpers) include(ECMAddTests) include(ECMOptionalAddSubdirectory) include(ECMSetupVersion) include(KDEInstallDirs) include(KDECMakeSettings) add_definitions( -DTRANSLATION_DOMAIN=\"kdevpython\" ) # find the system python 3 interpreter, only used for determining search paths. # must be called before find_package(KF5) because it searchs for python too, but finds python2 set(Python_ADDITIONAL_VERSIONS 3.6 3.5) find_package(PythonInterp 3.5 REQUIRED) find_package(PythonLibs 3.5 REQUIRED) configure_file( "${kdevpython_SOURCE_DIR}/kdevpythonversion.h.cmake" "${kdevpython_BINARY_DIR}/kdevpythonversion.h" @ONLY ) if ( (NOT PYTHON_LIBRARIES) OR "${PYTHON_VERSION_MINOR}" GREATER 6 ) message(FATAL_ERROR "Python >= 3.5.0 but < 3.7 with --enable-shared is required to build kdev-python") endif() find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED Widgets Test) find_package(KF5 REQUIRED I18n NewStuff ItemModels ThreadWeaver TextEditor KCMUtils) find_package(KDevPlatform ${KDEVPLATFORM_VERSION} REQUIRED) find_package(KDevelop ${KDEVPLATFORM_VERSION} REQUIRED) enable_testing() if ( NOT WIN32 ) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors -Wall") endif ( NOT WIN32 ) # then, build the plugin include_directories( ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/duchain ${CMAKE_CURRENT_SOURCE_DIR}/parser ${CMAKE_CURRENT_BINARY_DIR}/parser ) add_subdirectory(app_templates) add_subdirectory(parser) add_subdirectory(duchain) add_subdirectory(codecompletion) add_subdirectory(debugger) add_subdirectory(docfilekcm) set(kdevpythonlanguagesupport_PART_SRCS codegen/correctionfilegenerator.cpp codegen/refactoring.cpp codegen/codegendebug.cpp pythonlanguagesupport.cpp pythonparsejob.cpp pythonhighlighting.cpp pythondebug.cpp pythonstylechecking.cpp # config pages: docfilekcm/docfilewizard.cpp docfilekcm/docfilemanagerwidget.cpp docfilekcm/kcm_docfiles.cpp pep8kcm/kcm_pep8.cpp projectconfig/projectconfigpage.cpp ) ki18n_wrap_ui(kdevpythonlanguagesupport_PART_SRCS codegen/correctionwidget.ui projectconfig/projectconfig.ui pep8kcm/pep8.ui ) kdevplatform_add_plugin(kdevpythonlanguagesupport JSON kdevpythonsupport.json SOURCES ${kdevpythonlanguagesupport_PART_SRCS}) target_link_libraries(kdevpythonlanguagesupport KDev::Interfaces KDev::Language KDev::Util KF5::ThreadWeaver KF5::TextEditor KF5::NewStuff kdevpythoncompletion kdevpythonparser kdevpythonduchain ) get_target_property(DEFINESANDINCLUDES_INCLUDE_DIRS KDev::DefinesAndIncludesManager INTERFACE_INCLUDE_DIRECTORIES) include_directories(${DEFINESANDINCLUDES_INCLUDE_DIRS}) install(DIRECTORY documentation_files DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) install(DIRECTORY correction_files DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) install(FILES codestyle.py DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES) diff --git a/README.packagers b/README.packagers index a2d8fc04..9d6b1b43 100644 --- a/README.packagers +++ b/README.packagers @@ -1,40 +1,38 @@ This information is intended for package mantainers. First of all, thanks for packaging this software! If you encounter any issues which you think should be fixed upstream, please report a bug or send a patch, I'll be happy to accept it if it makes sense. Notes ===== This is a collection of notes about what might be of interest, related to packaging. -Conflict with other versions (IMPORTANT) ----------------------------------------- -The python3 version of kdev-python can NOT be installed alongside other versions. -Please ensure that a package containing the e.g. 1.6 release conflicts with -a package containing 1.6-py3. -I am working on eventually lifting this annoyance in the future. +Supported Python versions +------------------------- +Python 2 support was dropped, and we just support the latest 2 to 3 minor +releases of Python 3 right now. Python fork ----------- This branch, which contains the python3 version of kdev-python, does no longer contain a fork of python. Instead, it links against your system's python. Python is both a build- and a runtime dependency of this program. documentation_data directory ---------------------------- This directory does NOT contain user documentation for the plugin. It MUST be packaged together with the program, as it contains runtime data which is necessary for the program to work correctly (such as representations of python's built-in data types, which are then read by the parser etc.). Licensing notes --------------- The following files are not copyrighted: Everything in duchain/tests/data/ Everything in documentation_files/ Everything in correction_files/ Everything in app_templates/ example_ast.py diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index 509537fc..8505e36f 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1923 +1,1933 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/decorator.h" #include "duchain/declarations/functiondeclaration.h" #include "duchain/declarations/classdeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); qCDebug(KDEV_PYTHON_DUCHAIN) << "Building Declarations"; } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "building, but running pre-builder first"; DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); qCDebug(KDEV_PYTHON_DUCHAIN) << "pre-builder finished"; delete prebuilder; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "prebuilding"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, Ast* range, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = 0; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(range, range), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name, range); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. QList declaringContexts; declaringContexts << ExpressionAst::Store << ExpressionAst::Parameter << ExpressionAst::AugStore; if ( ! declaringContexts.contains(currentVariableDefinition->context) ) { return 0; } Identifier* id = currentVariableDefinition->identifier; return visitVariableDeclaration(id, currentVariableDefinition, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), nullptr, previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(0); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { QList existingDeclarations = currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); // append arguments context if ( m_mostRecentArgumentsContext ) { QList args = m_mostRecentArgumentsContext->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent ); existingDeclarations.append(args); } return existingDeclarations; } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = 0; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != 0 ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Ast* originalAst, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; Ast* rangeNode = originalAst ? originalAst : node; RangeInRevision range = editorFindRange(rangeNode, rangeNode); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = 0; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(identifierForNode(node), range); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << hints->toString(); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, 0, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); visitVariableDeclaration(node->optionalVars, 0, v.lastType()); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = 0; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), 0, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return 0; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = 0; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(0); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(0); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void spoofNodePosition(Ast* node, const CursorInRevision& pos) { // Ridiculous hack, see next comment. node->startLine = node->endLine = pos.line; node->startCol = node->endCol = pos.column - 1; if (node->astType == Ast::TupleAstType) { // Recursion to bodge all declarations, e.g. // [x + y * z for x, (y, z) in foo] foreach(auto elem, static_cast(node)->elements) { spoofNodePosition(elem, pos); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); // make the declaration zero chars long; it must appear at the beginning of the context, // because it is actually used *before* its real declaration: [foo for foo in bar] // The DUChain doesn't like this, so for now, the declaration is at the opening bracket, // and both other occurrences are uses of that declaration. // TODO add a special case to the usebuilder to display the second occurrence as a declaration spoofNodePosition(node->target, currentContext()->range().start); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(0); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = 0; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = 0; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = 0; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, &temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return 0; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = 0; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return 0; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return 0; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, 0, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, 0); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(0), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { Python::AstDefaultVisitor::visitLambda(node); DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } if (node->arguments->vararg) { visitVariableDeclaration(node->arguments->vararg->argumentName); } if (node->arguments->kwarg) { visitVariableDeclaration(node->arguments->kwarg->argumentName); } closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker wlock; auto contentType = Helper::contentOfIterable(argVisitor.lastType(), topContext()); container->addContentType(contentType); v.lastDeclaration()->setType(container); } }; auto docstring = function->comment(); if ( ! docstring.isEmpty() ) { foreach ( const auto& key, items.keys() ) { if ( Helper::docstringContainsHint(docstring, key, &args) ) { items[key](); } } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer called) { DUChainReadLocker lock; auto funcInfo = Helper::functionForCalled(called.data()); auto function = funcInfo.declaration; if ( ! function ) { return; } if ( function->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { return; } // Note: within this function: // - 'parameters' refers to the parameters of the function definition. // - 'arguments' refers to the arguments of the function call. DUContext* parameterContext = DUChainUtils::getArgumentContext(function); FunctionType::Ptr functionType = function->type(); if ( ! parameterContext || ! functionType ) { return; } QVector parameters = parameterContext->localDeclarations(); if ( parameters.isEmpty() ) { return; } const int specialParamsCount = (function->vararg() != -1) + (function->kwarg() != -1); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfParam = false; if ( ( function->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! function->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as parameters // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfParam = true; } } lock.unlock(); bool explicitSelfArgument = false; if ( hasSelfParam && ! function->isClassMethod() && node->function->astType == Ast::AttributeAstType ) { // Calling an attribute, e.g. `instance.foo(arg)` or `MyClass.foo(instance, arg)`. ExpressionVisitor valueVisitor(currentContext()); valueVisitor.visitNode(static_cast(node->function)->value); if ( valueVisitor.lastDeclaration().dynamicCast() && valueVisitor.isAlias() ) { // Function is attribute of a class _type_ (not instance), so first arg is used as `self`. explicitSelfArgument = true; } } int currentParamIndex = hasSelfParam; int currentArgumentIndex = explicitSelfArgument; int indexInVararg = -1; int paramsAvailable = qMin(functionType->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentArgumentIndex++ ) { atVararg = atVararg || currentParamIndex == function->vararg(); // Not >=, nonexistent vararg is -1. qCDebug(KDEV_PYTHON_DUCHAIN) << currentParamIndex << currentArgumentIndex << atVararg << function->vararg(); ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(function->vararg()); IndexedContainer::Ptr varargContainer = parameter->type(); qCDebug(KDEV_PYTHON_DUCHAIN) << "adding" << addType->toString() << "at position" << indexInVararg; if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functionType->removeArgument(currentParamIndex); functionType->addArgument(newType, currentParamIndex); function->setAbstractType(functionType.cast()); parameters.at(currentParamIndex)->setType(newType); currentParamIndex++; } } // **kwargs is always the last parameter MapType::Ptr kwargsDict; if ( function->kwarg() != -1 ) { kwargsDict = parameters.last()->abstractType().cast(); } lock.unlock(); DUChainWriteLocker wlock; foreach ( KeywordAst* keyword, node->keywords ) { wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); bool matchedNamedParam = false; HintedType::Ptr addType = HintedType::Ptr(new HintedType()); if ( keyword->argumentName ) { openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); for (int ip = currentParamIndex; ip < paramsAvailable; ++ip ) { if ( parameters.at(ip)->identifier().toString() != keyword->argumentName->value ) { continue; } matchedNamedParam = true; auto newType = Helper::mergeTypes(parameters.at(ip)->abstractType(), addType); functionType->removeArgument(ip); functionType->addArgument(newType, ip); parameters.at(ip)->setType(newType); } } else if ( auto unpackedDict = argumentVisitor.lastType().cast() ) { // 'keyword is actually an unpacked dict: `foo(**{'a': 12}). openType(addType); addType->setType(unpackedDict->contentType().abstractType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); } else { // Maybe the dict type wasn't loaded yet, or something else happened. continue; } if ( ! matchedNamedParam && kwargsDict ) { kwargsDict->addContentType(addType); parameters.last()->setAbstractType(kwargsDict); } } function->setAbstractType(functionType); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; Python::Identifier* identifier = target->identifier; AliasDeclaration* decl = eventuallyReopenDeclaration(identifier, target, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, 0, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(0); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreayOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attrib, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } else qCWarning(KDEV_PYTHON_DUCHAIN) << "No declaration created for " << attrib->attribute << "as parent is not a class"; closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 || (starred == -1 && spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } +void DeclarationBuilder::visitAnnotationAssignment(AnnotationAssignmentAst* node) { + ExpressionVisitor v(currentContext()); + v.visitNode(node->target); + v.visitNode(node->value); + auto assignType = v.lastType(); // Never mind aliasing, why annotate that? + v.visitNode(node->annotation); + assignType = Helper::mergeTypes(assignType, v.lastType()); + assignToUnknown(node->target, assignType); +} + void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, NoTypeRequired); lock.unlock(); visitDecorators(node->decorators, dec); lock.lock(); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); auto docstring = getDocstring(node->body); dec->setComment(docstring); if ( ! docstring.isEmpty() ) { // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(docstring, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(docstring, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(docstring, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassType = type; // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); foreach ( Ast* _node, node->body ) { AstDefaultVisitor::visitNode(_node); } lock.lock(); closeContext(); closeType(); closeDeclaration(); } template void DeclarationBuilder::visitDecorators(QList< Python::ExpressionAst* > decorators, T* addTo) { foreach ( ExpressionAst* decorator, decorators ) { AstDefaultVisitor::visitNode(decorator); if ( decorator->astType == Ast::CallAstType ) { CallAst* call = static_cast(decorator); Decorator d; if ( call->function->astType != Ast::NameAstType ) { continue; } d.setName(*static_cast(call->function)->identifier); foreach ( ExpressionAst* arg, call->arguments ) { if ( arg->astType == Ast::NumberAstType ) { d.setAdditionalInformation(QString::number(static_cast(arg)->value)); } else if ( arg->astType == Ast::StringAstType ) { d.setAdditionalInformation(static_cast(arg)->value); } break; // we only need the first argument for documentation analysis } addTo->addDecorator(d); } else if ( decorator->astType == Ast::NameAstType ) { NameAst* name = static_cast(decorator); Decorator d; d.setName(*(name->identifier)); addTo->addDecorator(d); } } } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); visitDecorators(node->decorators, dec); const bool isStatic = Helper::findDecoratorByName(dec, "staticmethod"); const bool isClassMethod = Helper::findDecoratorByName(dec, "classmethod"); dec->setStatic(isStatic); dec->setClassMethod(isClassMethod); visitFunctionArguments(node); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } dec->setType(type); } if ( ! isStatic ) { DUContext* args = DUChainUtils::getArgumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! isClassMethod ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); qCDebug(KDEV_PYTHON_DUCHAIN) << "updated function return type to " << type->toString(); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); if ( node->value ) { if ( ! hasCurrentType() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } else { TypePtr t = currentType(); AbstractType::Ptr encountered = v.lastType(); DUChainWriteLocker lock; if ( t ) { // Update the containing function's return type t->setReturnType(Helper::mergeTypes(t->returnType(), encountered)); } } } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; qCDebug(KDEV_PYTHON_DUCHAIN) << "arguments:" << node->arguments.size(); foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } qCDebug(KDEV_PYTHON_DUCHAIN) << "visiting argument:" << arg->argumentName->value; // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && workingOnDeclaration->isClassMethod() ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, arg, AliasDeclarationType); if ( m_currentClassType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting declaration:" << m_currentClassType->declaration(topContext())->toString(); decl->setAliasedDeclaration(m_currentClassType->declaration(currentContext()->topContext())); } closeDeclaration(); paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } qCDebug(KDEV_PYTHON_DUCHAIN) << "is first:" << isFirst << hasCurrentDeclaration() << currentDeclaration(); if ( isFirst && ! workingOnDeclaration->isStatic() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassType.cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); if ( argumentType ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "creating argument with type" << argumentType->toString(); } } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, 0, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, 0, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id, node); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/declarationbuilder.h b/duchain/declarationbuilder.h index 4452970c..900da5fa 100644 --- a/duchain/declarationbuilder.h +++ b/duchain/declarationbuilder.h @@ -1,289 +1,290 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2011-2014 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef PYTHON_DECLARATIONBUILDER_H #define PYTHON_DECLARATIONBUILDER_H #include #include #include "declarations/functiondeclaration.h" #include "typebuilder.h" #include "ast.h" namespace Python { class CorrectionHelper; typedef KDevelop::AbstractDeclarationBuilder DeclarationBuilderBase; class KDEVPYTHONDUCHAIN_EXPORT DeclarationBuilder: public DeclarationBuilderBase { public: DeclarationBuilder(PythonEditorIntegrator* editor, int ownPriority); virtual ~DeclarationBuilder(); /** * @brief Entry function, called by KDevPlatform. */ ReferencedTopDUContext build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext = ReferencedTopDUContext()) override; /** * @brief Set whether the current running pass is the first or the second one. * @param prebuilding true if first pass, false if second */ void setPrebuilding(bool prebuilding); /** * @brief Priority of this parse job. */ int jobPriority() const; /** * @brief Get the docstring which belongs to the given body * @param body Body of statements which is in the documented function or class */ QString getDocstring(QList body) const; /** * @brief Construct the dotted name of a module from an import ... from ... as statement. * * @param node the import ... from node * @param alias the ... as ... node * @param intermediate an additional string to prepend to the module name (dot is added internally) */ QString buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const; /** * @brief Get a list of all module names which were not found while parsing. */ QVector missingModules() const { return m_missingModules; } protected: /// AST visitor functions void visitClassDefinition(ClassDefinitionAst* node) override; void visitFunctionDefinition(FunctionDefinitionAst* node) override; void visitAssignment(AssignmentAst* node) override; + void visitAnnotationAssignment(AnnotationAssignmentAst* node) override; void visitFor(ForAst* node) override; void visitImport(ImportAst* node) override; void visitImportFrom(ImportFromAst* node) override; void visitArguments(ArgumentsAst* node) override; void visitExceptionHandler(ExceptionHandlerAst* node) override; void visitReturn(ReturnAst* node) override; void visitCode(CodeAst* node) override; void visitCall(CallAst* node) override; void visitYield(YieldAst* node) override; void visitWithItem(WithItemAst* node) override; void visitLambda(LambdaAst* node) override; void visitComprehension(ComprehensionAst* node) override; void visitGlobal(GlobalAst* node) override; void visitAssertion(AssertionAst* node) override; void visitIf(IfAst* node) override; void visitString(StringAst* node) override; void visitNode(Ast* node) override; protected: enum VisitVariableFlags { NoVisitVariableFlags = 0x0, AbortIfReopenMismatch = 0x1 }; /// Visitor helper functions template void visitDecorators(QList decorators, T* addTo); template T* visitVariableDeclaration(Python::Ast* node, Declaration* previous=nullptr, AbstractType::Ptr type = AbstractType::Ptr(), VisitVariableFlags flags=NoVisitVariableFlags); template T* visitVariableDeclaration(Identifier* node, Ast* originalAst = 0, Declaration* previous=nullptr, AbstractType::Ptr type = AbstractType::Ptr(), VisitVariableFlags flags=NoVisitVariableFlags); protected: /** * @brief Applies docstring hints, such as "addsType" * @param node the called function * @param function the declaration which belongs to @p node * * Used for example in a = []; a.append(3) to set the type of a to "list of int". */ void applyDocstringHints(CallAst* node, Python::FunctionDeclaration::Ptr function); /** * @brief Try to deduce types of function arguments from a call and stores it in the duchain * @param node the called function or class (i.e. constructor) * @param called the declaration which belongs to @p node * * Used for example in def f(x): pass; a = f(3) to set the type of x to "int" */ void addArgumentTypeHints(CallAst* node, DeclarationPointer called); /** * @brief Adjust the type of foo in an expression like assert isinstance(fooinstance, Foo) * Does nothing if the given expression isn't of any of the forms * a) isinstance(fooinstance, Foo) * b) type(fooinstance) == Foo */ void adjustForTypecheck(ExpressionAst* check, bool useUnsure); /// Helper for the above void adjustExpressionsForTypecheck(ExpressionAst* adjust, ExpressionAst* from, bool useUnsure); /// Represents a single source type in a tuple assignment. struct SourceType { AbstractType::Ptr type; DeclarationPointer declaration; bool isAlias; }; /** @brief If sourceType is a container that can be unpacked into outTypes, do so. */ void tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred); /** * @brief Handle a variable assignment to @p name and give it the type @p element. */ void assignToName(NameAst* name, const SourceType& element); /** * @brief Handle assignment to subscript @p subscript with rhs type @p element. */ void assignToSubscript(SubscriptAst* subscript, const SourceType& element); /** * @brief Handle assignment to an attribute @p attribute with rhs type @p element. */ void assignToAttribute(AttributeAst* attribute, const SourceType& element); /** * @brief Handle assignment to a target @p target with rhs type @p element. */ void assignToTuple(TupleAst* tuple, const SourceType& element); /** * @brief Handle assignment to a target @p target with rhs type @p element. */ void assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type); void assignToUnknown(ExpressionAst* target, const SourceType& element); /** * @brief Find all existing declarations for the identifier @p node */ QList existingDeclarationsForNode(Identifier* node); enum FitDeclarationType { NoTypeRequired, InstanceDeclarationType, AliasDeclarationType, FunctionDeclarationType }; FitDeclarationType kindForType(AbstractType::Ptr type, bool isAlias = false); /** * @brief schedule an object to be deleted when the declaration builder is destroyed * this is used to bypass the automated duchain cleanup for imports */ void scheduleForDeletion(DUChainBase* d, bool doschedule = true); /** * @brief python-specific version of openDeclaration which scans for existing declarations in previous versions of * this top-context in a more intelligent way. * Use this in normal declaratonbuilder code if you can't use visitVariableDeclaration. */ template T* eventuallyReopenDeclaration(Python::Identifier* name, Python::Ast* range, FitDeclarationType mustFitType); template QList reopenFittingDeclaration(QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok); /** * @brief Create a declaration for an import statement. * * @param dottedName The dotted name of the module, like "random.randint". * @param declarationIdentifier provides the name and range * @param rangeNode can be used to override the declarationIdentifier's range, if required. Defaults to 0. * @param createProblem whether or not to create DUChain problems Defaults to CreateProblems. * @return :Declaration* the declaration created, or 0 if none was found. **/ Declaration* createModuleImportDeclaration(QString dottedName, QString declarationName, Python::Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Python::Ast* rangeNode = 0); /** * @brief Create a tree of declarations for the specified list. * Give the list ["foo","bar","baz"], and you'll get a declaration "foo" containing "bar" in its internal context, * "bar" containing "baz" etc. * This is used in import handling. * This function automatically updates existing declaration trees to the maximum level possible! Thus, * if you call this with ["foo", "bar"], then ["foo", "baz"], "baz" will be added to "foo". * * @warning The DUChain must not be locked when this is called. * * @param nameComponents the list of names to create declarations for * @param declarationIdentifier provides the name and range * @param innerCtx the internalContext() to set on the last created declaration. Either this or aliasDeclaration must be provided! * @param aliasDeclaration the declaration to alias with the last created declaration * @param rangeNode can be used to override the declarationIdentifier's range, if required. Defaults to 0. * @return :Declaration* the top level declaration created **/ Declaration* createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration = 0, const RangeInRevision& range = RangeInRevision::invalid()); /** * @brief Find a declaration specified by "foo.bar.baz" in the given top context. * * @param dottedNameIdentifier string list of module names, starting with the most general one. * @param ctx top context to search * @return :Declaration* declaration if found, 0 otherwise. * * This will traverse nested classes and properties until the list of passed names is exhausted. * @warning The DUChain must not be locked. **/ Declaration* findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const; private: template T* openDeclaration(Identifier* name, Ast* range, DeclarationFlags flags = NoFlags) { T* decl = DeclarationBuilderBase::openDeclaration(name, range, flags); decl->setAlwaysForceDirect(true); return decl; }; template T* openDeclaration(const QualifiedIdentifier& id, const RangeInRevision& newRange, DeclarationFlags flags = NoFlags) { T* decl = DeclarationBuilderBase::openDeclaration(id, newRange, flags); decl->setAlwaysForceDirect(true); return decl; }; void closeDeclaration() override; private: /// HACK: List of items to delete after parsing finishes, to work around the built-in cleanup logic QList m_scheduledForDeletion; QScopedPointer m_correctionHelper; int m_ownPriority = 0; StructureType::Ptr m_currentClassType; // missing modules, for not reporting them as unknown variables QVector m_missingModules; StringAst* m_lastComment = nullptr; }; } #endif // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on diff --git a/duchain/tests/pyduchaintest.cpp b/duchain/tests/pyduchaintest.cpp index fd20cc81..ccb3493f 100644 --- a/duchain/tests/pyduchaintest.cpp +++ b/duchain/tests/pyduchaintest.cpp @@ -1,1662 +1,1664 @@ /***************************************************************************** * Copyright 2010 (c) Miquel Canes Gonzalez * * Copyright 2012 (c) Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include "duchaindebug.h" #include "pyduchaintest.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" QTEST_MAIN(PyDUChainTest) using namespace KDevelop; using namespace Python; PyDUChainTest::PyDUChainTest(QObject* parent): QObject(parent) { assetsDir = QDir(DUCHAIN_PY_DATA_DIR); if (!assetsDir.cd("data")) { qFatal("Failed find data directory for test files. Aborting"); } testDir = QDir(testDirOwner.path()); qputenv("PYTHONPATH", assetsDir.absolutePath().toUtf8()); initShell(); } QList PyDUChainTest::FindPyFiles(QDir& rootDir) { QList foundfiles; rootDir.setFilter(QDir::Files | QDir::Dirs | QDir::NoDot | QDir::NoDotDot); rootDir.setNameFilters(QStringList() << "*.py"); // We only want .py files QDirIterator it(rootDir, QDirIterator::Subdirectories); while(it.hasNext()) { foundfiles.append(it.next()); } return foundfiles; } void PyDUChainTest::init() { QString currentTest = QString(QTest::currentTestFunction()); if (lastTest == currentTest) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Already prepared assets for " << currentTest << ", skipping"; return; } else { lastTest = currentTest; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Preparing assets for test " << currentTest; QDir assetModuleDir = QDir(assetsDir.absolutePath()); if (!assetModuleDir.cd(currentTest)) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Asset directory " << currentTest << " does not exist under " << assetModuleDir.absolutePath() << ". Skipping it."; return; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Searching for python files in " << assetModuleDir.absolutePath(); QList foundfiles = FindPyFiles(assetModuleDir); QString correctionFileDir = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files", QStandardPaths::LocateDirectory); auto correctionFileUrl = QUrl(QDir::cleanPath(correctionFileDir + "/testCorrectionFiles/example.py")); foundfiles.prepend(correctionFileUrl.path()); for ( int i = 0; i < 2; i++ ) { // Parse each file twice, to ensure no parsing-order related bugs appear. // Such bugs will need separate unit tests and should not influence these. foreach(const QString filename, foundfiles) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Parsing asset: " << filename; DUChain::self()->updateContextForUrl(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); } foreach(const QString filename, foundfiles) { DUChain::self()->waitForUpdate(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); } while ( ICore::self()->languageController()->backgroundParser()->queuedCount() > 0 ) { // make sure to wait for all parsejobs to finish QTest::qWait(10); } } } void PyDUChainTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); auto doc_url = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); qCDebug(KDEV_PYTHON_DUCHAIN) << doc_url; DUChain::self()->updateContextForUrl(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); DUChain::self()->waitForUpdate(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } ReferencedTopDUContext PyDUChainTest::parse(const QString& code) { TestFile* testfile = new TestFile(code + "\n", "py", 0, testDir.absolutePath().append("/")); createdFiles << testfile; testfile->parse((TopDUContext::Features) (TopDUContext::ForceUpdate | TopDUContext::AST) ); testfile->waitForParsed(2000); if ( testfile->isReady() ) { Q_ASSERT(testfile->topContext()); m_ast = static_cast(testfile->topContext()->ast().data())->ast; return testfile->topContext(); } else Q_ASSERT(false && "Timed out waiting for parser results, aborting all tests"); return 0; } PyDUChainTest::~PyDUChainTest() { foreach ( TestFile* f, createdFiles ) { delete f; } testDir.rmdir(testDir.absolutePath()); } void PyDUChainTest::testMultiFromImport() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList a = ctx->findDeclarations(QualifiedIdentifier("a")); QList b = ctx->findDeclarations(QualifiedIdentifier("b")); QVERIFY(! a.isEmpty()); QVERIFY(! b.isEmpty()); QVERIFY(a.first()->abstractType()->toString().endsWith("int")); QVERIFY(b.first()->abstractType()->toString().endsWith("int")); } void PyDUChainTest::testMultiFromImport_data() { QTest::addColumn("code"); QTest::newRow("multiimport") << "import testMultiFromImport.i.localvar1\n" "import testMultiFromImport.i.localvar2\n" "a = testMultiFromImport.i.localvar1\n" "b = testMultiFromImport.i.localvar2\n"; } void PyDUChainTest::testRelativeImport() { QFETCH(QString, code); QFETCH(QString, token); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList t = ctx->findDeclarations(QualifiedIdentifier(token)); QVERIFY(! t.isEmpty()); QVERIFY(t.first()->abstractType()->toString().endsWith(type)); } void PyDUChainTest::testRelativeImport_data() { QTest::addColumn("code"); QTest::addColumn("token"); QTest::addColumn("type"); QTest::newRow(".local") << "from testRelativeImport.m.sm1.go import i1" << "i1" << "int"; QTest::newRow(".init") << "from testRelativeImport.m.sm1.go import i2" << "i2" << "int"; QTest::newRow("..local") << "from testRelativeImport.m.sm1.go import i3" << "i3" << "int"; QTest::newRow("..init") << "from testRelativeImport.m.sm1.go import i4" << "i4" << "int"; QTest::newRow("..sub.local") << "from testRelativeImport.m.sm1.go import i5" << "i5" << "int"; QTest::newRow("..sub.init") << "from testRelativeImport.m.sm1.go import i6" << "i6" << "int"; } void PyDUChainTest::testImportFiles() { QString code = "import testImportFiles\nk = testImportFiles.fromInit()\np = testImportFiles.other.fromOther()"; ReferencedTopDUContext ctx = parse(code.toUtf8()); DUChainReadLocker lock; QVERIFY(ctx); auto k = ctx->findDeclarations(QualifiedIdentifier("k")); auto p = ctx->findDeclarations(QualifiedIdentifier("p")); QCOMPARE(k.size(), 1); QCOMPARE(p.size(), 1); QVERIFY(k.first()->abstractType()); QCOMPARE(k.first()->abstractType()->toString(), QString("fromInit")); QCOMPARE(p.first()->abstractType()->toString(), QString("fromOther")); } void PyDUChainTest::testCrashes() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); QVERIFY(! m_ast->body.isEmpty()); } void PyDUChainTest::testCrashes_data() { QTest::addColumn("code"); QTest::newRow("unicode_char") << "a = \"í\""; QTest::newRow("unicode escape char") << "print(\"\\xe9\")"; QTest::newRow("augassign") << "a = 3\na += 5"; QTest::newRow("delete") << "a = 3\ndel a"; QTest::newRow("double_comprehension") << "q = [[x for x in a] + [x for x in a] for y in b]"; QTest::newRow("for_else") << "for i in range(3): pass\nelse: pass"; QTest::newRow("for_while") << "while i < 4: pass\nelse: pass"; QTest::newRow("ellipsis") << "a[...]"; QTest::newRow("tuple_assign_unknown") << "foo = (unknown, unknown, unknown)"; QTest::newRow("for_assign_unknown") << "for foo, bar, baz in unknown: pass"; QTest::newRow("negative slice index") << "t = (1, 2, 3)\nd = t[-1]"; QTest::newRow("decorator_with_args") << "@foo('bar', 'baz')\ndef myfunc(): pass"; QTest::newRow("non_name_decorator") << "@foo.crazy_decorators\ndef myfunc(): pass"; QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass"; QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)"; QTest::newRow("whatever") << "for attr in updated:\n " " getattr.update"; QTest::newRow("return_outside_function") << "return 3"; QTest::newRow("return_context_outside_function") << "return [x for x in range(3)]"; QTest::newRow("paren_attrib_access") << "a = (xxx or yyy).zzz"; QTest::newRow("func_call") << "a = xxx.func(yyy.zzz)"; QTest::newRow("comprehension_attrib") << "a = [foo for foo in bar].baz"; QTest::newRow("comprehension_attrib2") << "a = [foo.bar for foo in bar]"; QTest::newRow("attrib") << "(sep or ' ').join(xxxx.capitalize() for xxxx in ssss.split(sep))"; QTest::newRow("attrib2") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))"; QTest::newRow("attrib3") << "known_threads = {line.strip()}"; QTest::newRow("attrib4") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}"; QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()"; QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n" ".format(type(meth))) from None"; QTest::newRow("newline_attrib") << "some_instance \\\n" ". attr1 \\\n" ".funcfunc(argarg, arg2arg) \\\n" ".foo"; QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n" "in ambiguous_nucleotide_values.iteritems() \\\n" "if set([codon[0] for codon in codons]).issuperset(set(meanings)))"; QTest::newRow("fancy class range") << "class SchemeLexer(RegexLexer):\n" " valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+'\n" "\n" " tokens = {\n" " 'root' : [\n" " # the comments - always starting with semicolon\n" " # and going to the end of the line\n" " (r';.*$', Comment.Single),\n" "\n" " # whitespaces - usually not relevant\n" " (r'\\s+', Text),\n" "\n" " # numbers\n" " (r'-?\\d+\\.\\d+', Number.Float),\n" " (r'-?\\d+', Number.Integer)\n" " ],\n" " }\n"; QTest::newRow("another fancy range") << "setup_args['data_files'] = [\n" " (os.path.dirname(os.path.join(install_base_dir, pattern)),\n" " [ f for f in glob.glob(pattern) ])\n" " for pattern in patterns\n" "]\n"; QTest::newRow("kwarg_empty_crash") << "def myfun(): return\ncheckme = myfun(kw=something)"; QTest::newRow("stacked_tuple_hang") << "tree = (1,(2,(3,(4,(5,'Foo')))))"; QTest::newRow("stacked_tuple_hang2") << "tree = (257," "(264," "(285," "(259," "(272," "(275," "(1, 'return')))))))"; QTest::newRow("very_large_tuple_hang") << "tree = " "(257," "(264," "(285," "(259," "(1, 'def')," "(1, 'f')," "(260, (7, '('), (8, ')'))," "(11, ':')," "(291," "(4, '')," "(5, '')," "(264," "(265," "(266," "(272," "(275," "(1, 'return')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302, (303, (304, (305, (2, '1'))))))))))))))))))," "(264," "(265," "(266," "(272," "(276," "(1, 'yield')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302," "(303, (304, (305, (2, '1'))))))))))))))))))," "(4, '')))," "(6, '')))))," "(4, '')," "(0, ''))))"; QTest::newRow("attribute_hang") << "s = \"123\"\n" "s = s.replace(u'ł', 'l').\\\n" "replace(u'ó', 'o').\\\n" "replace(u'ą', 'a').\\\n" "replace(u'ę', 'e').\\\n" "replace(u'ś', 's').\\\n" "replace(u'ż', 'z').\\\n" "replace(u'ź', 'z').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo')\n"; QTest::newRow("function context range crash") << "def myfunc(arg):\n foo = 3 + \\\n[x for x in range(20)]"; QTest::newRow("decorator comprehension crash") << "@implementer_only(interfaces.ISSLTransport,\n" " *[i for i in implementedBy(tcp.Client)\n" " if i != interfaces.ITLSTransport])\n" "class Client(tcp.Client):\n" " pass\n"; QTest::newRow("comprehension_as_default_crash") << "def foo(bar = [item for (_, item) in items()]):\n return"; QTest::newRow("try_except") << "try: pass\nexcept: pass"; QTest::newRow("try_except_type") << "try: pass\nexcept FooException: pass"; QTest::newRow("try_except_type_as") << "try: pass\nexcept FooException as bar: pass"; QTest::newRow("import_missing") << "from this_does_not_exist import nor_does_this"; QTest::newRow("list_append_missing") << "foo = []\nfoo.append(missing)"; QTest::newRow("list_append_missing_arg") << "foo = []\nfoo.append()"; QTest::newRow("list_extend_missing") << "foo = []\nfoo.extend(missing)"; QTest::newRow("list_extend_missing_arg") << "foo = []\nfoo.extend()"; QTest::newRow("method_of_call_with_list_arg") << "class MyClass:\n" " def bar(self): pass\n" "def foo(x):\n" " return MyClass()\n" "foo([0]).bar()"; QTest::newRow("unpacked_dict_kwarg") << "def foo(arg): pass\nfoo(**{'arg': 2})"; QTest::newRow("negative_container_hints") << "class Evil:\n" " def aa(self, arg):\n" " \"\"\"! addsTypeOfArgContent ! -1\"\"\"\n" " def bb(self, arg):\n" " \"\"\"! addsTypeOfArg ! -2\"\"\"\n" " def cc(self, arg):\n" " \"\"\"! returnContentEqualsContentOf ! -3\"\"\"\n" "e = Evil()\n" "z = [e.aa(1), e.bb(2), e.cc(3)]"; QTest::newRow("comprehension_in_fstring") << "def crash(): return f'expr={ {x: y for x, y in [(1, 2), ]}}'"; } void PyDUChainTest::testClassVariables() { ReferencedTopDUContext ctx = parse("class c():\n myvar = 3;\n def meth(self):\n print(myvar)"); QVERIFY(ctx.data()); DUChainWriteLocker lock(DUChain::lock()); CursorInRevision relevantPosition(3, 10); DUContext* c = ctx->findContextAt(relevantPosition); QVERIFY(c); int useIndex = c->findUseAt(relevantPosition); if ( useIndex != -1 ) { QVERIFY(useIndex < c->usesCount()); const Use* u = &(c->uses()[useIndex]); QVERIFY(!u->usedDeclaration(c->topContext())); } } void PyDUChainTest::testWarnNewNotCls() { QFETCH(QString, code); QFETCH(int, probs); ReferencedTopDUContext ctx = parse(code); DUChainReadLocker lock; QCOMPARE(ctx->problems().count(), probs); } void PyDUChainTest::testWarnNewNotCls_data() { QTest::addColumn("code"); QTest::addColumn("probs"); QTest::newRow("check_for_new_first_arg_cls") << "class c():\n def __new__(clf, other):\n pass" << 1; QTest::newRow("check_for_new_first_arg_cls_0") << "class c():\n def __new__(cls, other):\n pass" << 0; QTest::newRow("check_first_arg_class_self") << "class c():\n def test(seff, masik):\n pass" << 1; QTest::newRow("check_first_arg_class_self_0") << "class c():\n def test(self, masik):\n pass" << 0; } void PyDUChainTest::testBinaryOperatorsUnsure() { QFETCH(QString, code); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), type); } void PyDUChainTest::testBinaryOperatorsUnsure_data() { QTest::addColumn("code"); QTest::addColumn("type"); QTest::newRow("check_unsure_type_0") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3.5;\ny = 3;\ncheckme = x * y;" << "unsure (float, int)"; QTest::newRow("check_unsure_type_1") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_2") << "class c():\n pass;\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_3") << "class c():\n pass;\nclass d():\n pass;\nx = c();\nx = d();\ny = 3;\ncheckme = x * y;" << "int"; } void PyDUChainTest::testFlickering() { QFETCH(QStringList, code); QFETCH(int, before); QFETCH(int, after); TestFile f(code[0], "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx = f.topContext(); QVERIFY(ctx); DUChainWriteLocker lock(DUChain::lock()); int count = ctx->localDeclarations().size(); qDebug() << "Declaration count before: " << count; QVERIFY(count == before); lock.unlock(); f.setFileContents(code[1]); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ctx = f.topContext(); QVERIFY(ctx); lock.lock(); count = ctx->localDeclarations().size(); qDebug() << "Declaration count afterwards: " << count; QVERIFY(count == after); foreach(Declaration* dec, ctx->localDeclarations()) { qDebug() << dec->toString() << dec->range(); qDebug() << dec->uses().size(); } } void PyDUChainTest::testFlickering_data() { QTest::addColumn("code"); QTest::addColumn("before"); QTest::addColumn("after"); QTest::newRow("declaration_flicker") << ( QStringList() << "a=2\n" << "b=3\na=2\n" ) << 1 << 2; } void PyDUChainTest::testCannotOverwriteBuiltins() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testCannotOverwriteBuiltins_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("list_assign") << "class list(): pass\ncheckme = []\ncheckme.append(3)" << "list of int"; QTest::newRow("str_assign") << "str = 5; checkme = 'Foo'" << "str"; QTest::newRow("str_assign2") << "class Foo: pass\nstr = Foo; checkme = 'Foo'" << "str"; QTest::newRow("str_assign3") << "from testCannotOverwriteBuiltins.i import Foo as str\ncheckme = 'Foo'" << "str"; QTest::newRow("for") << "for str in [1, 2, 3]: pass\ncheckme = 'Foo'" << "str"; QTest::newRow("assert") << "assert isinstance(str, int)\ncheckme = 'Foo'" << "str"; QTest::newRow("assert2") << "assert isinstance(str, int)\ncheckme = 3" << "int"; QTest::newRow("can_have_custom") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open()" << "int"; QTest::newRow("can_have_custom2") << "from testCannotOverwriteBuiltins import mod\ncheckme = open().read()" << "str"; QTest::newRow("can_have_custom3") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open().read()" << "mixed"; } void PyDUChainTest::testVarKWArgs() { ReferencedTopDUContext ctx = parse("def myfun(arg, *vararg, **kwarg):\n pass\n pass"); DUChainWriteLocker lock; QVERIFY(ctx); DUContext* func = ctx->findContextAt(CursorInRevision(1, 0)); QVERIFY(func); QVERIFY(! func->findDeclarations(QualifiedIdentifier("arg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("vararg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("kwarg")).isEmpty()); QVERIFY(func->findDeclarations(QualifiedIdentifier("vararg")).first()->abstractType()->toString().startsWith("tuple")); QCOMPARE(func->findDeclarations(QualifiedIdentifier("kwarg")).first()->abstractType()->toString(), QString("dict of str : unknown")); } void PyDUChainTest::testSimple() { QFETCH(QString, code); QFETCH(int, decls); QFETCH(int, uses); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVector< Declaration* > declarations = ctx->localDeclarations(); QCOMPARE(declarations.size(), decls); int usesCount = 0; foreach(Declaration* d, declarations) { usesCount += d->uses().size(); QVERIFY(d->abstractType()); } QCOMPARE(usesCount, uses); } void PyDUChainTest::testSimple_data() { QTest::addColumn("code"); QTest::addColumn("decls"); QTest::addColumn("uses"); QTest::newRow("assign") << "b = 2;" << 1 << 0; QTest::newRow("assign_str") << "b = 'hola';" << 1 << 0; QTest::newRow("op") << "a = 3; b = a+2;" << 2 << 1; QTest::newRow("bool") << "a = True" << 1 << 0; QTest::newRow("op") << "a = True and True;" << 1 << 0; } class AttributeRangeTestVisitor : public AstDefaultVisitor { public: bool found; KTextEditor::Range searchingForRange; QString searchingForIdentifier; void visitAttribute(AttributeAst* node) override { auto r = KTextEditor::Range(0, node->startCol, 0, node->endCol); qDebug() << "Found attr: " << r << node->attribute->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->attribute->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitAttribute(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found func: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; qDebug() << node->arguments->vararg << node->arguments->kwarg; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } if ( node->arguments->vararg ) { auto r = KTextEditor::Range(0, node->arguments->vararg->startCol, 0, node->arguments->vararg->startCol+node->arguments->vararg->argumentName->value.length()); qDebug() << "Found vararg: " << node->arguments->vararg->argumentName->value << r; if ( r == searchingForRange && node->arguments->vararg->argumentName->value == searchingForIdentifier ) { found = true; return; } } if ( node->arguments->kwarg ) { auto r = KTextEditor::Range(0, node->arguments->kwarg->startCol, 0, node->arguments->kwarg->startCol+node->arguments->kwarg->argumentName->value.length()); qDebug() << "Found kwarg: " << node->arguments->kwarg->argumentName->value << r; if ( r == searchingForRange && node->arguments->kwarg->argumentName->value == searchingForIdentifier ) { found = true; return; } } AstDefaultVisitor::visitFunctionDefinition(node); } void visitClassDefinition(ClassDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found cls: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitClassDefinition(node); } void visitImport(ImportAst* node) override { foreach ( const AliasAst* name, node->names ) { if ( name->name ) { qDebug() << "found import" << name->name->value << name->name->range(); } if ( name->name && name->name->value == searchingForIdentifier && name->name->range() == searchingForRange ) { found = true; return; } if ( name->asName ) { qDebug() << "found import" << name->asName->value << name->asName->range(); } if ( name->asName && name->asName->value == searchingForIdentifier && name->asName->range() == searchingForRange ) { found = true; return; } } } }; void PyDUChainTest::testRanges() { QFETCH(QString, code); QFETCH(int, expected_amount_of_variables); Q_UNUSED(expected_amount_of_variables); QFETCH(QStringList, column_ranges); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); for ( int i = 0; i < column_ranges.length(); i++ ) { int scol = column_ranges.at(i).split(",")[0].toInt(); int ecol = column_ranges.at(i).split(",")[1].toInt(); QString identifier = column_ranges.at(i).split(",")[2]; auto r = KTextEditor::Range(0, scol, 0, ecol); AttributeRangeTestVisitor* visitor = new AttributeRangeTestVisitor(); visitor->searchingForRange = r; visitor->searchingForIdentifier = identifier; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("attr_dot_name_hash", "Insufficiently magic hack", Continue); QCOMPARE(visitor->found, true); delete visitor; } } void PyDUChainTest::testRanges_data() { QTest::addColumn("code"); QTest::addColumn("expected_amount_of_variables"); QTest::addColumn("column_ranges"); QTest::newRow("attr_two_attributes") << "base.attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_binary") << "base.attr + base.attr" << 4 << ( QStringList() << "5,8,attr" << "17,20,attr" ); QTest::newRow("attr_same") << "aaa.aaa.aaa + aaa.aaa.aaa" << 6 << ( QStringList() << "4,6,aaa" << "8,10,aaa" << "18,20,aaa" << "22,24,aaa" ); QTest::newRow("attr_three_attributes") << "base.attr.subattr" << 3 << ( QStringList() << "5,8,attr" << "10,16,subattr" ); QTest::newRow("attr_functionCall") << "base.attr().subattr" << 3 << ( QStringList() << "5,8,attr" << "12,18,subattr" ); QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" ); QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" ); QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" ); QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" << "25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr"); QTest::newRow("attr_two_in_call") << "func(inst.aaa, inst.bbbb)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,bbbb" ); QTest::newRow("attr_two_in_call_same") << "func(inst.aaa, inst.aaaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,aaaa" ); QTest::newRow("attr_two_in_call_same2") << "func(inst.aaa, inst.aaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,22,aaa" ); QTest::newRow("attr_of_string_slash") << "'/'.join(a)" << 1 << ( QStringList() << "4,7,join" ); QTest::newRow("attr_of_string_in_list") << "[\"*{0}*\".format(foo)]" << 1 << ( QStringList() << "9,14,format" ); QTest::newRow("attr_of_call_in_list") << "[foo().format(foo)]" << 1 << ( QStringList() << "7,12,format" ); QTest::newRow("attr_parentheses") << "(\"foo\" + \"foo\").capitalize()" << 1 << ( QStringList() << "16,25,capitalize" ); QTest::newRow("attr_commented_name") << "base.attr # attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_name_in_strings") << "'attr' + base['attr'].attr # attr" << 4 << ( QStringList() << "22,25,attr" ); QTest::newRow("attr_dot_hash_in_strings") << "'.foo#' + base['.#'].attr # attr" << 4 << ( QStringList() << "21,24,attr" ); QTest::newRow("attr_dot_name_hash") << "base['.attr#'].attr" << 4 << ( QStringList() << "15,18,attr" ); QTest::newRow("string_parentheses") << "(\"asdf\".join())" << 1 << ( QStringList() << "8,11,join" ); QTest::newRow("string_parentheses2") << "(\"asdf\").join()" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("string_parentheses3") << "(\"asdf\".join()).join()" << 2 << ( QStringList() << "8,11,join" << "16,19,join" ); QTest::newRow("string_parentheses4") << "(\"asdf\".join()+2).join()" << 2 << ( QStringList() << "8,11,join" << "18,21,join" ); QTest::newRow("string_parentheses_call") << "f(\"asdf\".join())" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" ); QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" ); QTest::newRow("classdef_range") << "class cls(): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance") << "class cls(parent1, parent2): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance_spaces") << "class cls( parent1, parent2 ):pass" << 1 << ( QStringList() << "12,14,cls" ); QTest::newRow("vararg_kwarg") << "def func(*vararg, **kwargs): pass" << 2 << ( QStringList() << "10,16,vararg" << "20,26,kwargs" ); QTest::newRow("import") << "import sys" << 1 << ( QStringList() << "7,10,sys" ); QTest::newRow("import2") << "import i.localvar1" << 1 << ( QStringList() << "7,18,i.localvar1" ); QTest::newRow("import3") << "import sys as a" << 1 << ( QStringList() << "13,14,a" ); } class TypeTestVisitor : public AstDefaultVisitor { public: QString searchingForType; TopDUContextPointer ctx; bool found; void visitName(NameAst* node) override { if ( node->identifier->value != "checkme" ) return; QList decls = ctx->findDeclarations(QualifiedIdentifier(node->identifier->value)); if ( ! decls.length() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration found for " << node->identifier->value; return; } Declaration* d = decls.last(); QVERIFY(d->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << "found: " << node->identifier->value << "is" << d->abstractType()->toString() << "should be" << searchingForType; if ( d->abstractType()->toString().replace("__kdevpythondocumentation_builtin_", "").startsWith(searchingForType) ) { found = true; return; } }; }; void PyDUChainTest::testTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("return_builtin_iterator", "fake builtin iter()", Continue); QEXPECT_FAIL("parent_constructor_arg_type", "Not enough passes?", Continue); QEXPECT_FAIL("init_class_no_decl", "aliasing info lost", Continue); QCOMPARE(visitor->found, true); } void PyDUChainTest::testTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); - + + QTest::newRow("annotate_decl") << "checkme: int" << "int"; + QTest::newRow("annotate_assign") << "checkme: int = 3.5" << "unsure (float, int)"; QTest::newRow("listtype") << "checkme = []" << "list"; QTest::newRow("listtype_func") << "checkme = list()" << "list"; QTest::newRow("listtype_with_contents") << "checkme = [1, 2, 3, 4, 5]" << "list of int"; QTest::newRow("listtype_extended") << "some_misc_var = []; checkme = some_misc_var" << "list"; QTest::newRow("dicttype") << "checkme = {}" << "dict"; QTest::newRow("dicttype_get") << "d = {0.4:5}; checkme = d.get(0)" << "int"; QTest::newRow("dicttype_func") << "checkme = dict()" << "dict"; QTest::newRow("dicttype_extended") << "some_misc_var = {}; checkme = some_misc_var" << "dict"; QTest::newRow("bool") << "checkme = True" << "bool"; QTest::newRow("float") << "checkme = 3.7" << "float"; QTest::newRow("int") << "checkme = 3" << "int"; QTest::newRow("str") << "checkme = \"foo\"" << "str"; QTest::newRow("bytes") << "checkme = b\"foo\"" << "bytes"; QTest::newRow("function_arg_scope") << "class Foo:\n" " a = 3\n" " def func(self, x=a):\n" " return x\n" "f = Foo()\n" "checkme = f.func()" << "int"; QTest::newRow("with") << "with open('foo') as f: checkme = f.read()" << "str"; QTest::newRow("arg_after_vararg") << "def func(x, y, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("arg_after_vararg_with_default") << "def func(x=5, y=3, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("class_scope_end_inside") << "a = str()\nclass M:\n" " a = 2\n foo = a\n" "checkme = M().foo" << "int"; QTest::newRow("class_scope_end_outside") << "a = str()\nclass M:\n a = 2\ncheckme = a" << "str"; QTest::newRow("list_access_right_open_slice") << "some_list = []; checkme = some_list[2:]" << "list"; QTest::newRow("list_access_left_open_slice") << "some_list = []; checkme = some_list[:2]" << "list"; QTest::newRow("list_access_closed_slice") << "some_list = []; checkme = some_list[2:17]" << "list"; QTest::newRow("list_access_step") << "some_list = []; checkme = some_list[::2]" << "list"; QTest::newRow("list_access_singleItem") << "some_list = []; checkme = some_list[42]" << "mixed"; QTest::newRow("funccall_number") << "def foo(): return 3; \ncheckme = foo();" << "int"; QTest::newRow("funccall_string") << "def foo(): return 'a'; \ncheckme = foo();" << "str"; QTest::newRow("funccall_list") << "def foo(): return []; \ncheckme = foo();" << "list"; QTest::newRow("funccall_dict") << "def foo(): return {}; \ncheckme = foo();" << "dict"; QTest::newRow("tuple1") << "checkme, foo = 3, \"str\"" << "int"; QTest::newRow("tuple2") << "foo, checkme = 3, \"str\"" << "str"; QTest::newRow("tuple2_negative_index") << "foo = (1, 2, 'foo')\ncheckme = foo[-1]" << "str"; QTest::newRow("tuple_type") << "checkme = 1, 2" << "tuple"; QTest::newRow("tuple_rhs_unpack") << "foo = 1, 2.5\nbar = 1, *foo, 2\ncheckme = bar[2]" << "float"; QTest::newRow("dict_iteritems") << "d = {1:2, 3:4}\nfor checkme, k in d.iteritems(): pass" << "int"; QTest::newRow("enumerate_key") << "d = [str(), str()]\nfor checkme, value in enumerate(d): pass" << "int"; QTest::newRow("enumerate_value") << "d = [str(), str()]\nfor key, checkme in enumerate(d): pass" << "str"; QTest::newRow("dict_enumerate") << "d = {1:2, 3:4}\nfor key, checkme in enumerate(d.values()): pass" << "int"; QTest::newRow("dict_assign_twice") << "d = dict(); d[''] = 0; d = dict(); d[''] = 0; checkme = d" << "unsure (dict of str : int, dict)"; QTest::newRow("class_method_import") << "class c:\n attr = \"foo\"\n def m():\n return attr;\n return 3;\ni=c()\ncheckme=i.m()" << "int"; QTest::newRow("getsListDecorator") << "foo = [1, 2, 3]\ncheckme = foo.reverse()" << "list of int"; QTest::newRow("str_iter") << "checkme = [char for char in 'Hello, world!']" << "list of str"; QTest::newRow("str_subscript") << "checkme = 'Hello, world!'[0]" << "str"; QTest::newRow("bytes_iter") << "checkme = [byte for byte in b'Hello, world!']" << "list of int"; QTest::newRow("bytes_subscript") << "checkme = b'Hello, world!'[0]" << "int"; QTest::newRow("fromAssertIsinstance") << "class c(): pass\ncheckme = mixed()\nassert isinstance(checkme, c)\n" << "c"; QTest::newRow("fromAssertIsinstanceInvalid") << "class c(): pass\ncheckme = mixed()\nassert isinstance(c, checkme)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid2") << "class c(): pass\ncheckme = mixed()\nassert isinstance(D, c)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid3") << "checkme = int()\nassert isinstance(checkme, X)\n" << "int"; QTest::newRow("fromAssertIsinstanceInvalid4") << "checkme = int()\nassert isinstance(checkme)\n" << "int"; QTest::newRow("fromAssertType") << "class c(): pass\ncheckme = mixed()\nassert type(checkme) == c\n" << "c"; QTest::newRow("fromIfType") << "class c(): pass\ncheckme = mixed()\nif type(checkme) == c: pass\n" << "c"; QTest::newRow("fromIfIsinstance") << "class c(): pass\ncheckme = mixed()\nif isinstance(checkme, c): pass\n" << "c"; QTest::newRow("diff_local_classattr") << "class c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; QTest::newRow("diff_local_classattr2") << "local=3\nclass c(): attr = 1\ninst=c()\ncheckme = c.local" << "mixed"; QTest::newRow("diff_local_classattr3") << "attr=3.5\nclass c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; // QTest::newRow("class_method_self") << "class c:\n def func(checkme, arg, arg2):\n pass\n" << "c"; // QTest::newRow("funccall_dict") << "def foo(): return foo; checkme = foo();" << (uint) IntegralType::TypeFunction; QTest::newRow("tuple_simple") << "mytuple = 3, 5.5\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_simple2") << "mytuple = 3, 5.5\nfoobar, checkme = mytuple" << "float"; QTest::newRow("tuple_simple3") << "mytuple = 3, 5.5, \"str\", 3, \"str\"\na, b, c, d, checkme = mytuple" << "str"; QTest::newRow("tuple_single") << "checkme = 4," << "tuple"; QTest::newRow("tuple_single2") << "checkme, = 4," << "int"; QTest::newRow("tuple_single3") << "mytuple = 4,\ncheckme, = mytuple" << "int"; QTest::newRow("tuple_ext_unpack") << "mytuple = 3, 5.5\nfoobar, *starred, checkme = mytuple" << "float"; QTest::newRow("tuple_ext_unpack2") << "mytuple = 3, 5.5\nfoobar, *checkme, another = mytuple" << "list"; QTest::newRow("tuple_ext_unpack3") << "mytuple = 3, 5.5\nfoobar, *checkme = mytuple" << "list of float"; QTest::newRow("tuple_ext_unpack4") << "mytuple = 3, 5.5\n*checkme, = mytuple" << "list of unsure (int, float)"; QTest::newRow("tuple_nested") << "mytuple = 3, ('foo', 5.5)\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_nested2") << "mytuple = 3, ('foo', 5.5)\nfoobar, (checkme, other) = mytuple" << "str"; QTest::newRow("tuple_nested3") << "mytuple = ((7, 'foo'), 5.5), 3\n((baz, checkme), other), foo = mytuple" << "str"; QTest::newRow("tuple_nested_ext") << "mytuple = (2, ('foo', 'bar', 6), 7)\na, (b, *checkme, c), *d = mytuple" << "list of str"; QTest::newRow("tuple_multi_assign") << "mytuple = 2, 'foo'\ncheckme = a = mytuple" << "tuple"; QTest::newRow("tuple_multi_assign2") << "mytuple = 2, 'foo'\ncheckme, a = b = mytuple" << "int"; QTest::newRow("list_unpack") << "mylist = [1, 2, 3]\ncheckme, b, c = mylist" << "int"; QTest::newRow("list_unpack2") << "mylist = [1, 'x', 3]\ncheckme, b, c = mylist" << "unsure (int, str)"; QTest::newRow("list_ext_unpack") << "mylist = [1, 2, 3]\n*checkme, foo = mylist" << "list of int"; QTest::newRow("list_ext_unpack2") << "mylist = [1, 'x', 3]\n*checkme, foo = mylist" << "list of unsure (int, str)"; QTest::newRow("if_expr_sure") << "checkme = 3 if 7 > 9 else 5" << "int"; QTest::newRow("unary_op") << "checkme = -42" << "int"; QTest::newRow("tuple_funcret") << "def myfun(): return 3, 5\ncheckme, a = myfun()" << "int"; QTest::newRow("tuple_funcret2") << "def myfun():\n t = 3, 5\n return t\ncheckme, a = myfun()" << "int"; QTest::newRow("yield") << "def myfun():\n yield 3\ncheckme = myfun()" << "list of int"; QTest::newRow("yield_twice") << "def myfun():\n yield 3\n yield 'foo'\ncheckme = myfun()" << "list of unsure (int, str)"; // this is mostly a check that it doesn't crash QTest::newRow("yield_return") << "def myfun():\n return 3\n yield 'foo'\ncheckme = myfun()" << "unsure (int, list of str)"; QTest::newRow("lambda") << "x = lambda t: 3\ncheckme = x()" << "int"; QTest::newRow("lambda_failure") << "x = lambda t: 3\ncheckme = t" << "mixed"; QTest::newRow("function_arg_tuple") << "def func(*arg):\n foo, bar = arg\n return bar\ncheckme = func(3, 5)" << "int"; QTest::newRow("function_arg_tuple2") << "def func(*arg):\n return arg[-1]\ncheckme = func(3, \"Foo\")" << "str"; QTest::newRow("tuple_indexaccess") << "t = 3, 5.5\ncheckme = t[0]" << "int"; QTest::newRow("tuple_indexaccess2") << "t = 3, 5.5\ncheckme = t[1]" << "float"; QTest::newRow("tuple_indexaccess3") << "t = 3, 4\ncheckme = t[1]" << "int"; QTest::newRow("dict_unsure") << "t = dict(); t = {3: str()}\ncheckme = t[1].capitalize()" << "str"; QTest::newRow("unsure_attr_access") << "d = str(); d = 3; checkme = d.capitalize()" << "str"; QTest::newRow("class_create_var") << "class c: pass\nd = c()\nd.foo = 3\ncheckme = d.foo" << "int"; QTest::newRow("tuple_loop") << "t = [(1, \"str\")]\nfor checkme, a in t: pass" << "int"; QTest::newRow("no_hints_type") << "def myfun(arg): arg = 3; return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("hints_type") << "def myfun(arg): return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("args_type") << "def myfun(*args): return args[0]\ncheckme = myfun(3)" << "int"; QTest::newRow("kwarg_type") << "def myfun(**kwargs): return kwargs['a']\ncheckme = myfun(a=3)" << "int"; QTest::newRow("dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\ncheckme = foo(**{'a': 12})" << "int"; QTest::newRow("dict_norm_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2)" << "unsure (int, float)"; QTest::newRow("multi_dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2, **{'c': ''})" << "unsure (int, float, str)"; QTest::newRow("named_arg_type") << "def myfun(arg): return arg\ncheckme = myfun(arg=3)" << "int"; QTest::newRow("arg_args_type") << "def myfun(arg, *args): return args[0]\n" "checkme = myfun(3, str())" << "str"; QTest::newRow("arg_kwargs_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(12, a=str())" << "str"; QTest::newRow("named_kwargs_type_1") << "def myfun(arg, **kwargs): return arg\n" "checkme = myfun(arg=12, a=str())" << "int"; QTest::newRow("named_kwargs_type_2") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(arg=12, a=str())" << "str"; QTest::newRow("kwargs_named_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(a=str(), arg=12)" << "str"; QTest::newRow("varied_args_type_1") << "def myfun(arg, *args, **kwargs): return arg\n" "checkme = myfun(1, 1.5, a=str())" << "int"; QTest::newRow("varied_args_type_2") << "def myfun(arg, *args, **kwargs): return args[0]\n" "checkme = myfun(1, 1.5, a=str())" << "float"; QTest::newRow("varied_args_type_3") << "def myfun(arg, *args, **kwargs): return kwargs['a']\n" "checkme = myfun(1, 1.5, a=str())" << "str"; QTest::newRow("method_args_type_1") << "class MyClass:\n" " def method(self, arg): return self\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("method_args_type_2") << "class MyClass:\n" " def method(self, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("clsmethod_args_type_1") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return cls\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("clsmethod_args_type_2") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_args_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_vararg_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg, *args): return args[0]\n" "checkme = MyClass().method(12, 2.5)" << "float"; QTest::newRow("method_explicit_self") << "class MyClass:\n" " def method(self, arg): return arg\n" "instance = MyClass()\n" "checkme = MyClass.method(instance, 12)" << "int"; QTest::newRow("method_vararg_explicit_self") << "class MyClass:\n" " def foo(self, arg, *args): return args[0]\n" "mc = MyClass()\n" "checkme = MyClass.foo(mc, 'str', 3, 4.5)" << "int"; QTest::newRow("clsmethod_explicit_self") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg1, arg2): return arg2\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "int"; QTest::newRow("staticmethod_explicit_self") << "class MyClass:\n" " @staticmethod\n" " def method(arg1, arg2): return arg1\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "str"; QTest::newRow("parent_constructor_arg_type") << "class Base:\n" // https://bugs.kde.org/show_bug.cgi?id=369364 " def __init__(self, foo):\n" " self.foo = foo\n" "class Derived(Base):\n" " def __init__(self, foo):\n" " Base.__init__(self, foo)\n" "instance = Derived('string')\n" "checkme = instance.foo" << "str"; QTest::newRow("tuple_unsure") << "q = (3, str())\nq=(str(), 3)\ncheckme, _ = q" << "unsure (int, str)"; QTest::newRow("custom_iterable") << "class Gen2:\n" " def __iter__(self): return self\n" " def __next__(self): return 'blah'\n" "for checkme in Gen2(): pass" << "str"; QTest::newRow("separate_iterator") << "class Foo:\n" " def __iter__(self): return Bar()\n" " def __next__(self): return 'blah'\n" // Not used (or shouldn't be!) "class Bar:\n" " def __next__(self): return {1}\n" "checkme = [a for a in Foo()]" << "list of set of int"; QTest::newRow("return_builtin_iterator") << "class Gen2:\n" " contents = [1, 2, 3]\n" " def __iter__(self): return iter(Gen2.contents)\n" "for checkme in Gen2(): pass" << "int"; QTest::newRow("init_class") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "checkme = Foo()\n" << "Foo"; QTest::newRow("init_class_no_decl") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "a = [Foo]\n" "checkme = a[0]()\n" << "Foo"; QTest::newRow("call_class") << "class Foo:\n" " def __call__(self):\n" " return 0\n" "f = Foo()\n" "checkme = f()\n" << "int"; QTest::newRow("call_class_no_decl") << "class Foo:\n" " def __call__(self): return 1.5\n" "a = [Foo()]\n" "checkme = a[0]()" << "float"; QTest::newRow("classmethod") << "class Foo:\n" " @classmethod\n" " def foo(cls):\n" " k = cls()\n" " return k\n" "f = Foo.foo()\n" "checkme = f\n" << "Foo"; QTest::newRow("tuple_listof") << "l = [(1, 2), (3, 4)]\ncheckme = l[1][0]" << "int"; QTest::newRow("getitem") << "class c:\n def __getitem__(self, slice): return 3.14\na = c()\ncheckme = a[2]" << "float"; QTest::newRow("constructor_type_deduction") << "class myclass:\n" "\tdef __init__(self, param): self.foo=param\n" "checkme = myclass(3).foo" << "int"; QTest::newRow("simpe_type_deduction") << "def myfunc(arg): return arg\n" "checkme = myfunc(3)" << "int"; QTest::newRow("functionCall_functionArg_part1") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "checkme = f1()" << "str"; QTest::newRow("functionCall_functionArg_part2") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = identity(getstr)\n" "checkme = f1()\n" << "str"; QTest::newRow("functionCall_functionArg_full") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "f2 = identity(getstr)\n" "a = getstr()\n" "b = f1()\n" "checkme = f2()\n" << "str"; QTest::newRow("vararg_before_other_args") << "def myfun(a, b, *z, x): return z[0]\n" "checkme = myfun(False, False, 1, x = False)" << "int"; QTest::newRow("vararg_before_other_args2") << "def myfun(a, b, *z, x): return z[3]\n" "checkme = myfun(False, False, 1, 2, 3, \"str\", x = False)" << "str"; QTest::newRow("vararg_constructor") << "class myclass():\n" " def __init__(self, *arg): self.prop = arg[0]\n" "obj = myclass(3, 5); checkme = obj.prop" << "int"; QTest::newRow("global_variable") << "a = 3\n" "def f1():\n" " global a\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_variable2") << "a = 3\n" "def f1():\n" " global a\n" " a = \"str\"\n" " return a\n" "checkme = f1()\n" << "str"; QTest::newRow("global_scope_variable") << "a = 3\n" "def f1():\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_no_toplevel_dec") << "def f1():\n" " global a\n a = 3\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("top_level_vs_class_member") << "var = 3\n" "class myclass:\n" " def __init__(self): self.var = \"str\"\n" " def f1(): return var\n" "checkme = myclass.f1()" << "int"; } typedef QPair pair; void PyDUChainTest::testImportDeclarations() { QFETCH(QString, code); QFETCH(QStringList, expectedDecls); QFETCH(bool, shouldBeAliased); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); foreach ( const QString& expected, expectedDecls ) { bool found = false; QString name = expected; QList decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); qCDebug(KDEV_PYTHON_DUCHAIN) << "FOUND DECLARATIONS:"; foreach ( const pair& current, decls ) { qCDebug(KDEV_PYTHON_DUCHAIN) << current.first->toString() << current.first->identifier().identifier().byteArray() << name; } foreach ( const pair& current, decls ) { if ( ! ( current.first->identifier().identifier().byteArray() == name ) ) continue; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found: " << current.first->toString() << " for " << name; AliasDeclaration* isAliased = dynamic_cast(current.first); if ( isAliased && shouldBeAliased ) { found = true; // TODO fixme } else if ( ! isAliased && ! shouldBeAliased ) { found = true; } } QVERIFY(found); } } void PyDUChainTest::testProblemCount() { QFETCH(QString, code); QFETCH(int, problemsCount); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QCOMPARE(ctx->problems().size(), problemsCount); } void PyDUChainTest::testProblemCount_data() { QTest::addColumn("code"); QTest::addColumn("problemsCount"); QTest::newRow("list_comp") << "[foo for foo in range(3)]" << 0; QTest::newRow("list_comp_wrong") << "[bar for foo in range(3)]" << 1; QTest::newRow("list_comp_staticmethod") << "class A:\n @staticmethod\n def func(cls):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_decorator") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_wrong") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("list_comp_staticmethod_wrong") << "class A:\n @staticmethod\n def func(cls):\n" " [x for a in [1, 2, 3]]" << 1; } void PyDUChainTest::testImportDeclarations_data() { QTest::addColumn("code"); QTest::addColumn("expectedDecls"); QTest::addColumn("shouldBeAliased"); QTest::newRow("from_import") << "from testImportDeclarations.i import checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("import") << "import testImportDeclarations.i" << ( QStringList() << "testImportDeclarations" ) << false; QTest::newRow("import_as") << "import testImportDeclarations.i as checkme" << ( QStringList() << "checkme" ) << false; QTest::newRow("from_import_as") << "from testImportDeclarations.i import checkme as checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("from_import_missing") << "from testImportDeclarations.i import missing as checkme" << ( QStringList() ) << true; } typedef QPair p; void PyDUChainTest::testAutocompletionFlickering() { TestFile f("foo = 3\nfoo2 = 2\nfo", "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx1 = f.topContext(); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx1); QList

decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); QList declIds; foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); f.setFileContents("foo = 3\nfoo2 = 2\nfoo"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx2 = f.topContext(); QVERIFY(ctx2); lock.lock(); QList

decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@1: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); qDebug() << "========================="; TestFile g("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfo", "py"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx1 = g.topContext(); lock.lock(); QVERIFY(ctx1); decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); declIds.clear(); foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); g.setFileContents("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfoo"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx2 = g.topContext(); QVERIFY(ctx2); lock.lock(); decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@2: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); } void PyDUChainTest::testFunctionHints() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testFunctionHints_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("func_return_type") << "def myfun(arg) -> int: pass\ncheckme = myfun(\"3\")" << "unsure (void, int)"; QTest::newRow("argument_type") << "def myfun(arg : int): return arg\ncheckme = myfun(foobar)" << "int"; QTest::newRow("argument_type_only_if_typeof") << "def myfun(arg : 3): return arg\ncheckme = myfun(foobar)" << "mixed"; } void PyDUChainTest::testHintedTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testHintedTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("simple_hint") << "def myfunc(x): return x\ncheckme = myfunc(3)" << "int"; QTest::newRow("hint_unsure") << "def myfunc(x): return x\nmyfunc(3.5)\ncheckme = myfunc(3)" << "unsure (float, int)"; QTest::newRow("unsure_attribute") << "def myfunc(x): return x.capitalize()\nmyfunc(3.5)\ncheckme = myfunc(str())" << "str"; } void PyDUChainTest::testDecorators() { QFETCH(QString, code); // QFETCH(int, amountOfDecorators); QFETCH(QStringList, names); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); foreach ( const QString& decoratorName, names ) { QVERIFY(Helper::findDecoratorByName(decl, decoratorName)); } } void PyDUChainTest::testDecorators_data() { QTest::addColumn("code"); QTest::addColumn("amountOfDecorators"); QTest::addColumn("names"); QTest::newRow("one_decorator") << "@foo\ndef func(): pass" << 1 << ( QStringList() << "foo" ); QTest::newRow("decorator_with_args") << "@foo(2, \"bar\")\ndef func(): pass" << 1 << ( QStringList() << "foo"); QTest::newRow("two_decorators") << "@foo\n@bar(17)\ndef func(): pass" << 2 << ( QStringList() << "foo" << "bar" ); } void PyDUChainTest::testOperators() { QFETCH(QString, code); QFETCH(QString, expectedType); code.prepend("from testOperators.example import *\n\n"); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QVERIFY(visitor->found); } void PyDUChainTest::testOperators_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("add") << "checkme = Example() + Example()" << "Add"; QTest::newRow("sub") << "checkme = Example() - Example()" << "Sub"; QTest::newRow("mul") << "checkme = Example() * Example()" << "Mul"; QTest::newRow("floordiv") << "checkme = Example() // Example()" << "Floordiv"; QTest::newRow("mod") << "checkme = Example() % Example()" << "Mod"; QTest::newRow("pow") << "checkme = Example() ** Example()" << "Pow"; QTest::newRow("lshift") << "checkme = Example() << Example()" << "Lshift"; QTest::newRow("rshift") << "checkme = Example() >> Example()" << "Rshift"; QTest::newRow("and") << "checkme = Example() & Example()" << "And"; QTest::newRow("xor") << "checkme = Example() ^ Example()" << "Xor"; QTest::newRow("or") << "checkme = Example() | Example()" << "Or"; } void PyDUChainTest::testFunctionArgs() { ReferencedTopDUContext ctx = parse("def ASDF(arg1, arg2):\n" " arg1 = arg2"); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVERIFY(m_ast); // dumpDUContext(ctx); QCOMPARE(ctx->childContexts().size(), 2); DUContext* funcArgCtx = ctx->childContexts().first(); QCOMPARE(funcArgCtx->type(), DUContext::Function); QCOMPARE(funcArgCtx->localDeclarations().size(), 2); QVERIFY(!funcArgCtx->owner()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); QCOMPARE(decl->type()->arguments().length(), 2); qDebug() << decl->type()->arguments().length() << 2; DUContext* funcBodyCtx = ctx->childContexts().last(); QCOMPARE(funcBodyCtx->type(), DUContext::Other); QVERIFY(funcBodyCtx->owner()); QVERIFY(funcBodyCtx->localDeclarations().isEmpty()); } void PyDUChainTest::testInheritance() { QFETCH(QString, code); QFETCH(int, expectedBaseClasses); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList

decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); bool found = false; bool classDeclFound = false; foreach ( const p& item, decls ) { if ( item.first->identifier().toString() == "B" ) { auto klass = dynamic_cast(item.first); QVERIFY(klass); QCOMPARE(klass->baseClassesSize(), static_cast(expectedBaseClasses)); classDeclFound = true; } if ( item.first->identifier().toString() == "checkme" ) { QCOMPARE(item.first->abstractType()->toString(), QString("int")); found = true; } } QVERIFY(found); QVERIFY(classDeclFound); } void PyDUChainTest::testInheritance_data() { QTest::addColumn("code"); QTest::addColumn("expectedBaseClasses"); QTest::newRow("simple") << "class A():\n\tattr = 3\n\nclass B(A):\n\tpass\n\ninst=B()\ncheckme = inst.attr" << 1; QTest::newRow("context_import_prereq") << "import testInheritance.i\ninst=testInheritance.i.testclass()\n" "checkme = inst.attr\nclass B(): pass" << 1; // 1 because object QTest::newRow("context_import") << "import testInheritance.i\n\nclass B(testInheritance.i.testclass):\n" "\ti = 4\n\ninst=B()\ncheckme = inst.attr" << 1; } void PyDUChainTest::testClassContextRanges() { QString code = "class my_class():\n pass\n \n \n \n \n"; ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; DUContext* classContext = ctx->findContextAt(CursorInRevision(5, 0)); QVERIFY(classContext); QVERIFY(classContext->type() == DUContext::Class); } void PyDUChainTest::testContainerTypes() { QFETCH(QString, code); QFETCH(QString, contenttype); QFETCH(bool, use_type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList decls = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(decls.length() > 0); QVERIFY(decls.first()->abstractType()); if ( ! use_type ) { auto type = ListType::Ptr::dynamicCast(decls.first()->abstractType()); QVERIFY(type); QVERIFY(type->contentType()); QCOMPARE(type->contentType().abstractType()->toString(), contenttype); } else { QVERIFY(decls.first()->abstractType()); QEXPECT_FAIL("dict_of_int_call", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("dict_from_tuples", "returnContentEqualsContentOf isn't suitable", Continue); QCOMPARE(decls.first()->abstractType()->toString(), contenttype); } } void PyDUChainTest::testContainerTypes_data() { QTest::addColumn("code"); QTest::addColumn("contenttype"); QTest::addColumn("use_type"); QTest::newRow("list_of_int") << "checkme = [1, 2, 3]" << "int" << false; QTest::newRow("list_from_unpacked") << "foo = [1.3]\ncheckme = [1, *foo, 3]" << "unsure (int, float)" << false; QTest::newRow("list_of_int_call") << "checkme = list([1, 2, 3])" << "int" << false; QTest::newRow("list_from_tuple") << "checkme = list((1, 2, 3))" << "int" << false; QTest::newRow("list_from_dict") << "checkme = list({'a':1, 'b':2})" << "str" << false; // Gets key type! QTest::newRow("list_from_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = list(MyClass())" << "float" << false; QTest::newRow("generator") << "checkme = [i for i in [1, 2, 3]]" << "int" << false; QTest::newRow("list_access") << "list = [1, 2, 3]\ncheckme = list[0]" << "int" << true; QTest::newRow("set_of_int") << "checkme = {1, 2, 3}" << "int" << false; QTest::newRow("set_from_unpacked") << "foo = [1.3]\ncheckme = {1, *foo, 3}" << "unsure (int, float)" << false; QTest::newRow("set_of_int_call") << "checkme = set({1, 2, 3})" << "int" << false; QTest::newRow("set_from_tuple") << "checkme = set((1, 2, 3))" << "int" << false; QTest::newRow("set_generator") << "checkme = {i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_of_str_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; QTest::newRow("frozenset_of_int_call") << "checkme = frozenset({1, 2, 3})" << "int" << false; QTest::newRow("dict_of_int") << "checkme = {a:1, b:2, c:3}" << "int" << false; QTest::newRow("dict_from_unpacked") << "checkme = {**{'a': 1}}" << "dict of str : int" << true; QTest::newRow("dict_from_varied") << "checkme = {**{'a': 1}, 1: 1.5}" << "dict of unsure (str, int) : unsure (int, float)" << true; QTest::newRow("dict_of_int_call") << "checkme = dict({'a':1, 'b':2, 'c':3})" << "dict of str : int" << true; QTest::newRow("dict_from_tuples") << "checkme = dict([('a', 1), ('b', 2)])" << "dict of str : int" << true; QTest::newRow("dict_generator") << "checkme = {\"Foo\":i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_access") << "list = {'a':1, 'b':2, 'c':3}\ncheckme = list[0]" << "int" << true; QTest::newRow("generator_attribute") << "checkme = [item.capitalize() for item in ['foobar']]" << "str" << false; QTest::newRow("cannot_change_type") << "checkme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("cannot_change_type2") << "[1, 2, 3].append(5)\ncheckme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("list_append") << "d = []\nd.append(3)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend") << "d = []; q = [int()]\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_tuple") << "d = []; q = (1, 2)\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = []\ncheckme.extend(MyClass())" << "float" << false; QTest::newRow("for_loop") << "d = [3]\nfor item in d:\n checkme = item" << "int" << true; QTest::newRow("for_loop_unsure") << "d = [3, \"foo\"]\nfor item in d:\n checkme = item" << "unsure (int, str)" << true; QTest::newRow("for_loop_tuple_1") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = a" << "int" << true; QTest::newRow("for_loop_tuple_2") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = b" << "float" << true; QTest::newRow("for_loop_tuple_unsure") << "d = [(3, 3.5), (3.5, 3)]\nfor a, b in d:\n checkme = b" << "unsure (float, int)" << true; // Proposed by Nicolás Alvarez; why not? https://bugs.kde.org/show_bug.cgi?id=359915 QTest::newRow("comprehension_messy") << "users = {'a':19, 'b':42, 'c':35}\n" "sorted_list = sorted(users.items(), key=lambda kv: (-kv[1], kv[0]))\n" "checkme = [k for r,(k,v) in enumerate(sorted_list, 1)]" << "list of str" << true; QTest::newRow("comprehension_multiline") << "checkme = [a for\n a in \n (1, 2)]" << "list of int" << true; // From https://bugs.kde.org/show_bug.cgi?id=359912 QTest::newRow("subscript_multi") << "class Middle:\n def __getitem__(self, key):\n return str()\n" "class Outer:\n def __getitem__(self, key):\n return Middle()\n" "aaa = Outer()\ncheckme = aaa[0][0]" << "str" << true; QTest::newRow("subscript_func_call") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "def bar():\n return Foo()\n" "checkme = bar()[0]" << "str" << true; QTest::newRow("subscript_unknown_index") << "a = 1,str()\ncheckme = a[5-4]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure") << "a = 1,2\na=[str()]\ncheckme = a[0]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure_getitem") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "class Bar:\n def __getitem__(self, key):\n return float()\n" "a = Foo()\na=Bar()\na=[1,2]\ncheckme = a[1]" << "unsure (str, float, int)" << true; } void PyDUChainTest::testVariableCreation() { QFETCH(QString, code); QFETCH(QStringList, expected_local_declarations); QFETCH(QStringList, expected_types); ReferencedTopDUContext top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto localDecls = top->localDeclarations(); QVector localDeclNames; for ( const Declaration* d: localDecls ) { localDeclNames.append(d->identifier().toString()); } Q_ASSERT(expected_types.size() == expected_local_declarations.size()); int offset = 0; for ( const QString& expected : expected_local_declarations ) { int index = localDeclNames.indexOf(expected); QVERIFY(index != -1); QVERIFY(localDecls[index]->abstractType()); QCOMPARE(localDecls[index]->abstractType()->toString(), expected_types[offset]); offset++; } } void PyDUChainTest::testVariableCreation_data() { QTest::addColumn("code"); QTest::addColumn("expected_local_declarations"); QTest::addColumn("expected_types"); QTest::newRow("simple") << "a = 3" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("tuple_wrong") << "a, b = 3" << QStringList{"a", "b"} << QStringList{"mixed", "mixed"}; QTest::newRow("tuple_unpack_inplace") << "a, b = 3, 5.5" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_indirect") << "c = 3, 3.5\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_stacked_inplace") << "a, (b, c) = 1, (2, 3.5)" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("tuple_unpack_stacked_indirect") << "d = 3.5, (3, 1)\na, (b, c) = d" << QStringList{"a", "b", "c"} << QStringList{"float", "int", "int"}; QTest::newRow("unpack_from_list_inplace") << "a, b = [1, 2, 3]" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_from_list_indirect") << "c = [1, 2, 3]\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_custom_iterable") << "class Foo:\n" " def __iter__(self): return self\n" " def __next__(self): return 1.5\n" "a, *b = Foo()" << QStringList{"a", "b"} << QStringList {"float", "list of float"}; QTest::newRow("for_loop_simple") << "for i in range(3): pass" << QStringList{"i"} << QStringList{"int"}; QTest::newRow("for_loop_unpack") << "for a, b in [(3, 5.1)]: pass" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("for_loop_stacked") << "for a, (b, c) in [(1, (2, 3.5))]: pass" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("for_loop_tuple") << "for a in 1, 2: pass" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("for_loop_dict") << "for a in {'foo': 1}: pass" << QStringList{"a"} << QStringList{"str"}; } void PyDUChainTest::testCleanupMultiplePasses() { for ( int j = 0; j < 20; j++ ) { ReferencedTopDUContext top = parse("from testCleanupMultiplePasses import foo\ndef fonc(): return 3+2j\nfoo.foo.func = fonc"); } } void PyDUChainTest::testManyDeclarations() { ReferencedTopDUContext top = parse("from testManyDeclarations import test\nk=test.Foo()"); } void PyDUChainTest::testComments() { QFETCH(QString, code); auto top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto decls = top->findDeclarations(QualifiedIdentifier("a")); QCOMPARE(decls.size(), 1); auto a = decls.first(); QCOMPARE(a->comment(), QByteArray("comment")); decls = top->findDeclarations(QualifiedIdentifier("b")); if ( decls.isEmpty() ) { decls = top->childContexts().last()->findDeclarations(QualifiedIdentifier("b")); } auto b = decls.first(); QCOMPARE(b->comment(), QByteArray()); } void PyDUChainTest::testComments_data() { QTest::addColumn("code"); QTest::newRow("variable") << "b=5\n\"\"\"comment\"\"\"\na=5\nb=5"; QTest::newRow("function") << "def a():\n \"\"\"comment\"\"\"\n b=5"; QTest::newRow("class") << "class a:\n \"\"\"comment\"\"\"\n b=5"; } diff --git a/parser/astdefaultvisitor.cpp b/parser/astdefaultvisitor.cpp index 0df6e6ee..9a56fb81 100644 --- a/parser/astdefaultvisitor.cpp +++ b/parser/astdefaultvisitor.cpp @@ -1,448 +1,450 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astdefaultvisitor.h" #include "ast.h" namespace Python { void free_ast_recursive(CodeAst *node) { AstFreeVisitor v; v.visitCode(node); } AstDefaultVisitor::AstDefaultVisitor() { } AstDefaultVisitor::~AstDefaultVisitor() { } // The Ast "ends" here, those dont have child nodes // note that Identifier is not a node in this Ast void AstDefaultVisitor::visitNameConstant(NameConstantAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitPass(PassAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNonlocal(NonlocalAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBreak(BreakAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitContinue(ContinueAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitEllipsis(EllipsisAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNumber(NumberAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitString(StringAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBytes(BytesAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitIdentifier(Identifier* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitJoinedString(JoinedStringAst* node) { //TODO: Fix range/context/??? bugs, then re-enable this. Q_UNUSED(node); // foreach (Ast* value, node->values) { // visitNode(value); // } } void AstDefaultVisitor::visitFormattedValue(FormattedValueAst* node) { - visitNode(node->value); - visitNode(node->formatSpec); +// TODO: Fix range issues, then re-enable (if required) + Q_UNUSED(node); +// visitNode(node->value); +// visitNode(node->formatSpec); } void AstDefaultVisitor::visitStarred(StarredAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitArg(ArgAst* node) { visitNode(node->annotation); visitNode(node->argumentName); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitAlias(AliasAst* node) { visitIdentifier(node->name); visitIdentifier(node->asName); } void AstDefaultVisitor::visitName(NameAst* node) { visitIdentifier(node->identifier); } void AstDefaultVisitor::visitGlobal(GlobalAst* node) { foreach (Identifier* identifier, node->names) { visitIdentifier(identifier); } } void AstDefaultVisitor::visitCode(CodeAst* node) { foreach (Ast* statement, node->body) { visitNode(statement); } visitIdentifier(node->name); } void AstDefaultVisitor::visitExpression(ExpressionAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitYieldFrom(YieldFromAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAssertion(AssertionAst* node) { visitNode(node->condition); visitNode(node->message); } void AstDefaultVisitor::visitDelete(DeleteAst* node) { foreach (ExpressionAst* expression, node->targets) { visitNode(expression); } } void AstDefaultVisitor::visitExtendedSlice(ExtendedSliceAst* node) { foreach (SliceAst* slice, node->dims) { visitNode(slice); } } void AstDefaultVisitor::visitFor(ForAst* node) { visitNode(node->target); visitNode(node->iterator); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitGeneratorExpression(GeneratorExpressionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitIf(IfAst* node) { visitNode(node->condition); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitIfExpression(IfExpressionAst* node) { visitNode(node->condition); visitNode(node->body); visitNode(node->orelse); } void AstDefaultVisitor::visitImport(ImportAst* node) { foreach (AliasAst* alias, node->names) { visitNode(alias); } } void AstDefaultVisitor::visitImportFrom(ImportFromAst* node) { foreach (AliasAst* alias, node->names) { visitNode(alias); } visitIdentifier(node->module); } void AstDefaultVisitor::visitIndex(IndexAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitLambda(LambdaAst* node) { visitNode(node->arguments); visitNode(node->body); } void AstDefaultVisitor::visitRaise(RaiseAst* node) { visitNode(node->type); } void AstDefaultVisitor::visitReturn(ReturnAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAwait(AwaitAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitSet(SetAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitSetComprehension(SetComprehensionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitSlice(SliceAst* node) { visitNode(node->lower); visitNode(node->upper); visitNode(node->step); } void AstDefaultVisitor::visitSubscript(SubscriptAst* node) { visitNode(node->value); visitNode(node->slice); } void AstDefaultVisitor::visitTry(TryAst* node) { foreach (Ast* statement, node->body) { visitNode(statement); } foreach (ExceptionHandlerAst* handler, node->handlers) { visitNode(handler); } foreach (Ast* statement, node->orelse) { visitNode(statement); } foreach (Ast* statement, node->finally) { visitNode(statement); } } void AstDefaultVisitor::visitTuple(TupleAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitUnaryOperation(UnaryOperationAst* node) { visitNode(node->operand); } void AstDefaultVisitor::visitWhile(WhileAst* node) { visitNode(node->condition); foreach (Ast* statement, node->body) { visitNode(statement); } foreach (Ast* statement, node->orelse) { visitNode(statement); } } void AstDefaultVisitor::visitWith(WithAst* node) { foreach (Ast* item, node->items) { visitNode(item); } foreach (Ast* statement, node->body) { visitNode(statement); } } void AstDefaultVisitor::visitWithItem(WithItemAst* node) { visitNode(node->contextExpression); visitNode(node->optionalVars); } void AstDefaultVisitor::visitYield(YieldAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitList(ListAst* node) { foreach (ExpressionAst* expression, node->elements) { visitNode(expression); } } void AstDefaultVisitor::visitListComprehension(ListComprehensionAst* node) { visitNode(node->element); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitExceptionHandler(ExceptionHandlerAst* node) { visitNode(node->type); visitNode(node->name); foreach (Ast* statement, node->body) { visitNode(statement); } } void AstDefaultVisitor::visitDict(DictAst* node) { foreach (ExpressionAst* expression, node->keys) { visitNode(expression); } foreach (ExpressionAst* expression, node->values) { visitNode(expression); } } void AstDefaultVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { visitNode(node->key); visitNode(node->value); foreach (ComprehensionAst* comp, node->generators) { visitNode(comp); } } void AstDefaultVisitor::visitAugmentedAssignment(AugmentedAssignmentAst* node) { visitNode(node->target); visitNode(node->value); } void AstDefaultVisitor::visitAnnotationAssignment(AnnotationAssignmentAst* node) { visitNode(node->target); visitNode(node->annotation); visitNode(node->value); } void AstDefaultVisitor::visitBinaryOperation(BinaryOperationAst* node) { visitNode(node->lhs); visitNode(node->rhs); } void AstDefaultVisitor::visitBooleanOperation(BooleanOperationAst* node) { foreach (ExpressionAst* expression, node->values) { visitNode(expression); } } void AstDefaultVisitor::visitClassDefinition(ClassDefinitionAst* node) { foreach (ExpressionAst* expression, node->baseClasses) { visitNode(expression); } foreach (Ast* statement, node->body) { visitNode(statement); } foreach (ExpressionAst* expression, node->decorators) { visitNode(expression); } visitIdentifier(node->name); } void AstDefaultVisitor::visitCompare(CompareAst* node) { visitNode(node->leftmostElement); foreach (ExpressionAst* expression, node->comparands) { visitNode(expression); } } void AstDefaultVisitor::visitComprehension(ComprehensionAst* node) { visitNode(node->target); visitNode(node->iterator); foreach (ExpressionAst* expression, node->conditions) { visitNode(expression); } } void AstDefaultVisitor::visitAssignment(AssignmentAst* node) { foreach (ExpressionAst* expression, node->targets) { visitNode(expression); }; visitNode(node->value); } void AstDefaultVisitor::visitCall(CallAst* node) { visitNode(node->function); foreach (ExpressionAst* argument, node->arguments) { visitNode(argument); } foreach (KeywordAst* kwd, node->keywords) { visitNode(kwd); } } void AstDefaultVisitor::visitFunctionDefinition(FunctionDefinitionAst* node) { foreach (ExpressionAst* decorator, node->decorators) { visitNode(decorator); } visitNode(node->arguments); visitNode(node->returns); foreach (Ast* stmt, node->body) { visitNode(stmt); } visitIdentifier(node->name); } void AstDefaultVisitor::visitAttribute(AttributeAst* node) { visitNode(node->value); visitIdentifier(node->attribute); } void AstDefaultVisitor::visitKeyword(KeywordAst* node) { visitNode(node->value); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitArguments(ArgumentsAst* node) { foreach (ArgAst* expression, node->arguments) { visitNode(expression); } foreach (ExpressionAst* expression, node->defaultValues ) { visitNode(expression); } } }