diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f9e465b..09b089dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,160 +1,160 @@ cmake_minimum_required(VERSION 3.0) project(kdevpython VERSION 5.4.40) # KDevplatform dependency version set(KDEVPLATFORM_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") find_package (ECM "5.28.0" REQUIRED NO_MODULE) set(CMAKE_MODULE_PATH ${kdevpython_SOURCE_DIR}/cmake/modules ${CMAKE_MODULE_PATH} ${ECM_MODULE_PATH}) include(KDEInstallDirs) include(KDECMakeSettings) include(KDECompilerSettings NO_POLICY_SCOPE) include(GenerateExportHeader) include(ECMAddTests) include(ECMSetupVersion) include(ECMQtDeclareLoggingCategory) if(POLICY CMP0071) # Avoids compat messages from CMake 3.10+, with Qt < 5.9.4 # See https://bugreports.qt.io/browse/QTBUG-63442 cmake_policy(SET CMP0071 OLD) endif() if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wdocumentation") endif() add_definitions( -DTRANSLATION_DOMAIN=\"kdevpython\" ) # CMake looks for exactly the specified version first and ignores newer versions. # To avoid that, start looking for the newest supported version and work down. -set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4) +set(Python_ADDITIONAL_VERSIONS 3.8 3.7 3.6 3.5 3.4) foreach(_PYTHON_V ${Python_ADDITIONAL_VERSIONS}) find_package(PythonInterp ${_PYTHON_V}) if ( PYTHONINTERP_FOUND ) break() endif() endforeach() # Must unset before searching for libs, otherwise these are checked before the required version... unset(Python_ADDITIONAL_VERSIONS) if ( PYTHONINTERP_FOUND AND PYTHON_VERSION_STRING VERSION_GREATER "3.4" ) # Find libraries that match the found interpreter (mismatched versions not supported). # This assumes libs are available for the newest Python version on the system. # KDevelop should _always_ be built against the newest possible version, so notabug. find_package(PythonLibs "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" REQUIRED EXACT) endif() if ( NOT PYTHONLIBS_FOUND OR PYTHONLIBS_VERSION_STRING VERSION_LESS "3.4.3" ) - message(FATAL_ERROR "Python >= 3.4.3 but < 3.8 with --enable-shared is required to build kdev-python") + message(FATAL_ERROR "Python >= 3.4.3 but < 3.9 with --enable-shared is required to build kdev-python") endif() configure_file(kdevpythonversion.h.cmake "${CMAKE_CURRENT_BINARY_DIR}/kdevpythonversion.h" @ONLY) set(QT_MIN_VERSION "5.7.0") find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED Core Widgets Test) set(KF5_DEP_VERSION "5.28.0") find_package(KF5 ${KF5_DEP_VERSION} REQUIRED I18n ThreadWeaver TextEditor ) find_package(KDevPlatform ${KDEVPLATFORM_VERSION} CONFIG REQUIRED) find_package(KDevelop ${KDEVPLATFORM_VERSION} REQUIRED) if(NOT CMAKE_VERSION VERSION_LESS "3.10.0" AND KF5_VERSION VERSION_LESS "5.42.0") # Avoids bogus warnings with CMake 3.10+, KF5.42+ has workaround list(APPEND CMAKE_AUTOMOC_MACRO_NAMES "K_PLUGIN_FACTORY_WITH_JSON") endif() if ( NOT WIN32 ) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors -Wall") endif ( NOT WIN32 ) # then, build the plugin include_directories( ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/duchain ${CMAKE_CURRENT_SOURCE_DIR}/parser ${CMAKE_CURRENT_BINARY_DIR}/parser ) add_subdirectory(app_templates) add_subdirectory(parser) add_subdirectory(duchain) add_subdirectory(codecompletion) add_subdirectory(debugger) add_subdirectory(docfilekcm) set(kdevpythonlanguagesupport_PART_SRCS codegen/correctionfilegenerator.cpp codegen/refactoring.cpp pythonlanguagesupport.cpp pythonparsejob.cpp pythonhighlighting.cpp pythonstylechecking.cpp # config pages: docfilekcm/docfilewizard.cpp docfilekcm/docfilemanagerwidget.cpp docfilekcm/kcm_docfiles.cpp pep8kcm/kcm_pep8.cpp projectconfig/projectconfigpage.cpp ) ecm_qt_declare_logging_category(kdevpythonlanguagesupport_PART_SRCS HEADER codegendebug.h IDENTIFIER KDEV_PYTHON_CODEGEN CATEGORY_NAME "kdevelop.plugins.python.codegen" ) ecm_qt_declare_logging_category(kdevpythonlanguagesupport_PART_SRCS HEADER pythondebug.h IDENTIFIER KDEV_PYTHON CATEGORY_NAME "kdevelop.plugins.python" ) ki18n_wrap_ui(kdevpythonlanguagesupport_PART_SRCS codegen/correctionwidget.ui projectconfig/projectconfig.ui pep8kcm/pep8.ui ) kdevplatform_add_plugin(kdevpythonlanguagesupport JSON kdevpythonsupport.json SOURCES ${kdevpythonlanguagesupport_PART_SRCS}) target_link_libraries(kdevpythonlanguagesupport KDev::Interfaces KDev::Language KDev::Util KF5::ThreadWeaver KF5::TextEditor kdevpythoncompletion kdevpythonparser kdevpythonduchain ) get_target_property(DEFINESANDINCLUDES_INCLUDE_DIRS KDev::DefinesAndIncludesManager INTERFACE_INCLUDE_DIRECTORIES) include_directories(${DEFINESANDINCLUDES_INCLUDE_DIRS}) install(DIRECTORY documentation_files DESTINATION ${KDE_INSTALL_DATADIR}/kdevpythonsupport) install(DIRECTORY correction_files DESTINATION ${KDE_INSTALL_DATADIR}/kdevpythonsupport) install(FILES codestyle.py DESTINATION ${KDE_INSTALL_DATADIR}/kdevpythonsupport) install(FILES org.kde.kdev-python.metainfo.xml DESTINATION ${KDE_INSTALL_METAINFODIR}) # kdebugsettings file # kdebugsettings >= 18.12 supports/pushes for some newer, not backward-compatible format. # For ECM >= 5.59 we install categories files to new location, which is only supported by # newer kdebugsettings also supporting the new content format, so we use that as base. if (ECM_VERSION VERSION_GREATER "5.58.0") install(FILES kdevpythonsupport-newformat.categories RENAME kdevpythonsupport.categories DESTINATION ${KDE_INSTALL_LOGGINGCATEGORIESDIR} ) else() install(FILES kdevpythonsupport-oldformat.categories RENAME kdevpythonsupport.categories DESTINATION ${KDE_INSTALL_CONFDIR}) endif() feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES) diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index e3cb2b9c..61e144c5 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1893 +1,1901 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/functiondeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/nonetype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, const ReferencedTopDUContext& updateContext_) { ReferencedTopDUContext updateContext(updateContext_); m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); delete prebuilder; qCDebug(KDEV_PYTHON_DUCHAIN) << "Second declarationbuilder pass"; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Prebuilding declarations"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = nullptr; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(name, name), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. if ( currentVariableDefinition->context != ExpressionAst::Context::Store ) { return nullptr; } return visitVariableDeclaration(currentVariableDefinition->identifier, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(nullptr); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { return currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), nullptr, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = nullptr; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != nullptr ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; RangeInRevision range = editorFindRange(node, node); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = nullptr; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, nullptr, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); auto mgrType = v.lastType(); auto enterType = mgrType; // If we can't find __enter__(), assume it returns `self` like file objects. static const IndexedIdentifier enterId(KDevelop::Identifier("__enter__")); DUChainReadLocker lock; if ( auto enterFunc = dynamic_cast( Helper::accessAttribute(mgrType, enterId, topContext()))) { if ( auto enterFuncType = enterFunc->type() ) { enterType = enterFuncType->returnType(); } } lock.unlock(); // This may be any assignable expression, e.g. `with foo() as bar[3]: ...` assignToUnknown(node->optionalVars, enterType); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = nullptr; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), nullptr, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return nullptr; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = nullptr; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(nullptr); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(nullptr); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(nullptr); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = nullptr; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = nullptr; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = nullptr; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return nullptr; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = nullptr; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return nullptr; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return nullptr; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, nullptr, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, nullptr); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(nullptr), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. DUChainWriteLocker lock; previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } visitNodeList(node->arguments->defaultValues); if (node->arguments->vararg) { visitVariableDeclaration(node->arguments->vararg->argumentName); } if (node->arguments->kwarg) { visitVariableDeclaration(node->arguments->kwarg->argumentName); } visitNode(node->body); closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker wlock; auto contentType = Helper::contentOfIterable(argVisitor.lastType(), topContext()); container->addContentType(contentType); v.lastDeclaration()->setType(container); } }; auto docstring = function->comment(); if ( ! docstring.isEmpty() ) { foreach ( const auto& key, items.keys() ) { if ( Helper::docstringContainsHint(docstring, key, &args) ) { items[key](); } } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer called) { DUChainReadLocker lock; auto funcInfo = Helper::functionForCalled(called.data()); auto function = funcInfo.declaration; if ( ! function ) { return; } if ( function->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Note: within this function: // - 'parameters' refers to the parameters of the function definition. // - 'arguments' refers to the arguments of the function call. DUContext* parameterContext = DUChainUtils::argumentContext(function); FunctionType::Ptr functionType = function->type(); if ( ! parameterContext || ! functionType ) { return; } QVector parameters = parameterContext->localDeclarations(); if ( parameters.isEmpty() ) { return; } const int specialParamsCount = (function->vararg() != -1) + (function->kwarg() != -1); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfParam = false; if ( ( function->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! function->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as parameters // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfParam = true; } } lock.unlock(); bool explicitSelfArgument = false; if ( hasSelfParam && ! function->isClassMethod() && node->function->astType == Ast::AttributeAstType ) { // Calling an attribute, e.g. `instance.foo(arg)` or `MyClass.foo(instance, arg)`. ExpressionVisitor valueVisitor(currentContext()); valueVisitor.visitNode(static_cast(node->function)->value); if ( valueVisitor.lastDeclaration().dynamicCast() && valueVisitor.isAlias() ) { // Function is attribute of a class _type_ (not instance), so first arg is used as `self`. explicitSelfArgument = true; } } int currentParamIndex = hasSelfParam; int currentArgumentIndex = explicitSelfArgument; int indexInVararg = -1; int paramsAvailable = qMin(functionType->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentArgumentIndex++ ) { atVararg = atVararg || currentParamIndex == function->vararg(); // Not >=, nonexistent vararg is -1. ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(function->vararg()); IndexedContainer::Ptr varargContainer = parameter->type(); if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functionType->removeArgument(currentParamIndex); functionType->addArgument(newType, currentParamIndex); function->setAbstractType(functionType.cast()); parameters.at(currentParamIndex)->setType(newType); currentParamIndex++; } } // **kwargs is always the last parameter MapType::Ptr kwargsDict; if ( function->kwarg() != -1 ) { kwargsDict = parameters.last()->abstractType().cast(); } lock.unlock(); DUChainWriteLocker wlock; foreach ( KeywordAst* keyword, node->keywords ) { wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); bool matchedNamedParam = false; HintedType::Ptr addType = HintedType::Ptr(new HintedType()); if ( keyword->argumentName ) { openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); for (int ip = currentParamIndex; ip < paramsAvailable; ++ip ) { if ( parameters.at(ip)->identifier().toString() != keyword->argumentName->value ) { continue; } matchedNamedParam = true; auto newType = Helper::mergeTypes(parameters.at(ip)->abstractType(), addType); functionType->removeArgument(ip); functionType->addArgument(newType, ip); parameters.at(ip)->setType(newType); } } else if ( auto unpackedDict = argumentVisitor.lastType().cast() ) { // 'keyword is actually an unpacked dict: `foo(**{'a': 12}). openType(addType); addType->setType(unpackedDict->contentType().abstractType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); } else { // Maybe the dict type wasn't loaded yet, or something else happened. continue; } if ( ! matchedNamedParam && kwargsDict ) { DUChainWriteLocker lock; kwargsDict->addContentType(addType); parameters.last()->setAbstractType(kwargsDict); } } function->setAbstractType(functionType); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(target->identifier, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, nullptr, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { DUChainWriteLocker lock; list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(nullptr); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreadyOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 || (starred == -1 && spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } void DeclarationBuilder::visitAnnotationAssignment(AnnotationAssignmentAst* node) { AstDefaultVisitor::visitAnnotationAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->target); v.visitNode(node->value); auto assignType = v.lastType(); // Never mind aliasing, why annotate that? v.visitNode(node->annotation); assignType = Helper::mergeTypes(assignType, v.lastType()); assignToUnknown(node->target, assignType); } +void DeclarationBuilder::visitAssignmentExpression(AssignmentExpressionAst* node) { + AstDefaultVisitor::visitAssignmentExpression(node); + + ExpressionVisitor v(currentContext()); + v.visitNode(node->value); + assignToUnknown(node->target, v.lastType()); +} + void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { visitNodeList(node->decorators); visitNodeList(node->baseClasses); const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, NoTypeRequired); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); auto docstring = getDocstring(node->body); dec->setComment(docstring); if ( ! docstring.isEmpty() ) { // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(docstring, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(docstring, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(docstring, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassTypes.append(type); // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); visitNodeList(node->body); lock.lock(); closeContext(); m_currentClassTypes.removeLast(); closeType(); closeDeclaration(); } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); dec->setStatic(false); dec->setClassMethod(false); dec->setProperty(false); foreach ( auto decorator, node->decorators) { visitNode(decorator); switch (decorator->astType) { case Ast::AttributeAstType: { auto attr = static_cast(decorator)->attribute->value; if ( attr == QStringLiteral("setter") || attr == QStringLiteral("getter") || attr == QStringLiteral("deleter") ) dec->setProperty(true); break; } case Ast::NameAstType: { auto name = static_cast(decorator)->identifier->value; if ( name == QStringLiteral("staticmethod") ) dec->setStatic(true); else if ( name == QStringLiteral("classmethod") ) dec->setClassMethod(true); else if ( name == QStringLiteral("property") ) dec->setProperty(true); break; } default: {} } } visitFunctionArguments(node); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new NoneType())); } dec->setType(type); } if ( ! dec->isStatic() ) { DUContext* args = DUChainUtils::argumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! dec->isClassMethod() ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { static auto noneType = AbstractType::Ptr(new NoneType()); if ( auto function = currentType() ) { // Statements with no explicit value return `None`. auto encountered = noneType; if ( node->value ) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); encountered = v.lastType(); } // Update the containing function's return type DUChainWriteLocker lock; function->setReturnType(Helper::mergeTypes(function->returnType(), encountered)); } else { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; - foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { + foreach ( ArgAst* arg, node->posonlyargs + node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && workingOnDeclaration->isClassMethod() ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, AliasDeclarationType); if ( ! m_currentClassTypes.isEmpty() ) { auto classDecl = m_currentClassTypes.last()->declaration(topContext()); decl->setAliasedDeclaration(classDecl); } closeDeclaration(); paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } if ( isFirst && ! workingOnDeclaration->isStatic() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassTypes.last().cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, nullptr, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, nullptr, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/declarationbuilder.h b/duchain/declarationbuilder.h index adee6d6d..1992b781 100644 --- a/duchain/declarationbuilder.h +++ b/duchain/declarationbuilder.h @@ -1,285 +1,286 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2011-2014 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef PYTHON_DECLARATIONBUILDER_H #define PYTHON_DECLARATIONBUILDER_H #include #include #include #include "contextbuilder.h" #include "declarations/functiondeclaration.h" #include "ast.h" namespace Python { class CorrectionHelper; typedef KDevelop::AbstractTypeBuilder TypeBuilderBase; typedef KDevelop::AbstractDeclarationBuilder DeclarationBuilderBase; class KDEVPYTHONDUCHAIN_EXPORT DeclarationBuilder: public DeclarationBuilderBase { public: DeclarationBuilder(PythonEditorIntegrator* editor, int ownPriority); ~DeclarationBuilder() override; /** * @brief Entry function, called by KDevPlatform. */ ReferencedTopDUContext build(const IndexedString& url, Ast* node, const ReferencedTopDUContext& updateContext = ReferencedTopDUContext()) override; /** * @brief Set whether the current running pass is the first or the second one. * @param prebuilding true if first pass, false if second */ void setPrebuilding(bool prebuilding); /** * @brief Priority of this parse job. */ int jobPriority() const; /** * @brief Get the docstring which belongs to the given body * @param body Body of statements which is in the documented function or class */ QString getDocstring(QList body) const; /** * @brief Construct the dotted name of a module from an import ... from ... as statement. * * @param node the import ... from node * @param alias the ... as ... node * @param intermediate an additional string to prepend to the module name (dot is added internally) */ QString buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const; /** * @brief Get a list of all module names which were not found while parsing. */ QVector missingModules() const { return m_missingModules; } protected: /// AST visitor functions void visitClassDefinition(ClassDefinitionAst* node) override; void visitFunctionDefinition(FunctionDefinitionAst* node) override; void visitAssignment(AssignmentAst* node) override; void visitAnnotationAssignment(AnnotationAssignmentAst* node) override; + void visitAssignmentExpression(AssignmentExpressionAst* node) override; void visitFor(ForAst* node) override; void visitImport(ImportAst* node) override; void visitImportFrom(ImportFromAst* node) override; void visitArguments(ArgumentsAst* node) override; void visitExceptionHandler(ExceptionHandlerAst* node) override; void visitReturn(ReturnAst* node) override; void visitCode(CodeAst* node) override; void visitCall(CallAst* node) override; void visitYield(YieldAst* node) override; void visitWithItem(WithItemAst* node) override; void visitLambda(LambdaAst* node) override; void visitComprehension(ComprehensionAst* node) override; void visitGlobal(GlobalAst* node) override; void visitAssertion(AssertionAst* node) override; void visitIf(IfAst* node) override; void visitString(StringAst* node) override; void visitNode(Ast* node) override; protected: enum VisitVariableFlags { NoVisitVariableFlags = 0x0, AbortIfReopenMismatch = 0x1 }; /// Visitor helper functions template T* visitVariableDeclaration(Python::Ast* node, Declaration* previous=nullptr, AbstractType::Ptr type = AbstractType::Ptr(), VisitVariableFlags flags=NoVisitVariableFlags); template T* visitVariableDeclaration(Identifier* node, Declaration* previous=nullptr, AbstractType::Ptr type = AbstractType::Ptr(), VisitVariableFlags flags=NoVisitVariableFlags); protected: /** * @brief Applies docstring hints, such as "addsType" * @param node the called function * @param function the declaration which belongs to @p node * * Used for example in a = []; a.append(3) to set the type of a to "list of int". */ void applyDocstringHints(CallAst* node, Python::FunctionDeclaration::Ptr function); /** * @brief Try to deduce types of function arguments from a call and stores it in the duchain * @param node the called function or class (i.e. constructor) * @param called the declaration which belongs to @p node * * Used for example in def f(x): pass; a = f(3) to set the type of x to "int" */ void addArgumentTypeHints(CallAst* node, DeclarationPointer called); /** * @brief Adjust the type of foo in an expression like assert isinstance(fooinstance, Foo) * Does nothing if the given expression isn't of any of the forms * a) isinstance(fooinstance, Foo) * b) type(fooinstance) == Foo */ void adjustForTypecheck(ExpressionAst* check, bool useUnsure); /// Helper for the above void adjustExpressionsForTypecheck(ExpressionAst* adjust, ExpressionAst* from, bool useUnsure); /// Represents a single source type in a tuple assignment. struct SourceType { AbstractType::Ptr type; DeclarationPointer declaration; bool isAlias; }; /** @brief If sourceType is a container that can be unpacked into outTypes, do so. */ void tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred); /** * @brief Handle a variable assignment to @p name and give it the type @p element. */ void assignToName(NameAst* name, const SourceType& element); /** * @brief Handle assignment to subscript @p subscript with rhs type @p element. */ void assignToSubscript(SubscriptAst* subscript, const SourceType& element); /** * @brief Handle assignment to an attribute @p attribute with rhs type @p element. */ void assignToAttribute(AttributeAst* attribute, const SourceType& element); /** * @brief Handle assignment to a target @p target with rhs type @p element. */ void assignToTuple(TupleAst* tuple, const SourceType& element); /** * @brief Handle assignment to a target @p target with rhs type @p element. */ void assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type); void assignToUnknown(ExpressionAst* target, const SourceType& element); /** * @brief Find all existing declarations for the identifier @p node */ QList existingDeclarationsForNode(Identifier* node); enum FitDeclarationType { NoTypeRequired, InstanceDeclarationType, AliasDeclarationType, FunctionDeclarationType }; FitDeclarationType kindForType(AbstractType::Ptr type, bool isAlias = false); /** * @brief schedule an object to be deleted when the declaration builder is destroyed * this is used to bypass the automated duchain cleanup for imports */ void scheduleForDeletion(DUChainBase* d, bool doschedule = true); /** * @brief python-specific version of openDeclaration which scans for existing declarations in previous versions of * this top-context in a more intelligent way. * Use this in normal declaratonbuilder code if you can't use visitVariableDeclaration. */ template T* eventuallyReopenDeclaration(Python::Identifier* name, FitDeclarationType mustFitType); template QList reopenFittingDeclaration(QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok); /** * @brief Create a declaration for an import statement. * * @param dottedName The dotted name of the module, like "random.randint". * @param declarationIdentifier provides the name and range * @param rangeNode can be used to override the declarationIdentifier's range, if required. Defaults to 0. * @param problemEncountered the encountered problem, if there's any * @return :Declaration* the declaration created, or 0 if none was found. **/ Declaration* createModuleImportDeclaration(QString dottedName, QString declarationName, Python::Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Python::Ast* rangeNode = nullptr); /** * @brief Create a tree of declarations for the specified list. * Give the list ["foo","bar","baz"], and you'll get a declaration "foo" containing "bar" in its internal context, * "bar" containing "baz" etc. * This is used in import handling. * This function automatically updates existing declaration trees to the maximum level possible! Thus, * if you call this with ["foo", "bar"], then ["foo", "baz"], "baz" will be added to "foo". * * @warning The DUChain must not be locked when this is called. * * @param nameComponents the list of names to create declarations for * @param declarationIdentifier provides the name and range * @param innerCtx the internalContext() to set on the last created declaration. Either this or aliasDeclaration must be provided! * @param aliasDeclaration the declaration to alias with the last created declaration * @param range can be used to override the declarationIdentifier's range, if required. Defaults to an invalid range. * @return :Declaration* the top level declaration created **/ Declaration* createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration = nullptr, const RangeInRevision& range = RangeInRevision::invalid()); /** * @brief Find a declaration specified by "foo.bar.baz" in the given top context. * * @param dottedNameIdentifier string list of module names, starting with the most general one. * @param ctx top context to search * @return :Declaration* declaration if found, 0 otherwise. * * This will traverse nested classes and properties until the list of passed names is exhausted. * @warning The DUChain must not be locked. **/ Declaration* findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const; private: template T* openDeclaration(Identifier* name, DeclarationFlags flags = NoFlags) { T* decl = DeclarationBuilderBase::openDeclaration(KDevelop::Identifier(name->value), editorFindRange(name, name), flags); decl->setAlwaysForceDirect(true); return decl; }; void closeDeclaration() override; private: /// HACK: List of items to delete after parsing finishes, to work around the built-in cleanup logic QList m_scheduledForDeletion; QScopedPointer m_correctionHelper; int m_ownPriority = 0; QVector m_currentClassTypes; // missing modules, for not reporting them as unknown variables QVector m_missingModules; StringAst* m_lastComment = nullptr; }; } #endif // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on diff --git a/duchain/expressionvisitor.cpp b/duchain/expressionvisitor.cpp index 5c5286ae..055c5b49 100644 --- a/duchain/expressionvisitor.cpp +++ b/duchain/expressionvisitor.cpp @@ -1,761 +1,765 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2010 Miquel Canes Gonzalez * * Copyright 2011-2013 by Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "expressionvisitor.h" #include "types/nonetype.h" #include "types/indexedcontainer.h" #include "declarations/functiondeclaration.h" #include "pythonduchainexport.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include #include using namespace KDevelop; using namespace Python; using namespace KTextEditor; namespace Python { QHash ExpressionVisitor::m_defaultTypes; AbstractType::Ptr ExpressionVisitor::encounterPreprocess(AbstractType::Ptr type) { return Helper::resolveAliasType(type); } ExpressionVisitor::ExpressionVisitor(const DUContext* ctx) : DynamicLanguageExpressionVisitor(ctx) { if ( m_defaultTypes.isEmpty() ) { m_defaultTypes.insert(NameConstantAst::True, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::False, AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); m_defaultTypes.insert(NameConstantAst::None, AbstractType::Ptr(new NoneType())); } Q_ASSERT(context()); Q_ASSERT(context()->topContext()); } ExpressionVisitor::ExpressionVisitor(ExpressionVisitor* parent, const DUContext* overrideContext) : DynamicLanguageExpressionVisitor(parent) , m_forceGlobalSearching(parent->m_forceGlobalSearching) , m_reportUnknownNames(parent->m_reportUnknownNames) , m_scanUntilCursor(parent->m_scanUntilCursor) { if ( overrideContext ) { m_context = overrideContext; } Q_ASSERT(context()); } void ExpressionVisitor::encounter(AbstractType::Ptr type, DeclarationPointer declaration, bool alias) { setLastIsAlias(alias); DynamicLanguageExpressionVisitor::encounter(type, declaration); } void ExpressionVisitor::visitAttribute(AttributeAst* node) { ExpressionVisitor v(this); v.visitNode(node->value); setConfident(false); // Find a matching declaration which is made inside the type of the accessed object. // Like, for B.C where B is an instance of foo, find a property of foo called C. DUChainReadLocker lock; auto attribute = Helper::accessAttribute(v.lastType(), node->attribute->value, topContext()); auto resolved = Helper::resolveAliasDeclaration(attribute); if ( ! resolved ) { encounterUnknown(); return; } auto function = dynamic_cast(resolved); if ( function && function->type() && function->isProperty() ) { encounter(function->type()->returnType(), DeclarationPointer(function)); return; } encounter(attribute->abstractType(), DeclarationPointer(attribute)); setLastIsAlias(function || dynamic_cast(attribute) || dynamic_cast(resolved) ); } void ExpressionVisitor::visitCall(CallAst* node) { visitNodeList(node->arguments); ExpressionVisitor v(this); v.visitNode(node->function); auto declaration = Helper::resolveAliasDeclaration(v.lastDeclaration().data()); if ( ! v.isAlias() && v.lastType() ) { if ( auto functionType = v.lastType().cast() ) { encounter(functionType->returnType()); return; } if ( auto classType = v.lastType().cast() ) { declaration = classType->declaration(topContext()); } } if ( ! declaration ) { encounterUnknown(); return; } ClassDeclaration* classDecl = dynamic_cast(declaration); DUChainReadLocker lock; auto function = Helper::functionForCalled(declaration, v.isAlias()); lock.unlock(); AbstractType::Ptr type; Declaration* decl; if ( function.isConstructor && classDecl ) { // Don't use return type from constructor. // It's wrong for builtins, or classes without their own __init__ methods(). type = classDecl->abstractType(); decl = classDecl; } else if ( function.declaration && function.declaration->type() ) { // But do use the return value of normal functions or __call__(). type = function.declaration->type()->returnType(); decl = function.declaration; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration is not a class or function declaration"; encounterUnknown(); return; } if ( function.declaration ) { auto docstring = function.declaration->comment(); if ( ! docstring.isEmpty() ) { // Our documentation data uses special docstrings that override the return type // of some functions (including constructors). type = docstringTypeOverride(node, type, docstring); } } encounter(type, DeclarationPointer(decl)); } AbstractType::Ptr ExpressionVisitor::docstringTypeOverride( CallAst* node, const AbstractType::Ptr normalType, const QString& docstring) { auto docstringType = normalType; auto listOfTuples = [&](AbstractType::Ptr key, AbstractType::Ptr value) { auto newType = typeObjectForIntegralType("list"); IndexedContainer::Ptr newContents = typeObjectForIntegralType("tuple"); if ( ! newType || ! newContents ) { return AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! key ) { key = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } if ( ! value ) { value = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } newContents->addEntry(key); newContents->addEntry(value); newType->addContentType(AbstractType::Ptr::staticCast(newContents)); AbstractType::Ptr resultingType = AbstractType::Ptr::staticCast(newType); return resultingType; }; QHash< QString, std::function > knownDocstringHints; knownDocstringHints["getsType"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); if ( auto t = baseTypeVisitor.lastType().cast() ) { docstringType = t->contentType().abstractType(); return true; } return false; }; knownDocstringHints["getsList"] = [&](QStringList /*arguments*/, QString currentHint) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainReadLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { auto newType = typeObjectForIntegralType("list"); if ( ! newType ) { return false; } AbstractType::Ptr contentType; if ( currentHint == "getsList" ) { contentType = t->contentType().abstractType(); } else if ( auto map = MapType::Ptr::dynamicCast(t) ) { contentType = map->keyType().abstractType(); } newType->addContentType(contentType); docstringType = newType.cast(); return true; } return false; }; knownDocstringHints["getListOfKeys"] = knownDocstringHints["getsList"]; knownDocstringHints["enumerate"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::NameAstType || node->arguments.size() < 1 ) { return false; } ExpressionVisitor enumeratedTypeVisitor(this); enumeratedTypeVisitor.visitNode(node->arguments.first()); DUChainReadLocker lock; auto intType = typeObjectForIntegralType("int"); auto enumerated = enumeratedTypeVisitor.lastType(); docstringType = listOfTuples(intType, Helper::contentOfIterable(enumerated, topContext())); return true; }; knownDocstringHints["getsListOfBoth"] = [&](QStringList /*arguments*/, QString /*currentHint*/) { if ( node->function->astType != Ast::AttributeAstType ) { return false; } ExpressionVisitor baseTypeVisitor(this); // when calling foo.bar[3].baz.iteritems(), find the type of "foo.bar[3].baz" baseTypeVisitor.visitNode(static_cast(node->function)->value); DUChainReadLocker lock; if ( auto t = baseTypeVisitor.lastType().cast() ) { docstringType = listOfTuples(t->keyType().abstractType(), t->contentType().abstractType()); return true; } return false; }; knownDocstringHints["returnContentEqualsContentOf"] = [&](QStringList arguments, QString /*currentHint*/) { const int argNum = ! arguments.isEmpty() ? (int) arguments.at(0).toUInt() : 0; if ( argNum >= node->arguments.length() ) { return false; } ExpressionAst* relevantArgument = node->arguments.at(argNum); ExpressionVisitor v(this); v.visitNode(relevantArgument); if ( ! v.lastType() ) { return false; } AbstractType::Ptr newType; if ( auto targetContainer = ListType::Ptr::dynamicCast(normalType) ) { // Copy the return type, to set contents for this call only. docstringType = AbstractType::Ptr(targetContainer->clone()); // Add content type of the source. auto sourceContentType = Helper::contentOfIterable(v.lastType(), topContext()); ListType::Ptr::staticCast(docstringType)->addContentType(sourceContentType); } else if ( auto sourceContainer = ListType::Ptr::dynamicCast(v.lastType()) ) { // if the function does not force a return type, just copy the source (like for reversed()) docstringType = AbstractType::Ptr(sourceContainer->clone()); } else { return false; // No target container type } Q_ASSERT(docstringType); return true; }; foreach ( const QString& currentHint, knownDocstringHints.keys() ) { QStringList arguments; if ( ! Helper::docstringContainsHint(docstring, currentHint, &arguments) ) { continue; } // If the hint word appears in the docstring, run the evaluation function. if ( knownDocstringHints[currentHint](arguments, currentHint) ) { // We indeed found something, so we're done. break; } } return docstringType; } long integerValue(ExpressionAst* node, long wrapTo=0) { bool invert = false; if ( node->astType == Ast::UnaryOperationAstType ) { auto unaryOp = static_cast(node); if ( unaryOp->type == Ast::UnaryOperatorSub ) { node = unaryOp->operand; invert = true; } } if ( node->astType == Ast::NumberAstType ) { auto number = static_cast(node); if ( number->isInt ) { // Clamp in case of `a[-9999999:9999999]` or similar. // -1 is just as out-of-range as -99999999, but doesn't cause a huge loop. long upperBound = wrapTo ? wrapTo : LONG_MAX; if (invert) { return qBound(-1L, wrapTo - number->value, upperBound); } return qBound(-1L, number->value, upperBound); } } return LONG_MIN; } void ExpressionVisitor::visitSubscript(SubscriptAst* node) { AstDefaultVisitor::visitNode(node->value); auto valueTypes = Helper::filterType(lastType(), [](AbstractType::Ptr) { return true; }); AbstractType::Ptr result(new IntegralType(IntegralType::TypeMixed)); foreach (const auto& type, valueTypes) { if ( node->slice->astType == Ast::SliceAstType ) { auto slice = static_cast(node->slice); if ( auto tupleType = type.cast() ) { DUChainReadLocker lock; auto newTuple = typeObjectForIntegralType("tuple"); if ( ! newTuple ) { continue; } long step = slice->step ? integerValue(slice->step) : 1; int len = tupleType->typesCount(); long lower = slice->lower ? integerValue(slice->lower, len) : ((step > 0) ? 0 : len - 1); long upper = slice->upper ? integerValue(slice->upper, len) : ((step > 0) ? len : -1); if ( step != LONG_MIN && lower != LONG_MIN && upper != LONG_MIN) { long ii = lower; while ( (upper - ii) * step > 0 ) { if ( 0 <= ii && ii < len ) { newTuple->addEntry(tupleType->typeAt(ii).abstractType()); } ii += step; } } result = Helper::mergeTypes(result, newTuple); continue; } } if ( (node->slice->astType != Ast::IndexAstType) && type.cast() ) { if ( type.cast() ) { continue; // Can't slice dicts. } // Assume that slicing (e.g. foo[3:5]) a list returns the same type. result = Helper::mergeTypes(result, type); } else if ( const auto& indexed = type.cast() ) { long sliceIndex = integerValue(static_cast(node->slice)->value, indexed->typesCount()); if ( 0 <= sliceIndex && sliceIndex < indexed->typesCount() ) { result = Helper::mergeTypes(result, indexed->typeAt(sliceIndex).abstractType()); continue; } // Index is unknown or invalid, could be returning any of the content types. result = Helper::mergeTypes(result, indexed->asUnsureType()); } else if ( const auto& listType = type.cast() ) { result = Helper::mergeTypes(result, listType->contentType().abstractType()); } else { // Type wasn't one with custom handling, so use return type of __getitem__(). DUChainReadLocker lock; static const IndexedIdentifier getitemIdentifier(KDevelop::Identifier("__getitem__")); auto function = Helper::accessAttribute(type, getitemIdentifier, topContext()); if ( function && function->isFunctionDeclaration() ) { if ( FunctionType::Ptr functionType = function->type() ) { result = Helper::mergeTypes(result, functionType->returnType()); } } } } encounter(result); } void ExpressionVisitor::visitLambda(LambdaAst* node) { AstDefaultVisitor::visitLambda(node); FunctionType::Ptr type(new FunctionType()); AbstractType::Ptr mixed(new IntegralType(IntegralType::TypeMixed)); for (int ii = 0; ii < node->arguments->arguments.length(); ++ii) { type->addArgument(mixed); } type->setReturnType(lastType()); encounter(type); } void ExpressionVisitor::visitList(ListAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); if ( content->astType == Ast::StarredAstType ) { auto contentType = Helper::contentOfIterable(contentVisitor.lastType(), topContext()); type->addContentType(contentType); } else { type->addContentType(contentVisitor.lastType()); } } } else { encounterUnknown(); qCWarning(KDEV_PYTHON_DUCHAIN) << " [ !!! ] did not get a typetrack container object when expecting one! Fix code / setup."; } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol)); lock.unlock(); Q_ASSERT(comprehensionContext); DUContext* ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->value); if ( v.lastType() ) { type->addContentType(v.lastType()); } ExpressionVisitor k(this, ctx); k.visitNode(node->key); if ( k.lastType() ) { type->addKeyType(k.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitSetComprehension(SetComprehensionAst* node) { Python::AstDefaultVisitor::visitSetComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); if ( type ) { DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); auto ctx = m_forceGlobalSearching ? context()->topContext() : comprehensionContext; ExpressionVisitor v(this, ctx); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitListComprehension(ListComprehensionAst* node) { AstDefaultVisitor::visitListComprehension(node); DUChainReadLocker lock; auto type = typeObjectForIntegralType("list"); if ( type && ! m_forceGlobalSearching ) { // TODO fixme DUContext* comprehensionContext = context()->findContextAt(CursorInRevision(node->startLine, node->startCol), true); lock.unlock(); ExpressionVisitor v(this, comprehensionContext); Q_ASSERT(comprehensionContext); v.visitNode(node->element); if ( v.lastType() ) { type->addContentType(v.lastType()); } } else { return encounterUnknown(); } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitTuple(TupleAst* node) { DUChainReadLocker lock; IndexedContainer::Ptr type = typeObjectForIntegralType("tuple"); if ( type ) { lock.unlock(); foreach ( ExpressionAst* expr, node->elements ) { ExpressionVisitor v(this); v.visitNode(expr); if ( expr->astType == Ast::StarredAstType ) { // foo = a, *b, c if ( auto unpackedType = v.lastType().cast() ) { for ( int ii = 0; ii < unpackedType->typesCount(); ++ii ) { type->addEntry(unpackedType->typeAt(ii).abstractType()); } } // Unpacking something else, do nothing (i.e. assume it was empty). } else { type->addEntry(v.lastType()); } } encounter(AbstractType::Ptr::staticCast(type)); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "tuple type object is not available"; return encounterUnknown(); } } void ExpressionVisitor::visitIfExpression(IfExpressionAst* node) { AstDefaultVisitor::visitIfExpression(node); if ( node->body && node->orelse ) { ExpressionVisitor v(this); v.visitNode(node->body); AbstractType::Ptr first = v.lastType(); v.visitNode(node->orelse); AbstractType::Ptr second = v.lastType(); encounter(Helper::mergeTypes(first, second)); } } void ExpressionVisitor::visitSet(SetAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("set"); lock.unlock(); ExpressionVisitor contentVisitor(this); if ( type ) { foreach ( ExpressionAst* content, node->elements ) { contentVisitor.visitNode(content); if ( content->astType == Ast::StarredAstType ) { auto contentType = Helper::contentOfIterable(contentVisitor.lastType(), topContext()); type->addContentType(contentType); } else { type->addContentType(contentVisitor.lastType()); } } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitDict(DictAst* node) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("dict"); lock.unlock(); ExpressionVisitor contentVisitor(this); ExpressionVisitor keyVisitor(this); if ( type ) { Q_ASSERT(node->keys.length() == node->values.length()); for ( int ii = 0; ii < node->values.length(); ++ii ) { contentVisitor.visitNode(node->values.at(ii)); if ( node->keys.at(ii) ) { type->addContentType(contentVisitor.lastType()); keyVisitor.visitNode(node->keys.at(ii)); type->addKeyType(keyVisitor.lastType()); } else if ( auto unpackedType = contentVisitor.lastType().cast() ) { // Key is null for `{**foo}` type->addContentType(unpackedType->contentType().abstractType()); type->addKeyType(unpackedType->keyType().abstractType()); } } } encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitNumber(Python::NumberAst* number) { AbstractType::Ptr type; DUChainReadLocker lock; if ( number->isInt ) { type = typeObjectForIntegralType("int"); } else { type = typeObjectForIntegralType("float"); } encounter(type); } void ExpressionVisitor::visitString(Python::StringAst* ) { DUChainReadLocker lock; StructureType::Ptr type = typeObjectForIntegralType("str"); encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitBytes(Python::BytesAst* ) { DUChainReadLocker lock; auto type = typeObjectForIntegralType("bytes"); encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitFormattedValue(Python::FormattedValueAst* ) { DUChainReadLocker lock; StructureType::Ptr type = typeObjectForIntegralType("str"); encounter(AbstractType::Ptr::staticCast(type)); } void ExpressionVisitor::visitJoinedString(Python::JoinedStringAst* ) { DUChainReadLocker lock; StructureType::Ptr type = typeObjectForIntegralType("str"); encounter(AbstractType::Ptr::staticCast(type)); } RangeInRevision nodeRange(Python::Ast* node) { return RangeInRevision(node->startLine, node->startCol, node->endLine,node->endCol); } void ExpressionVisitor::addUnknownName(const QString& name) { if ( m_parentVisitor ) { static_cast(m_parentVisitor)->addUnknownName(name); } else if ( ! m_unknownNames.contains(name) ) { m_unknownNames.insert(name); } } void ExpressionVisitor::visitNameConstant(NameConstantAst* node) { // handles "True", "False", "None" auto defId = m_defaultTypes.constFind(node->value); if ( defId != m_defaultTypes.constEnd() ) { return encounter(*defId); } } void ExpressionVisitor::visitName(Python::NameAst* node) { CursorInRevision findNameBefore; if ( m_scanUntilCursor.isValid() ) { findNameBefore = m_scanUntilCursor; } else if ( m_forceGlobalSearching ) { findNameBefore = CursorInRevision::invalid(); } else { findNameBefore = CursorInRevision(node->endLine, node->endCol); } DUChainReadLocker lock; Declaration* d = Helper::declarationForName(node, findNameBefore, DUChainPointer(context())); if ( d ) { bool isAlias = dynamic_cast(d) || d->isFunctionDeclaration() || dynamic_cast(d); return encounter(d->abstractType(), DeclarationPointer(d), isAlias); } else { if ( m_reportUnknownNames ) { addUnknownName(node->identifier->value); } return encounterUnknown(); } } void ExpressionVisitor::visitCompare(CompareAst* node) { Python::AstDefaultVisitor::visitCompare(node); encounter(AbstractType::Ptr(new IntegralType(IntegralType::TypeBoolean))); } AbstractType::Ptr ExpressionVisitor::fromBinaryOperator(AbstractType::Ptr lhs, AbstractType::Ptr rhs, const QString& op) { DUChainReadLocker lock; auto operatorReturnType = [&op, this](const AbstractType::Ptr& p) { StructureType::Ptr type = p.cast(); if ( ! type ) { return AbstractType::Ptr(); } auto func = Helper::accessAttribute(type, op, topContext()); if ( ! func ) { return AbstractType::Ptr(); } auto operatorFunctionType = func->type(); DUChainReadLocker lock; auto context = Helper::getDocumentationFileContext(); if ( context ) { auto object_decl = context->findDeclarations(QualifiedIdentifier("object")); if ( ! object_decl.isEmpty() && object_decl.first()->internalContext() == func->context() ) { // if the operator is only declared in object(), do not include its type (which is void). return AbstractType::Ptr(); } } return operatorFunctionType ? operatorFunctionType->returnType() : AbstractType::Ptr(); }; return Helper::mergeTypes(operatorReturnType(lhs), operatorReturnType(rhs)); } void ExpressionVisitor::visitBinaryOperation(Python::BinaryOperationAst* node) { ExpressionVisitor lhsVisitor(this); ExpressionVisitor rhsVisitor(this); AbstractType::Ptr result; lhsVisitor.visitNode(node->lhs); rhsVisitor.visitNode(node->rhs); if ( lhsVisitor.lastType() && lhsVisitor.lastType()->whichType() == AbstractType::TypeUnsure ) { KDevelop::UnsureType::Ptr unsure = lhsVisitor.lastType().cast(); const IndexedType* types = unsure->types(); for( uint i = 0; i < unsure->typesSize(); i++ ) { result = Helper::mergeTypes(result, fromBinaryOperator(types[i].abstractType(), rhsVisitor.lastType(), node->methodName())); } } else { result = fromBinaryOperator(lhsVisitor.lastType(), rhsVisitor.lastType(), node->methodName()); } if ( ! Helper::isUsefulType(result) ) { result = Helper::mergeTypes(lhsVisitor.lastType(), rhsVisitor.lastType()); } return encounter(result); } void ExpressionVisitor::visitUnaryOperation(Python::UnaryOperationAst* node) { // Only visit the value, and use that as the result. Unary operators usually // don't change the type of the object (i.e. -a has the same type as a) visitNode(node->operand); } void ExpressionVisitor::visitBooleanOperation(Python::BooleanOperationAst* node) { ExpressionVisitor v(this); AbstractType::Ptr result; for (const auto& expr : node->values) { v.visitNode(expr); result = Helper::mergeTypes(result, v.lastType()); } encounter(result); } +void ExpressionVisitor::visitAssignmentExpression(Python::AssignmentExpressionAst* node) { + visitNode(node->value); +} + } diff --git a/duchain/expressionvisitor.h b/duchain/expressionvisitor.h index 25ca2319..a56481ad 100644 --- a/duchain/expressionvisitor.h +++ b/duchain/expressionvisitor.h @@ -1,155 +1,156 @@ /***************************************************************************** * Copyright 2010 Miquel Canes Gonzalez * * Copyright 2011-2014 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef EXPRESSIONVISITOR_H #define EXPRESSIONVISITOR_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include "astdefaultvisitor.h" #include "pythonduchainexport.h" #include "pythoneditorintegrator.h" #include "duchain/declarations/functiondeclaration.h" #include "duchain/helpers.h" using namespace KDevelop; class Identifier; namespace Python { typedef DUChainPointer FunctionDeclarationPointer; class KDEVPYTHONDUCHAIN_EXPORT ExpressionVisitor : public AstDefaultVisitor, public DynamicLanguageExpressionVisitor { public: ExpressionVisitor(const KDevelop::DUContext* ctx); /// Use this to construct the expression-visitor recursively ExpressionVisitor(Python::ExpressionVisitor* parent, const DUContext* overrideContext=nullptr); void visitBinaryOperation(BinaryOperationAst* node) override; void visitUnaryOperation(UnaryOperationAst* node) override; void visitBooleanOperation(BooleanOperationAst* node) override; void visitCompare(CompareAst* node) override; void visitString(StringAst* node) override; void visitBytes(BytesAst* node) override; void visitFormattedValue(FormattedValueAst * node) override; void visitJoinedString(JoinedStringAst* node) override; void visitNumber(NumberAst* node) override; void visitName(NameAst* node) override; void visitList(ListAst* node) override; void visitDict(DictAst* node) override; void visitSet(SetAst* node) override; void visitSubscript(SubscriptAst* node) override; void visitCall(CallAst* node) override; void visitAttribute(AttributeAst* node) override; void visitTuple(TupleAst* node) override; void visitLambda(LambdaAst* node) override; void visitListComprehension(ListComprehensionAst* node) override; void visitDictionaryComprehension(DictionaryComprehensionAst* node) override; void visitSetComprehension(SetComprehensionAst* node) override; void visitIfExpression(IfExpressionAst* node) override; void visitNameConstant(NameConstantAst* node) override; + void visitAssignmentExpression(AssignmentExpressionAst* node) override; /** * @brief Checks for magic docstrings that override a call's return type. * * @param node The node to visit. * @param normalType The return type as determined without docstrings. * @param docstring Docstring of the function. */ AbstractType::Ptr docstringTypeOverride(CallAst* node, const AbstractType::Ptr normalType, const QString& docstring); bool isAlias() const { return m_isAlias; } void enableGlobalSearching() { m_forceGlobalSearching = true; } void enableUnknownNameReporting() { m_reportUnknownNames = true; } void scanUntil(const CursorInRevision& end) { m_scanUntilCursor = end; } QSet unknownNames() const { return m_unknownNames; } template static TypePtr typeObjectForIntegralType(const QString& typeDescriptor) { auto context = Helper::getDocumentationFileContext(); if ( ! context ) { AbstractType::Ptr null; return null.cast(); } auto decls = context->findDeclarations(QualifiedIdentifier(typeDescriptor)); auto decl = decls.isEmpty() ? nullptr : dynamic_cast(decls.first()); auto type = decl ? decl->abstractType() : AbstractType::Ptr(); return type.cast(); } private: AbstractType::Ptr fromBinaryOperator(AbstractType::Ptr lhs, AbstractType::Ptr rhs, const QString& op); AbstractType::Ptr encounterPreprocess(AbstractType::Ptr type, bool merge=false); void encounter(AbstractType::Ptr type, DeclarationPointer declaration=DeclarationPointer(), bool alias=false); void addUnknownName(const QString& name); AbstractType::Ptr encounterPreprocess(AbstractType::Ptr type) override; void setLastIsAlias(bool alias) { m_isAlias = alias; } private: /// tells whether the returned declaration is an alias bool m_isAlias = false; /// used by code completion to disable range checks on declaration searches bool m_forceGlobalSearching = false; /// used by code completion to detect unknown NameAst elements in expressions bool m_reportUnknownNames = false; CursorInRevision m_scanUntilCursor = CursorInRevision::invalid(); static QHash m_defaultTypes; QSet m_unknownNames; }; } #endif // EXPRESSIONVISITOR_H diff --git a/duchain/tests/pyduchaintest.cpp b/duchain/tests/pyduchaintest.cpp index 7142ed2b..17ba34e5 100644 --- a/duchain/tests/pyduchaintest.cpp +++ b/duchain/tests/pyduchaintest.cpp @@ -1,1825 +1,1844 @@ /***************************************************************************** * Copyright 2010 (c) Miquel Canes Gonzalez * * Copyright 2012 (c) Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include "duchaindebug.h" #include "pyduchaintest.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" #include "kdevpythonversion.h" QTEST_MAIN(PyDUChainTest) using namespace KDevelop; using namespace Python; PyDUChainTest::PyDUChainTest(QObject* parent): QObject(parent) { assetsDir = QDir(DUCHAIN_PY_DATA_DIR); if (!assetsDir.cd("data")) { qFatal("Failed find data directory for test files. Aborting"); } testDir = QDir(testDirOwner.path()); qputenv("PYTHONPATH", assetsDir.absolutePath().toUtf8()); initShell(); } QList PyDUChainTest::FindPyFiles(QDir& rootDir) { QList foundfiles; rootDir.setFilter(QDir::Files | QDir::Dirs | QDir::NoDot | QDir::NoDotDot); rootDir.setNameFilters(QStringList() << "*.py"); // We only want .py files QDirIterator it(rootDir, QDirIterator::Subdirectories); while(it.hasNext()) { foundfiles.append(it.next()); } return foundfiles; } void PyDUChainTest::init() { QString currentTest = QString(QTest::currentTestFunction()); if (lastTest == currentTest) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Already prepared assets for " << currentTest << ", skipping"; return; } else { lastTest = currentTest; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Preparing assets for test " << currentTest; QDir assetModuleDir = QDir(assetsDir.absolutePath()); if (!assetModuleDir.cd(currentTest)) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Asset directory " << currentTest << " does not exist under " << assetModuleDir.absolutePath() << ". Skipping it."; return; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Searching for python files in " << assetModuleDir.absolutePath(); QList foundfiles = FindPyFiles(assetModuleDir); QString correctionFileDir = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files", QStandardPaths::LocateDirectory); auto correctionFileUrl = QUrl(QDir::cleanPath(correctionFileDir + "/testCorrectionFiles/example.py")); foundfiles.prepend(correctionFileUrl.path()); for ( int i = 0; i < 2; i++ ) { // Parse each file twice, to ensure no parsing-order related bugs appear. // Such bugs will need separate unit tests and should not influence these. foreach(const QString filename, foundfiles) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Parsing asset: " << filename; DUChain::self()->updateContextForUrl(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); } foreach(const QString filename, foundfiles) { DUChain::self()->waitForUpdate(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); } while ( ICore::self()->languageController()->backgroundParser()->queuedCount() > 0 ) { // make sure to wait for all parsejobs to finish QTest::qWait(10); } } } void PyDUChainTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); auto doc_url = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); qCDebug(KDEV_PYTHON_DUCHAIN) << doc_url; DUChain::self()->updateContextForUrl(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); DUChain::self()->waitForUpdate(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } ReferencedTopDUContext PyDUChainTest::parse(const QString& code) { TestFile* testfile = new TestFile(code + "\n", "py", nullptr, testDir.absolutePath().append("/")); createdFiles << testfile; testfile->parse((TopDUContext::Features) (TopDUContext::ForceUpdate | TopDUContext::AST) ); testfile->waitForParsed(2000); if ( testfile->isReady() ) { Q_ASSERT(testfile->topContext()); m_ast = static_cast(testfile->topContext()->ast().data())->ast; return testfile->topContext(); } else Q_ASSERT(false && "Timed out waiting for parser results, aborting all tests"); return nullptr; } PyDUChainTest::~PyDUChainTest() { foreach ( TestFile* f, createdFiles ) { delete f; } testDir.rmdir(testDir.absolutePath()); } void PyDUChainTest::testMultiFromImport() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList a = ctx->findDeclarations(QualifiedIdentifier("a")); QList b = ctx->findDeclarations(QualifiedIdentifier("b")); QVERIFY(! a.isEmpty()); QVERIFY(! b.isEmpty()); QVERIFY(a.first()->abstractType()->toString().endsWith("int")); QVERIFY(b.first()->abstractType()->toString().endsWith("int")); } void PyDUChainTest::testMultiFromImport_data() { QTest::addColumn("code"); QTest::newRow("multiimport") << "import testMultiFromImport.i.localvar1\n" "import testMultiFromImport.i.localvar2\n" "a = testMultiFromImport.i.localvar1\n" "b = testMultiFromImport.i.localvar2\n"; } void PyDUChainTest::testRelativeImport() { QFETCH(QString, code); QFETCH(QString, token); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList t = ctx->findDeclarations(QualifiedIdentifier(token)); QVERIFY(! t.isEmpty()); QVERIFY(t.first()->abstractType()->toString().endsWith(type)); } void PyDUChainTest::testRelativeImport_data() { QTest::addColumn("code"); QTest::addColumn("token"); QTest::addColumn("type"); QTest::newRow(".local") << "from testRelativeImport.m.sm1.go import i1" << "i1" << "int"; QTest::newRow(".init") << "from testRelativeImport.m.sm1.go import i2" << "i2" << "int"; QTest::newRow("..local") << "from testRelativeImport.m.sm1.go import i3" << "i3" << "int"; QTest::newRow("..init") << "from testRelativeImport.m.sm1.go import i4" << "i4" << "int"; QTest::newRow("..sub.local") << "from testRelativeImport.m.sm1.go import i5" << "i5" << "int"; QTest::newRow("..sub.init") << "from testRelativeImport.m.sm1.go import i6" << "i6" << "int"; } void PyDUChainTest::testImportFiles() { QString code = "import testImportFiles\nk = testImportFiles.fromInit()\np = testImportFiles.other.fromOther()"; ReferencedTopDUContext ctx = parse(code.toUtf8()); DUChainReadLocker lock; QVERIFY(ctx); auto k = ctx->findDeclarations(QualifiedIdentifier("k")); auto p = ctx->findDeclarations(QualifiedIdentifier("p")); QCOMPARE(k.size(), 1); QCOMPARE(p.size(), 1); QVERIFY(k.first()->abstractType()); QCOMPARE(k.first()->abstractType()->toString(), QString("fromInit")); QCOMPARE(p.first()->abstractType()->toString(), QString("fromOther")); } void PyDUChainTest::testCrashes() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); QVERIFY(! m_ast->body.isEmpty()); } void PyDUChainTest::testCrashes_data() { QTest::addColumn("code"); QTest::newRow("unicode_char") << "a = \"í\""; QTest::newRow("unicode escape char") << "print(\"\\xe9\")"; QTest::newRow("augassign") << "a = 3\na += 5"; QTest::newRow("delete") << "a = 3\ndel a"; QTest::newRow("double_comprehension") << "q = [[x for x in a] + [x for x in a] for y in b]"; QTest::newRow("for_else") << "for i in range(3): pass\nelse: pass"; QTest::newRow("for_while") << "while i < 4: pass\nelse: pass"; QTest::newRow("ellipsis") << "a[...]"; QTest::newRow("tuple_assign_unknown") << "foo = (unknown, unknown, unknown)"; QTest::newRow("for_assign_unknown") << "for foo, bar, baz in unknown: pass"; QTest::newRow("negative slice index") << "t = (1, 2, 3)\nd = t[-1]"; QTest::newRow("decorator_with_args") << "@foo('bar', 'baz')\ndef myfunc(): pass"; QTest::newRow("non_name_decorator") << "@foo.crazy_decorators\ndef myfunc(): pass"; QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass"; QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)"; QTest::newRow("whatever") << "for attr in updated:\n " " getattr.update"; QTest::newRow("return_outside_function") << "return 3"; QTest::newRow("return_context_outside_function") << "return [x for x in range(3)]"; QTest::newRow("paren_attrib_access") << "a = (xxx or yyy).zzz"; QTest::newRow("func_call") << "a = xxx.func(yyy.zzz)"; QTest::newRow("comprehension_attrib") << "a = [foo for foo in bar].baz"; QTest::newRow("comprehension_attrib2") << "a = [foo.bar for foo in bar]"; QTest::newRow("lambda_cmpr_defarg") << "a = lambda foo=[b for b in (1, 2, 3)]: foo"; QTest::newRow("attrib") << "(sep or ' ').join(xxxx.capitalize() for xxxx in ssss.split(sep))"; QTest::newRow("attrib2") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))"; QTest::newRow("attrib3") << "known_threads = {line.strip()}"; QTest::newRow("attrib4") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}"; QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()"; QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n" ".format(type(meth))) from None"; QTest::newRow("newline_attrib") << "some_instance \\\n" ". attr1 \\\n" ".funcfunc(argarg, arg2arg) \\\n" ".foo"; QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n" "in ambiguous_nucleotide_values.iteritems() \\\n" "if set([codon[0] for codon in codons]).issuperset(set(meanings)))"; QTest::newRow("fancy class range") << "class SchemeLexer(RegexLexer):\n" " valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+'\n" "\n" " tokens = {\n" " 'root' : [\n" " # the comments - always starting with semicolon\n" " # and going to the end of the line\n" " (r';.*$', Comment.Single),\n" "\n" " # whitespaces - usually not relevant\n" " (r'\\s+', Text),\n" "\n" " # numbers\n" " (r'-?\\d+\\.\\d+', Number.Float),\n" " (r'-?\\d+', Number.Integer)\n" " ],\n" " }\n"; QTest::newRow("another fancy range") << "setup_args['data_files'] = [\n" " (os.path.dirname(os.path.join(install_base_dir, pattern)),\n" " [ f for f in glob.glob(pattern) ])\n" " for pattern in patterns\n" "]\n"; QTest::newRow("kwarg_empty_crash") << "def myfun(): return\ncheckme = myfun(kw=something)"; QTest::newRow("stacked_tuple_hang") << "tree = (1,(2,(3,(4,(5,'Foo')))))"; QTest::newRow("stacked_tuple_hang2") << "tree = (257," "(264," "(285," "(259," "(272," "(275," "(1, 'return')))))))"; QTest::newRow("very_large_tuple_hang") << "tree = " "(257," "(264," "(285," "(259," "(1, 'def')," "(1, 'f')," "(260, (7, '('), (8, ')'))," "(11, ':')," "(291," "(4, '')," "(5, '')," "(264," "(265," "(266," "(272," "(275," "(1, 'return')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302, (303, (304, (305, (2, '1'))))))))))))))))))," "(264," "(265," "(266," "(272," "(276," "(1, 'yield')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302," "(303, (304, (305, (2, '1'))))))))))))))))))," "(4, '')))," "(6, '')))))," "(4, '')," "(0, ''))))"; QTest::newRow("attribute_hang") << "s = \"123\"\n" "s = s.replace(u'ł', 'l').\\\n" "replace(u'ó', 'o').\\\n" "replace(u'ą', 'a').\\\n" "replace(u'ę', 'e').\\\n" "replace(u'ś', 's').\\\n" "replace(u'ż', 'z').\\\n" "replace(u'ź', 'z').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo')\n"; QTest::newRow("function context range crash") << "def myfunc(arg):\n foo = 3 + \\\n[x for x in range(20)]"; QTest::newRow("decorator comprehension crash") << "@implementer_only(interfaces.ISSLTransport,\n" " *[i for i in implementedBy(tcp.Client)\n" " if i != interfaces.ITLSTransport])\n" "class Client(tcp.Client):\n" " pass\n"; QTest::newRow("comprehension_as_default_crash") << "def foo(bar = [item for (_, item) in items()]):\n return"; QTest::newRow("try_except") << "try: pass\nexcept: pass"; QTest::newRow("try_except_type") << "try: pass\nexcept FooException: pass"; QTest::newRow("try_except_type_as") << "try: pass\nexcept FooException as bar: pass"; QTest::newRow("import_missing") << "from this_does_not_exist import nor_does_this"; QTest::newRow("list_append_missing") << "foo = []\nfoo.append(missing)"; QTest::newRow("list_append_missing_arg") << "foo = []\nfoo.append()"; QTest::newRow("list_extend_missing") << "foo = []\nfoo.extend(missing)"; QTest::newRow("list_extend_missing_arg") << "foo = []\nfoo.extend()"; QTest::newRow("method_of_call_with_list_arg") << "class MyClass:\n" " def bar(self): pass\n" "def foo(x):\n" " return MyClass()\n" "foo([0]).bar()"; QTest::newRow("unpacked_dict_kwarg") << "def foo(arg): pass\nfoo(**{'arg': 2})"; QTest::newRow("negative_container_hints") << "class Evil:\n" " def aa(self, arg):\n" " \"\"\"! addsTypeOfArgContent ! -1\"\"\"\n" " def bb(self, arg):\n" " \"\"\"! addsTypeOfArg ! -2\"\"\"\n" " def cc(self, arg):\n" " \"\"\"! returnContentEqualsContentOf ! -3\"\"\"\n" "e = Evil()\n" "z = [e.aa(1), e.bb(2), e.cc(3)]"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("comprehension_in_fstring") << "def crash(): return f'expr={ {x: y for x, y in [(1, 2), ]}}'"; #endif QTest::newRow("comprehension_in_lambda") << "lambda foo: [bar for bar in foo]"; QTest::newRow("comprehension_in_annassign_1") << "foo: int = [x for x in (42,)][0]"; QTest::newRow("comprehension_in_annassign_2") << "foo: [t for t in (int,)][0] = 42"; QTest::newRow("lambda_in_annassign_1") << "foo: int = (lambda: 42)()"; QTest::newRow("lambda_in_annassign_2") << "foo: (lambda: int)() = 42"; QTest::newRow("definition_in_baseclass_1") << "class Foo(lambda x: 1): pass"; QTest::newRow("definition_in_baseclass_2") << "class Foo([x for x in (1, 2)]): pass"; } void PyDUChainTest::testClassVariables() { ReferencedTopDUContext ctx = parse("class c():\n myvar = 3;\n def meth(self):\n print(myvar)"); QVERIFY(ctx.data()); DUChainWriteLocker lock(DUChain::lock()); CursorInRevision relevantPosition(3, 10); DUContext* c = ctx->findContextAt(relevantPosition); QVERIFY(c); int useIndex = c->findUseAt(relevantPosition); if ( useIndex != -1 ) { QVERIFY(useIndex < c->usesCount()); const Use* u = &(c->uses()[useIndex]); QVERIFY(!u->usedDeclaration(c->topContext())); } } void PyDUChainTest::testWarnNewNotCls() { QFETCH(QString, code); QFETCH(int, probs); ReferencedTopDUContext ctx = parse(code); DUChainReadLocker lock; QCOMPARE(ctx->problems().count(), probs); } void PyDUChainTest::testWarnNewNotCls_data() { QTest::addColumn("code"); QTest::addColumn("probs"); QTest::newRow("check_for_new_first_arg_cls") << "class c():\n def __new__(clf, other):\n pass" << 1; QTest::newRow("check_for_new_first_arg_cls_0") << "class c():\n def __new__(cls, other):\n pass" << 0; QTest::newRow("check_first_arg_class_self") << "class c():\n def test(seff, masik):\n pass" << 1; QTest::newRow("check_first_arg_class_self_0") << "class c():\n def test(self, masik):\n pass" << 0; } // this is actually for both binary and boolean operators void PyDUChainTest::testBinaryOperatorsUnsure() { QFETCH(QString, code); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), type); } void PyDUChainTest::testBinaryOperatorsUnsure_data() { QTest::addColumn("code"); QTest::addColumn("type"); QTest::newRow("check_unsure_type_0") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3.5;\ny = 3;\ncheckme = x * y;" << "unsure (float, int)"; QTest::newRow("check_unsure_type_1") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_2") << "class c():\n pass;\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_3") << "class c():\n pass;\nclass d():\n pass;\nx = c();\nx = d();\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_4") << "checkme = True or False" << "bool"; QTest::newRow("check_unsure_type_5") << "a = 'foo'; checkme = a or 'bar';" << "str"; QTest::newRow("check_unsure_type_6") << "class A(): pass\nclass B(): pass;\ncheckme = A() or B() or 42;" << "unsure (A, B, int)"; } void PyDUChainTest::testFlickering() { QFETCH(QStringList, code); QFETCH(int, before); QFETCH(int, after); TestFile f(code[0], "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx = f.topContext(); QVERIFY(ctx); DUChainWriteLocker lock(DUChain::lock()); int count = ctx->localDeclarations().size(); qDebug() << "Declaration count before: " << count; QVERIFY(count == before); lock.unlock(); f.setFileContents(code[1]); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ctx = f.topContext(); QVERIFY(ctx); lock.lock(); count = ctx->localDeclarations().size(); qDebug() << "Declaration count afterwards: " << count; QVERIFY(count == after); foreach(Declaration* dec, ctx->localDeclarations()) { qDebug() << dec->toString() << dec->range(); qDebug() << dec->uses().size(); } } void PyDUChainTest::testFlickering_data() { QTest::addColumn("code"); QTest::addColumn("before"); QTest::addColumn("after"); QTest::newRow("declaration_flicker") << ( QStringList() << "a=2\n" << "b=3\na=2\n" ) << 1 << 2; } void PyDUChainTest::testCannotOverwriteBuiltins() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testCannotOverwriteBuiltins_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("list_assign") << "class list(): pass\ncheckme = []\ncheckme.append(3)" << "list of int"; QTest::newRow("str_assign") << "str = 5; checkme = 'Foo'" << "str"; QTest::newRow("str_assign2") << "class Foo: pass\nstr = Foo; checkme = 'Foo'" << "str"; QTest::newRow("str_assign3") << "from testCannotOverwriteBuiltins.i import Foo as str\ncheckme = 'Foo'" << "str"; QTest::newRow("for") << "for str in [1, 2, 3]: pass\ncheckme = 'Foo'" << "str"; QTest::newRow("assert") << "assert isinstance(str, int)\ncheckme = 'Foo'" << "str"; QTest::newRow("assert2") << "assert isinstance(str, int)\ncheckme = 3" << "int"; QTest::newRow("can_have_custom") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open()" << "int"; QTest::newRow("can_have_custom2") << "from testCannotOverwriteBuiltins import mod\ncheckme = open().read()" << "str"; QTest::newRow("can_have_custom3") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open().read()" << "mixed"; } void PyDUChainTest::testVarKWArgs() { ReferencedTopDUContext ctx = parse("def myfun(arg, *vararg, **kwarg):\n pass\n pass"); DUChainWriteLocker lock; QVERIFY(ctx); DUContext* func = ctx->findContextAt(CursorInRevision(1, 0)); QVERIFY(func); QVERIFY(! func->findDeclarations(QualifiedIdentifier("arg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("vararg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("kwarg")).isEmpty()); QVERIFY(func->findDeclarations(QualifiedIdentifier("vararg")).first()->abstractType()->toString().startsWith("tuple")); QCOMPARE(func->findDeclarations(QualifiedIdentifier("kwarg")).first()->abstractType()->toString(), QString("dict of str : unknown")); } void PyDUChainTest::testSimple() { QFETCH(QString, code); QFETCH(int, decls); QFETCH(int, uses); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVector< Declaration* > declarations = ctx->localDeclarations(); QCOMPARE(declarations.size(), decls); int usesCount = 0; foreach(Declaration* d, declarations) { usesCount += d->uses().size(); QVERIFY(d->abstractType()); } QCOMPARE(usesCount, uses); } void PyDUChainTest::testSimple_data() { QTest::addColumn("code"); QTest::addColumn("decls"); QTest::addColumn("uses"); QTest::newRow("assign") << "b = 2;" << 1 << 0; QTest::newRow("assign_str") << "b = 'hola';" << 1 << 0; QTest::newRow("op") << "a = 3; b = a+2;" << 2 << 1; QTest::newRow("bool") << "a = True" << 1 << 0; QTest::newRow("op") << "a = True and True;" << 1 << 0; } class AttributeRangeTestVisitor : public AstDefaultVisitor { public: bool found; KTextEditor::Range searchingForRange; QString searchingForIdentifier; void visitAttribute(AttributeAst* node) override { auto r = KTextEditor::Range(0, node->startCol, 0, node->endCol); qDebug() << "Found attr: " << r << node->attribute->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->attribute->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitAttribute(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found func: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; qDebug() << node->arguments->vararg << node->arguments->kwarg; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } if ( node->arguments->vararg ) { auto r = KTextEditor::Range(0, node->arguments->vararg->startCol, 0, node->arguments->vararg->startCol+node->arguments->vararg->argumentName->value.length()); qDebug() << "Found vararg: " << node->arguments->vararg->argumentName->value << r; if ( r == searchingForRange && node->arguments->vararg->argumentName->value == searchingForIdentifier ) { found = true; return; } } if ( node->arguments->kwarg ) { auto r = KTextEditor::Range(0, node->arguments->kwarg->startCol, 0, node->arguments->kwarg->startCol+node->arguments->kwarg->argumentName->value.length()); qDebug() << "Found kwarg: " << node->arguments->kwarg->argumentName->value << r; if ( r == searchingForRange && node->arguments->kwarg->argumentName->value == searchingForIdentifier ) { found = true; return; } } AstDefaultVisitor::visitFunctionDefinition(node); } void visitClassDefinition(ClassDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found cls: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitClassDefinition(node); } void visitImport(ImportAst* node) override { foreach ( const AliasAst* name, node->names ) { if ( name->name ) { qDebug() << "found import" << name->name->value << name->name->range(); } if ( name->name && name->name->value == searchingForIdentifier && name->name->range() == searchingForRange ) { found = true; return; } if ( name->asName ) { qDebug() << "found import" << name->asName->value << name->asName->range(); } if ( name->asName && name->asName->value == searchingForIdentifier && name->asName->range() == searchingForRange ) { found = true; return; } } } }; void PyDUChainTest::testRanges() { QFETCH(QString, code); QFETCH(int, expected_amount_of_variables); Q_UNUSED(expected_amount_of_variables); QFETCH(QStringList, column_ranges); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); for ( int i = 0; i < column_ranges.length(); i++ ) { int scol = column_ranges.at(i).split(",")[0].toInt(); int ecol = column_ranges.at(i).split(",")[1].toInt(); QString identifier = column_ranges.at(i).split(",")[2]; auto r = KTextEditor::Range(0, scol, 0, ecol); AttributeRangeTestVisitor* visitor = new AttributeRangeTestVisitor(); visitor->searchingForRange = r; visitor->searchingForIdentifier = identifier; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("attr_dot_name_hash", "Insufficiently magic hack", Continue); QCOMPARE(visitor->found, true); delete visitor; } } void PyDUChainTest::testRanges_data() { QTest::addColumn("code"); QTest::addColumn("expected_amount_of_variables"); QTest::addColumn("column_ranges"); QTest::newRow("attr_two_attributes") << "base.attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_binary") << "base.attr + base.attr" << 4 << ( QStringList() << "5,8,attr" << "17,20,attr" ); QTest::newRow("attr_same") << "aaa.aaa.aaa + aaa.aaa.aaa" << 6 << ( QStringList() << "4,6,aaa" << "8,10,aaa" << "18,20,aaa" << "22,24,aaa" ); QTest::newRow("attr_three_attributes") << "base.attr.subattr" << 3 << ( QStringList() << "5,8,attr" << "10,16,subattr" ); QTest::newRow("attr_functionCall") << "base.attr().subattr" << 3 << ( QStringList() << "5,8,attr" << "12,18,subattr" ); QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" ); QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" ); QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" ); QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" << "25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr"); QTest::newRow("attr_two_in_call") << "func(inst.aaa, inst.bbbb)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,bbbb" ); QTest::newRow("attr_two_in_call_same") << "func(inst.aaa, inst.aaaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,aaaa" ); QTest::newRow("attr_two_in_call_same2") << "func(inst.aaa, inst.aaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,22,aaa" ); QTest::newRow("attr_of_string_slash") << "'/'.join(a)" << 1 << ( QStringList() << "4,7,join" ); QTest::newRow("attr_of_string_in_list") << "[\"*{0}*\".format(foo)]" << 1 << ( QStringList() << "9,14,format" ); QTest::newRow("attr_of_call_in_list") << "[foo().format(foo)]" << 1 << ( QStringList() << "7,12,format" ); QTest::newRow("attr_parentheses") << "(\"foo\" + \"foo\").capitalize()" << 1 << ( QStringList() << "16,25,capitalize" ); QTest::newRow("attr_commented_name") << "base.attr # attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_name_in_strings") << "'attr' + base['attr'].attr # attr" << 4 << ( QStringList() << "22,25,attr" ); QTest::newRow("attr_dot_hash_in_strings") << "'.foo#' + base['.#'].attr # attr" << 4 << ( QStringList() << "21,24,attr" ); QTest::newRow("attr_dot_name_hash") << "base['.attr#'].attr" << 4 << ( QStringList() << "15,18,attr" ); QTest::newRow("string_parentheses") << "(\"asdf\".join())" << 1 << ( QStringList() << "8,11,join" ); QTest::newRow("string_parentheses2") << "(\"asdf\").join()" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("string_parentheses3") << "(\"asdf\".join()).join()" << 2 << ( QStringList() << "8,11,join" << "16,19,join" ); QTest::newRow("string_parentheses4") << "(\"asdf\".join()+2).join()" << 2 << ( QStringList() << "8,11,join" << "18,21,join" ); QTest::newRow("string_parentheses_call") << "f(\"asdf\".join())" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" ); QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" ); QTest::newRow("classdef_range") << "class cls(): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance") << "class cls(parent1, parent2): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance_spaces") << "class cls( parent1, parent2 ):pass" << 1 << ( QStringList() << "12,14,cls" ); QTest::newRow("vararg_kwarg") << "def func(*vararg, **kwargs): pass" << 2 << ( QStringList() << "10,16,vararg" << "20,26,kwargs" ); QTest::newRow("import") << "import sys" << 1 << ( QStringList() << "7,10,sys" ); QTest::newRow("import2") << "import i.localvar1" << 1 << ( QStringList() << "7,18,i.localvar1" ); QTest::newRow("import3") << "import sys as a" << 1 << ( QStringList() << "13,14,a" ); } class TypeTestVisitor : public AstDefaultVisitor { public: QString searchingForType; TopDUContextPointer ctx; bool found; void visitName(NameAst* node) override { if ( node->identifier->value != "checkme" ) return; QList decls = ctx->findDeclarations(QualifiedIdentifier(node->identifier->value)); if ( ! decls.length() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration found for " << node->identifier->value; return; } Declaration* d = decls.last(); QVERIFY(d->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << "found: " << node->identifier->value << "is" << d->abstractType()->toString() << "should be" << searchingForType; if ( d->abstractType()->toString().replace("__kdevpythondocumentation_builtin_", "").startsWith(searchingForType) ) { found = true; return; } }; }; void PyDUChainTest::testTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("tuple_func", "no suitable docstring hint", Continue); QEXPECT_FAIL("tuple_add", "not implemented", Continue); QEXPECT_FAIL("tuple_mul", "not implemented", Continue); QEXPECT_FAIL("return_builtin_iterator", "fake builtin iter()", Continue); QEXPECT_FAIL("parent_constructor_arg_type", "Not enough passes?", Continue); QEXPECT_FAIL("init_class_no_decl", "aliasing info lost", Continue); QEXPECT_FAIL("property_wrong", "visitCall uses declaration if no type", Continue); QEXPECT_FAIL("property_setter", "very basic property support", Continue); + QEXPECT_FAIL("assignment_expr_context", "not implemented", Continue); QCOMPARE(visitor->found, true); } void PyDUChainTest::testTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("annotate_decl") << "checkme: int" << "int"; QTest::newRow("annotate_assign") << "checkme: int = 3.5" << "unsure (float, int)"; #endif QTest::newRow("listtype") << "checkme = []" << "list"; QTest::newRow("listtype_func") << "checkme = list()" << "list"; QTest::newRow("listtype_with_contents") << "checkme = [1, 2, 3, 4, 5]" << "list of int"; QTest::newRow("listtype_extended") << "some_misc_var = []; checkme = some_misc_var" << "list"; QTest::newRow("dicttype") << "checkme = {}" << "dict"; QTest::newRow("dicttype_get") << "d = {0.4:5}; checkme = d.get(0)" << "int"; QTest::newRow("dicttype_func") << "checkme = dict()" << "dict"; QTest::newRow("dicttype_extended") << "some_misc_var = {}; checkme = some_misc_var" << "dict"; QTest::newRow("tuple") << "checkme = ()" << "tuple of ()"; QTest::newRow("tuple_func") << "checkme = tuple((1, 2.3))" << "tuple of (int, float)"; QTest::newRow("tuple_with_contents") << "checkme = 1, 2.3" << "tuple of (int, float)"; QTest::newRow("tuple_extended") << "some_misc_var = (); checkme = some_misc_var" << "tuple of ()"; QTest::newRow("tuple_max_display") << "checkme = 1,2,3,4,5,6" << "tuple of (int, int, int, int, int, ...)"; QTest::newRow("bool") << "checkme = True" << "bool"; QTest::newRow("float") << "checkme = 3.7" << "float"; QTest::newRow("int") << "checkme = 3" << "int"; QTest::newRow("str") << "checkme = \"foo\"" << "str"; QTest::newRow("bytes") << "checkme = b\"foo\"" << "bytes"; QTest::newRow("function_arg_scope") << "class Foo:\n" " a = 3\n" " def func(self, x=a):\n" " return x\n" "f = Foo()\n" "checkme = f.func()" << "int"; QTest::newRow("with") << "with open('foo') as f: checkme = f.read()" << "str"; QTest::newRow("with_list_target") << "bar = [1, 2, 3]\n" "with open('foo') as bar[1]: checkme = bar[1].read()" << "str"; QTest::newRow("with_attr_target") << "bar = object()\n" "with open('foo') as bar.zep: checkme = bar.zep.read()" << "str"; QTest::newRow("with_nonself_enter") << // From https://bugs.kde.org/show_bug.cgi?id=399534 "class Mgr:\n" " def __enter__(self): return 42\n" " def __exit__(self, *args): pass\n" "with Mgr() as asd:\n" " checkme = asd" << "int"; QTest::newRow("with_tuple_target") << "class Mgr:\n" " def __enter__(self): return (42, 3.4)\n" " def __exit__(self, *args): pass\n" "with Mgr() as (aa, bb):\n" " checkme = bb" << "float"; QTest::newRow("arg_after_vararg") << "def func(x, y, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("arg_after_vararg_with_default") << "def func(x=5, y=3, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("class_scope_end_inside") << "a = str()\nclass M:\n" " a = 2\n foo = a\n" "checkme = M().foo" << "int"; QTest::newRow("class_scope_end_outside") << "a = str()\nclass M:\n a = 2\ncheckme = a" << "str"; QTest::newRow("list_access_right_open_slice") << "some_list = []; checkme = some_list[2:]" << "list"; QTest::newRow("list_access_left_open_slice") << "some_list = []; checkme = some_list[:2]" << "list"; QTest::newRow("list_access_closed_slice") << "some_list = []; checkme = some_list[2:17]" << "list"; QTest::newRow("list_access_step") << "some_list = []; checkme = some_list[::2]" << "list"; QTest::newRow("list_access_singleItem") << "some_list = []; checkme = some_list[42]" << "mixed"; QTest::newRow("funccall_number") << "def foo(): return 3; \ncheckme = foo();" << "int"; QTest::newRow("funccall_string") << "def foo(): return 'a'; \ncheckme = foo();" << "str"; QTest::newRow("funccall_list") << "def foo(): return []; \ncheckme = foo();" << "list"; QTest::newRow("funccall_dict") << "def foo(): return {}; \ncheckme = foo();" << "dict"; QTest::newRow("funccall_no_return") << "def foo(): pass\ncheckme = foo()" << "None"; QTest::newRow("funccall_def_return") << "def foo(): return\ncheckme = foo()" << "None"; QTest::newRow("funccall_maybe_def_return") << "def foo():\n if False: return\n return 7\ncheckme = foo()" << "unsure (None, int)"; QTest::newRow("tuple1") << "checkme, foo = 3, \"str\"" << "int"; QTest::newRow("tuple2") << "foo, checkme = 3, \"str\"" << "str"; QTest::newRow("tuple2_negative_index") << "foo = (1, 2, 'foo')\ncheckme = foo[-1]" << "str"; QTest::newRow("tuple_type") << "checkme = 1, 2" << "tuple"; QTest::newRow("tuple_rhs_unpack") << "foo = 1, 2.5\nbar = 1, *foo, 2\ncheckme = bar[2]" << "float"; QTest::newRow("dict_iteritems") << "d = {1:2, 3:4}\nfor checkme, k in d.iteritems(): pass" << "int"; QTest::newRow("enumerate_key") << "d = [str(), str()]\nfor checkme, value in enumerate(d): pass" << "int"; QTest::newRow("enumerate_value") << "d = [str(), str()]\nfor key, checkme in enumerate(d): pass" << "str"; QTest::newRow("dict_enumerate") << "d = {1:2, 3:4}\nfor key, checkme in enumerate(d.values()): pass" << "int"; QTest::newRow("dict_assign_twice") << "d = dict(); d[''] = 0; d = dict(); d[''] = 0; checkme = d" << "unsure (dict of str : int, dict)"; QTest::newRow("class_method_import") << "class c:\n attr = \"foo\"\n def m():\n return attr;\n return 3;\ni=c()\ncheckme=i.m()" << "int"; QTest::newRow("getsListDocstring") << "foo = [1, 2, 3]\ncheckme = foo.reverse()" << "list of int"; QTest::newRow("str_iter") << "checkme = [char for char in 'Hello, world!']" << "list of str"; QTest::newRow("str_subscript") << "checkme = 'Hello, world!'[0]" << "str"; QTest::newRow("bytes_iter") << "checkme = [byte for byte in b'Hello, world!']" << "list of int"; QTest::newRow("bytes_subscript") << "checkme = b'Hello, world!'[0]" << "int"; QTest::newRow("fromAssertIsinstance") << "class c(): pass\ncheckme = mixed()\nassert isinstance(checkme, c)\n" << "c"; QTest::newRow("fromAssertIsinstanceInvalid") << "class c(): pass\ncheckme = mixed()\nassert isinstance(c, checkme)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid2") << "class c(): pass\ncheckme = mixed()\nassert isinstance(D, c)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid3") << "checkme = int()\nassert isinstance(checkme, X)\n" << "int"; QTest::newRow("fromAssertIsinstanceInvalid4") << "checkme = int()\nassert isinstance(checkme)\n" << "int"; QTest::newRow("fromAssertType") << "class c(): pass\ncheckme = mixed()\nassert type(checkme) == c\n" << "c"; QTest::newRow("fromIfType") << "class c(): pass\ncheckme = mixed()\nif type(checkme) == c: pass\n" << "c"; QTest::newRow("fromIfIsinstance") << "class c(): pass\ncheckme = mixed()\nif isinstance(checkme, c): pass\n" << "c"; QTest::newRow("diff_local_classattr") << "class c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; QTest::newRow("diff_local_classattr2") << "local=3\nclass c(): attr = 1\ninst=c()\ncheckme = c.local" << "mixed"; QTest::newRow("diff_local_classattr3") << "attr=3.5\nclass c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; // QTest::newRow("class_method_self") << "class c:\n def func(checkme, arg, arg2):\n pass\n" << "c"; // QTest::newRow("funccall_dict") << "def foo(): return foo; checkme = foo();" << (uint) IntegralType::TypeFunction; // With only one subbed value we get a FormattedValue node QTest::newRow("fstring_formattedvalue") << "name = 'Jim'; checkme = f'{name}'" << "str"; // Otherwise a JoinedString, with FormattedValues as children. QTest::newRow("fstring_joinedstring") << "name = 'Jim'; checkme = f'Hello, {name}! Your name is {name}.'" << "str"; QTest::newRow("tuple_simple") << "mytuple = 3, 5.5\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_simple2") << "mytuple = 3, 5.5\nfoobar, checkme = mytuple" << "float"; QTest::newRow("tuple_simple3") << "mytuple = 3, 5.5, \"str\", 3, \"str\"\na, b, c, d, checkme = mytuple" << "str"; QTest::newRow("tuple_single") << "checkme = 4," << "tuple"; QTest::newRow("tuple_single2") << "checkme, = 4," << "int"; QTest::newRow("tuple_single3") << "mytuple = 4,\ncheckme, = mytuple" << "int"; QTest::newRow("tuple_ext_unpack") << "mytuple = 3, 5.5\nfoobar, *starred, checkme = mytuple" << "float"; QTest::newRow("tuple_ext_unpack2") << "mytuple = 3, 5.5\nfoobar, *checkme, another = mytuple" << "list"; QTest::newRow("tuple_ext_unpack3") << "mytuple = 3, 5.5\nfoobar, *checkme = mytuple" << "list of float"; QTest::newRow("tuple_ext_unpack4") << "mytuple = 3, 5.5\n*checkme, = mytuple" << "list of unsure (int, float)"; QTest::newRow("tuple_nested") << "mytuple = 3, ('foo', 5.5)\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_nested2") << "mytuple = 3, ('foo', 5.5)\nfoobar, (checkme, other) = mytuple" << "str"; QTest::newRow("tuple_nested3") << "mytuple = ((7, 'foo'), 5.5), 3\n((baz, checkme), other), foo = mytuple" << "str"; QTest::newRow("tuple_nested_ext") << "mytuple = (2, ('foo', 'bar', 6), 7)\na, (b, *checkme, c), *d = mytuple" << "list of str"; QTest::newRow("tuple_multi_assign") << "mytuple = 2, 'foo'\ncheckme = a = mytuple" << "tuple"; QTest::newRow("tuple_multi_assign2") << "mytuple = 2, 'foo'\ncheckme, a = b = mytuple" << "int"; QTest::newRow("list_unpack") << "mylist = [1, 2, 3]\ncheckme, b, c = mylist" << "int"; QTest::newRow("list_unpack2") << "mylist = [1, 'x', 3]\ncheckme, b, c = mylist" << "unsure (int, str)"; QTest::newRow("list_ext_unpack") << "mylist = [1, 2, 3]\n*checkme, foo = mylist" << "list of int"; QTest::newRow("list_ext_unpack2") << "mylist = [1, 'x', 3]\n*checkme, foo = mylist" << "list of unsure (int, str)"; QTest::newRow("if_expr_sure") << "checkme = 3 if 7 > 9 else 5" << "int"; QTest::newRow("unary_op") << "checkme = -42" << "int"; QTest::newRow("tuple_funcret") << "def myfun(): return 3, 5\ncheckme, a = myfun()" << "int"; QTest::newRow("tuple_funcret2") << "def myfun():\n t = 3, 5\n return t\ncheckme, a = myfun()" << "int"; QTest::newRow("yield") << "def myfun():\n yield 3\ncheckme = myfun()" << "list of int"; QTest::newRow("yield_twice") << "def myfun():\n yield 3\n yield 'foo'\ncheckme = myfun()" << "list of unsure (int, str)"; // this is mostly a check that it doesn't crash QTest::newRow("yield_return") << "def myfun():\n return 3\n yield 'foo'\ncheckme = myfun()" << "unsure (int, list of str)"; QTest::newRow("lambda") << "x = lambda t: 3\ncheckme = x()" << "int"; QTest::newRow("lambda_failure") << "x = lambda t: 3\ncheckme = t" << "mixed"; QTest::newRow("function_arg_tuple") << "def func(*arg):\n foo, bar = arg\n return bar\ncheckme = func(3, 5)" << "int"; QTest::newRow("function_arg_tuple2") << "def func(*arg):\n return arg[-1]\ncheckme = func(3, \"Foo\")" << "str"; QTest::newRow("tuple_indexaccess") << "t = 3, 5.5\ncheckme = t[0]" << "int"; QTest::newRow("tuple_indexaccess2") << "t = 3, 5.5\ncheckme = t[1]" << "float"; QTest::newRow("tuple_indexaccess3") << "t = 3, 4\ncheckme = t[1]" << "int"; QTest::newRow("tuple_indexaccess4") << "t = 3, 4.5\ncheckme = t[2]" << "unsure (int, float)"; QTest::newRow("tuple_indexaccess_neg") << "t = 3, 4.5; checkme = t[-1]" << "float"; QTest::newRow("tuple_indexaccess_neg2") << "t = 3, 4.5; checkme = t[-2]" << "int"; QTest::newRow("tuple_indexaccess_neg3") << "t = 3, 4.5; checkme = t[-3]" << "unsure (int, float)"; QTest::newRow("tuple_slice") << "t = 3, 'q', 4.5; checkme = t[-3: 2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_normal") << "t = 1, 2.3, 'a', {}; checkme = t[1:3]" << "tuple of (float, str)"; QTest::newRow("tuple_slice_defstart") << "t = 1, 2.3, 'a', {}; checkme = t[:3]" << "tuple of (int, float, str)"; QTest::newRow("tuple_slice_defstop") << "t = 1, 2.3, 'a', {}; checkme = t[1:]" << "tuple of (float, str, dict)"; QTest::newRow("tuple_slice_defboth") << "t = 1, 2.3, 'a', {}; checkme = t[:]" << "tuple of (int, float, str, dict)"; QTest::newRow("tuple_slice_step") << "t = 1, 2.3, 'a', {}; checkme = t[0:3:2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_reverse") << "t = 1, 2.3, 'a', {}; checkme = t[3:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstart") << "t = 1, 2.3, 'a', {}; checkme = t[:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[2::-1]" << "tuple of (str, float, int)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[::-1]" << "tuple of (dict, str, float, int)"; QTest::newRow("tuple_slice_no_elems") << "t = 1, 2.3, 'a', {}; checkme = t[1:1]" << "tuple of ()"; // TODO unsure-tuples. QTest::newRow("tuple_slice_not_literal") << "n = 2; t = 1, 2.3, 'a', {}; checkme = t[0:n]" << "tuple of ()"; // These are allowed, for whatever reason. QTest::newRow("tuple_slice_past_range") << "t = 1, 2.3; checkme = t[-999999999:8888888888]" << "tuple of (int, float)"; QTest::newRow("tuple_slice_wrong_direction") << "t = 1, 2.3, 'a'; checkme = t[0:3:-1]" << "tuple of ()"; // This isn't. QTest::newRow("tuple_slice_zero_step") << "t = 1, 2.3; checkme = t[::0]" << "tuple of ()"; QTest::newRow("tuple_add") << "t, u = (3,), ('q', 4.5); checkme = t + u" << "tuple of (int, str, float)"; QTest::newRow("tuple_mul") << "t = 3, 4.5; checkme = t * 2" << "tuple of (int, float, int, float)"; QTest::newRow("dict_unsure") << "t = dict(); t = {3: str()}\ncheckme = t[1].capitalize()" << "str"; QTest::newRow("unsure_attr_access") << "d = str(); d = 3; checkme = d.capitalize()" << "str"; QTest::newRow("class_create_var") << "class c: pass\nd = c()\nd.foo = 3\ncheckme = d.foo" << "int"; QTest::newRow("tuple_loop") << "t = [(1, \"str\")]\nfor checkme, a in t: pass" << "int"; QTest::newRow("no_hints_type") << "def myfun(arg): arg = 3; return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("hints_type") << "def myfun(arg): return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("args_type") << "def myfun(*args): return args[0]\ncheckme = myfun(3)" << "int"; QTest::newRow("kwarg_type") << "def myfun(**kwargs): return kwargs['a']\ncheckme = myfun(a=3)" << "int"; QTest::newRow("dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\ncheckme = foo(**{'a': 12})" << "int"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("dict_norm_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2)" << "unsure (int, float)"; QTest::newRow("multi_dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2, **{'c': ''})" << "unsure (int, float, str)"; #endif QTest::newRow("named_arg_type") << "def myfun(arg): return arg\ncheckme = myfun(arg=3)" << "int"; QTest::newRow("arg_args_type") << "def myfun(arg, *args): return args[0]\n" "checkme = myfun(3, str())" << "str"; QTest::newRow("arg_kwargs_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(12, a=str())" << "str"; QTest::newRow("named_kwargs_type_1") << "def myfun(arg, **kwargs): return arg\n" "checkme = myfun(arg=12, a=str())" << "int"; QTest::newRow("named_kwargs_type_2") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(arg=12, a=str())" << "str"; QTest::newRow("kwargs_named_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(a=str(), arg=12)" << "str"; QTest::newRow("varied_args_type_1") << "def myfun(arg, *args, **kwargs): return arg\n" "checkme = myfun(1, 1.5, a=str())" << "int"; QTest::newRow("varied_args_type_2") << "def myfun(arg, *args, **kwargs): return args[0]\n" "checkme = myfun(1, 1.5, a=str())" << "float"; QTest::newRow("varied_args_type_3") << "def myfun(arg, *args, **kwargs): return kwargs['a']\n" "checkme = myfun(1, 1.5, a=str())" << "str"; QTest::newRow("nested_arg_name_type") << "def foo(xxx):\n" " def bar(xxx): return xxx\n" " return bar('test')\n" "checkme = foo(10)\n" << "str"; QTest::newRow("method_args_type_1") << "class MyClass:\n" " def method(self, arg): return self\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("method_args_type_2") << "class MyClass:\n" " def method(self, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("clsmethod_args_type_1") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return cls\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("clsmethod_args_type_2") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_args_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_vararg_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg, *args): return args[0]\n" "checkme = MyClass().method(12, 2.5)" << "float"; QTest::newRow("method_explicit_self") << "class MyClass:\n" " def method(self, arg): return arg\n" "instance = MyClass()\n" "checkme = MyClass.method(instance, 12)" << "int"; QTest::newRow("method_vararg_explicit_self") << "class MyClass:\n" " def foo(self, arg, *args): return args[0]\n" "mc = MyClass()\n" "checkme = MyClass.foo(mc, 'str', 3, 4.5)" << "int"; QTest::newRow("clsmethod_explicit_self") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg1, arg2): return arg2\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "int"; QTest::newRow("staticmethod_explicit_self") << "class MyClass:\n" " @staticmethod\n" " def method(arg1, arg2): return arg1\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "str"; QTest::newRow("parent_constructor_arg_type") << "class Base:\n" // https://bugs.kde.org/show_bug.cgi?id=369364 " def __init__(self, foo):\n" " self.foo = foo\n" "class Derived(Base):\n" " def __init__(self, foo):\n" " Base.__init__(self, foo)\n" "instance = Derived('string')\n" "checkme = instance.foo" << "str"; QTest::newRow("nested_class_self_inside") << "class Foo:\n" " def foo(self):\n" " class Bar:\n" " def bar(self): return self\n" " return Bar().bar()\n" "checkme = Foo().foo()\n" << "Foo::foo::Bar"; QTest::newRow("nested_class_self_after") << "class Foo:\n" " class Bar: pass\n" " def foo(self): return self\n" "checkme = Foo().foo()\n" << "Foo"; QTest::newRow("tuple_unsure") << "q = (3, str())\nq=(str(), 3)\ncheckme, _ = q" << "unsure (int, str)"; QTest::newRow("custom_iterable") << "class Gen2:\n" " def __iter__(self): return self\n" " def __next__(self): return 'blah'\n" "for checkme in Gen2(): pass" << "str"; QTest::newRow("separate_iterator") << "class Foo:\n" " def __iter__(self): return Bar()\n" " def __next__(self): return 'blah'\n" // Not used (or shouldn't be!) "class Bar:\n" " def __next__(self): return {1}\n" "checkme = [a for a in Foo()]" << "list of set of int"; QTest::newRow("return_builtin_iterator") << "class Gen2:\n" " contents = [1, 2, 3]\n" " def __iter__(self): return iter(Gen2.contents)\n" "for checkme in Gen2(): pass" << "int"; QTest::newRow("init_class") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "checkme = Foo()\n" << "Foo"; QTest::newRow("init_class_no_decl") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "a = [Foo]\n" "checkme = a[0]()\n" << "Foo"; QTest::newRow("call_class") << "class Foo:\n" " def __call__(self):\n" " return 0\n" "f = Foo()\n" "checkme = f()\n" << "int"; QTest::newRow("call_class_no_decl") << "class Foo:\n" " def __call__(self): return 1.5\n" "a = [Foo()]\n" "checkme = a[0]()" << "float"; QTest::newRow("classmethod") << "class Foo:\n" " @classmethod\n" " def foo(cls):\n" " k = cls()\n" " return k\n" "f = Foo.foo()\n" "checkme = f\n" << "Foo"; QTest::newRow("property_getter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" "checkme = Foo().bar" << "int"; QTest::newRow("property_wrong") << "class Foo:\n" " @property\n" " def bar(self): return True\n" "checkme = Foo().bar()" << "mixed"; QTest::newRow("property_setter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" " @bar.setter\n" " def bar(self, value): return 18.3\n" // Return should be ignored "checkme = Foo().bar" << "int"; QTest::newRow("tuple_listof") << "l = [(1, 2), (3, 4)]\ncheckme = l[1][0]" << "int"; QTest::newRow("getitem") << "class c:\n def __getitem__(self, slice): return 3.14\na = c()\ncheckme = a[2]" << "float"; QTest::newRow("constructor_type_deduction") << "class myclass:\n" "\tdef __init__(self, param): self.foo=param\n" "checkme = myclass(3).foo" << "int"; QTest::newRow("simpe_type_deduction") << "def myfunc(arg): return arg\n" "checkme = myfunc(3)" << "int"; QTest::newRow("functionCall_functionArg_part1") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "checkme = f1()" << "str"; QTest::newRow("functionCall_functionArg_part2") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = identity(getstr)\n" "checkme = f1()\n" << "str"; QTest::newRow("functionCall_functionArg_full") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "f2 = identity(getstr)\n" "a = getstr()\n" "b = f1()\n" "checkme = f2()\n" << "str"; QTest::newRow("vararg_before_other_args") << "def myfun(a, b, *z, x): return z[0]\n" "checkme = myfun(False, False, 1, x = False)" << "int"; QTest::newRow("vararg_before_other_args2") << "def myfun(a, b, *z, x): return z[3]\n" "checkme = myfun(False, False, 1, 2, 3, \"str\", x = False)" << "str"; QTest::newRow("vararg_constructor") << "class myclass():\n" " def __init__(self, *arg): self.prop = arg[0]\n" "obj = myclass(3, 5); checkme = obj.prop" << "int"; QTest::newRow("declaration_order_var") << "aaa = 2\n" "checkme = aaa" << "int"; QTest::newRow("declaration_order_var2") << "checkme = aaa\n" "aaa = 2" << "mixed"; QTest::newRow("declaration_order_func_defarg") << "aaa = 2\n" "def foo(x=aaa): return x\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_defarg2") << "def foo(x=aaa): return x\n" "aaa = 2\n" "checkme = foo()" << "mixed"; QTest::newRow("declaration_order_func_body") << "aaa = 2\n" "def foo(): return aaa\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_body2") << "def foo(): return aaa\n" "aaa = 2\n" "checkme = foo()" << "int"; QTest::newRow("global_variable") << "a = 3\n" "def f1():\n" " global a\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_variable2") << "a = 3\n" "def f1():\n" " global a\n" " a = \"str\"\n" " return a\n" "checkme = f1()\n" << "str"; QTest::newRow("global_scope_variable") << "a = 3\n" "def f1():\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_no_toplevel_dec") << "def f1():\n" " global a\n a = 3\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("top_level_vs_class_attr") << "var = 3\n" "class MyClass:\n" " var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("top_level_vs_instance_attr") << "var = 3\n" "class MyClass:\n" " def __init__(self): self.var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("intermediate_vs_class/instance_attrs") << "def func():\n" " aa, bb = 3, 4\n" " class Foo:\n" " aa = 'a'\n" " def __init__(self):\n" " self.bb = 'b'\n" " def foo(self):\n" " return aa, bb\n" " return Foo().foo()\n" "checkme = func()" << "tuple of (int, int)"; QTest::newRow("top_level_vs_nested_class_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " aaa = 1\n" " class Bar:\n" " bbb = 2\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // Bar.bbb is visible here, Foo.aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; QTest::newRow("top_level_vs_nested_instance_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " def __init__(self): self.aaa = 1\n" " class Bar:\n" " def __init__(self): self.bbb = 1\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // self.bbb is visible here, Foo().aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + QTest::newRow("assignment_expr_while") << + "file = open('foo.txt')\n" + "while q := file.readline():\n" + " checkme = q\n" << "str"; + QTest::newRow("assignment_expr_comprehension") << + "checkme = [z for q in (1, 2, 3) if (z := q % 2)]" << "list of int"; + QTest::newRow("assignment_expr_context") << + "a = [z for q in (1, 2, 3) if (z := q % 2)]\n" + "checkme = z" << "int"; + QTest::newRow("positional_params") << + "def foo(a, b, /, c, d):\n" + " return a, b, c, d\n" + "checkme = foo(10, 'x', 2.3, d='y')\n" << "tuple of (int, str, float, str)"; +#endif } typedef QPair pair; void PyDUChainTest::testImportDeclarations() { QFETCH(QString, code); QFETCH(QStringList, expectedDecls); QFETCH(bool, shouldBeAliased); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); foreach ( const QString& expected, expectedDecls ) { bool found = false; QString name = expected; const auto decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); qCDebug(KDEV_PYTHON_DUCHAIN) << "FOUND DECLARATIONS:"; foreach ( const pair& current, decls ) { qCDebug(KDEV_PYTHON_DUCHAIN) << current.first->toString() << current.first->identifier().identifier().byteArray() << name; } foreach ( const pair& current, decls ) { if ( ! ( current.first->identifier().identifier().byteArray() == name ) ) continue; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found: " << current.first->toString() << " for " << name; AliasDeclaration* isAliased = dynamic_cast(current.first); if ( isAliased && shouldBeAliased ) { found = true; // TODO fixme } else if ( ! isAliased && ! shouldBeAliased ) { found = true; } } QVERIFY(found); } } void PyDUChainTest::testProblemCount() { QFETCH(QString, code); QFETCH(int, problemsCount); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QEXPECT_FAIL("fstring_visit_inside", "Ranges are broken so we don't visit the expression", Continue); QCOMPARE(ctx->problems().size(), problemsCount); } void PyDUChainTest::testProblemCount_data() { QTest::addColumn("code"); QTest::addColumn("problemsCount"); QTest::newRow("list_comp") << "[foo for foo in range(3)]" << 0; QTest::newRow("list_comp_wrong") << "[bar for foo in range(3)]" << 1; QTest::newRow("list_comp_staticmethod") << "class A:\n @staticmethod\n def func(cls):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_decorator") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_wrong") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("list_comp_staticmethod_wrong") << "class A:\n @staticmethod\n def func(cls):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("misplaced_return_plain") << "return" << 1; QTest::newRow("misplaced_return_value") << "return 15" << 1; QTest::newRow("misplaced_return_class") << "class A:\n return 25" << 1; QTest::newRow("correct_return") << "def foo():\n return" << 0; QTest::newRow("lambda_argument_outside") << "def bar():\n lambda foo: 3\n foo" << 1; QTest::newRow("use_found_at_decl") << "foo = 3" << 0; QTest::newRow("fstring_visit_inside") << "checkme = f'{name}'" << 1; } void PyDUChainTest::testImportDeclarations_data() { QTest::addColumn("code"); QTest::addColumn("expectedDecls"); QTest::addColumn("shouldBeAliased"); QTest::newRow("from_import") << "from testImportDeclarations.i import checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("import") << "import testImportDeclarations.i" << ( QStringList() << "testImportDeclarations" ) << false; QTest::newRow("import_as") << "import testImportDeclarations.i as checkme" << ( QStringList() << "checkme" ) << false; QTest::newRow("from_import_as") << "from testImportDeclarations.i import checkme as checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("from_import_missing") << "from testImportDeclarations.i import missing as checkme" << ( QStringList() ) << true; } typedef QPair p; void PyDUChainTest::testAutocompletionFlickering() { TestFile f("foo = 3\nfoo2 = 2\nfo", "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx1 = f.topContext(); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx1); auto decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); QList declIds; foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); f.setFileContents("foo = 3\nfoo2 = 2\nfoo"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx2 = f.topContext(); QVERIFY(ctx2); lock.lock(); auto decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@1: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); qDebug() << "========================="; TestFile g("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfo", "py"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx1 = g.topContext(); lock.lock(); QVERIFY(ctx1); decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); declIds.clear(); foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); g.setFileContents("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfoo"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx2 = g.topContext(); QVERIFY(ctx2); lock.lock(); decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@2: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); } void PyDUChainTest::testFunctionHints() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testFunctionHints_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("func_return_type") << "def myfun(arg) -> int: pass\ncheckme = myfun(\"3\")" << "unsure (None, int)"; QTest::newRow("argument_type") << "def myfun(arg : int): return arg\ncheckme = myfun(foobar)" << "int"; QTest::newRow("argument_type_only_if_typeof") << "def myfun(arg : 3): return arg\ncheckme = myfun(foobar)" << "mixed"; } void PyDUChainTest::testHintedTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testHintedTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("simple_hint") << "def myfunc(x): return x\ncheckme = myfunc(3)" << "int"; QTest::newRow("hint_unsure") << "def myfunc(x): return x\nmyfunc(3.5)\ncheckme = myfunc(3)" << "unsure (float, int)"; QTest::newRow("unsure_attribute") << "def myfunc(x): return x.capitalize()\nmyfunc(3.5)\ncheckme = myfunc(str())" << "str"; } void PyDUChainTest::testOperators() { QFETCH(QString, code); QFETCH(QString, expectedType); code.prepend("from testOperators.example import *\n\n"); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QVERIFY(visitor->found); } void PyDUChainTest::testOperators_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("add") << "checkme = Example() + Example()" << "Add"; QTest::newRow("sub") << "checkme = Example() - Example()" << "Sub"; QTest::newRow("mul") << "checkme = Example() * Example()" << "Mul"; QTest::newRow("floordiv") << "checkme = Example() // Example()" << "Floordiv"; QTest::newRow("mod") << "checkme = Example() % Example()" << "Mod"; QTest::newRow("pow") << "checkme = Example() ** Example()" << "Pow"; QTest::newRow("lshift") << "checkme = Example() << Example()" << "Lshift"; QTest::newRow("rshift") << "checkme = Example() >> Example()" << "Rshift"; QTest::newRow("and") << "checkme = Example() & Example()" << "And"; QTest::newRow("xor") << "checkme = Example() ^ Example()" << "Xor"; QTest::newRow("or") << "checkme = Example() | Example()" << "Or"; } void PyDUChainTest::testFunctionArgs() { ReferencedTopDUContext ctx = parse("def ASDF(arg1, arg2):\n" " arg1 = arg2"); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVERIFY(m_ast); // dumpDUContext(ctx); QCOMPARE(ctx->childContexts().size(), 2); DUContext* funcArgCtx = ctx->childContexts().first(); QCOMPARE(funcArgCtx->type(), DUContext::Function); QCOMPARE(funcArgCtx->localDeclarations().size(), 2); QVERIFY(!funcArgCtx->owner()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); QCOMPARE(decl->type()->arguments().length(), 2); qDebug() << decl->type()->arguments().length() << 2; DUContext* funcBodyCtx = ctx->childContexts().last(); QCOMPARE(funcBodyCtx->type(), DUContext::Other); QVERIFY(funcBodyCtx->owner()); QVERIFY(funcBodyCtx->localDeclarations().isEmpty()); } void PyDUChainTest::testInheritance() { QFETCH(QString, code); QFETCH(int, expectedBaseClasses); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); auto decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); bool found = false; bool classDeclFound = false; foreach ( const p& item, decls ) { if ( item.first->identifier().toString() == "B" ) { auto klass = dynamic_cast(item.first); QVERIFY(klass); QCOMPARE(klass->baseClassesSize(), static_cast(expectedBaseClasses)); classDeclFound = true; } if ( item.first->identifier().toString() == "checkme" ) { QCOMPARE(item.first->abstractType()->toString(), QString("int")); found = true; } } QVERIFY(found); QVERIFY(classDeclFound); } void PyDUChainTest::testInheritance_data() { QTest::addColumn("code"); QTest::addColumn("expectedBaseClasses"); QTest::newRow("simple") << "class A():\n\tattr = 3\n\nclass B(A):\n\tpass\n\ninst=B()\ncheckme = inst.attr" << 1; QTest::newRow("context_import_prereq") << "import testInheritance.i\ninst=testInheritance.i.testclass()\n" "checkme = inst.attr\nclass B(): pass" << 1; // 1 because object QTest::newRow("context_import") << "import testInheritance.i\n\nclass B(testInheritance.i.testclass):\n" "\ti = 4\n\ninst=B()\ncheckme = inst.attr" << 1; } void PyDUChainTest::testClassContextRanges() { QString code = "class my_class():\n pass\n \n \n \n \n"; ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; DUContext* classContext = ctx->findContextAt(CursorInRevision(5, 0)); QVERIFY(classContext); QVERIFY(classContext->type() == DUContext::Class); } void PyDUChainTest::testContainerTypes() { QFETCH(QString, code); QFETCH(QString, contenttype); QFETCH(bool, use_type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList decls = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(decls.length() > 0); QVERIFY(decls.first()->abstractType()); if ( ! use_type ) { auto type = ListType::Ptr::dynamicCast(decls.first()->abstractType()); QVERIFY(type); QVERIFY(type->contentType()); QCOMPARE(type->contentType().abstractType()->toString(), contenttype); } else { QVERIFY(decls.first()->abstractType()); QEXPECT_FAIL("dict_of_int_call", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("dict_from_tuples", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("comprehension_shadowing_ms", "Nothing is foolproof to a sufficiently capable fool", Continue); QEXPECT_FAIL("comprehension_shadowing_nest2", "See above", Continue); QCOMPARE(decls.first()->abstractType()->toString(), contenttype); } } void PyDUChainTest::testContainerTypes_data() { QTest::addColumn("code"); QTest::addColumn("contenttype"); QTest::addColumn("use_type"); QTest::newRow("list_of_int") << "checkme = [1, 2, 3]" << "int" << false; QTest::newRow("list_from_unpacked") << "foo = [1.3]\ncheckme = [1, *foo, 3]" << "unsure (int, float)" << false; QTest::newRow("list_of_int_call") << "checkme = list([1, 2, 3])" << "int" << false; QTest::newRow("list_from_tuple") << "checkme = list((1, 2, 3))" << "int" << false; QTest::newRow("list_from_dict") << "checkme = list({'a':1, 'b':2})" << "str" << false; // Gets key type! QTest::newRow("list_from_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = list(MyClass())" << "float" << false; QTest::newRow("generator") << "checkme = [i for i in [1, 2, 3]]" << "int" << false; QTest::newRow("list_access") << "list = [1, 2, 3]\ncheckme = list[0]" << "int" << true; QTest::newRow("set_of_int") << "checkme = {1, 2, 3}" << "int" << false; QTest::newRow("set_of_int_call") << "checkme = set({1, 2, 3})" << "int" << false; QTest::newRow("set_from_tuple") << "checkme = set((1, 2, 3))" << "int" << false; QTest::newRow("set_generator") << "checkme = {i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_of_str_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; QTest::newRow("frozenset_of_int_call") << "checkme = frozenset({1, 2, 3})" << "int" << false; QTest::newRow("dict_of_int") << "checkme = {a:1, b:2, c:3}" << "int" << false; QTest::newRow("dict_of_int_call") << "checkme = dict({'a':1, 'b':2, 'c':3})" << "dict of str : int" << true; QTest::newRow("dict_from_tuples") << "checkme = dict([('a', 1), ('b', 2)])" << "dict of str : int" << true; QTest::newRow("dict_generator") << "checkme = {\"Foo\":i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_access") << "list = {'a':1, 'b':2, 'c':3}\ncheckme = list[0]" << "int" << true; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("set_from_unpacked") << "foo = [1.3]\ncheckme = {1, *foo, 3}" << "unsure (int, float)" << false; QTest::newRow("dict_from_unpacked") << "checkme = {**{'a': 1}}" << "dict of str : int" << true; QTest::newRow("dict_from_varied") << "checkme = {**{'a': 1}, 1: 1.5}" << "dict of unsure (str, int) : unsure (int, float)" << true; #endif QTest::newRow("generator_attribute") << "checkme = [item.capitalize() for item in ['foobar']]" << "str" << false; QTest::newRow("cannot_change_type") << "checkme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("cannot_change_type2") << "[1, 2, 3].append(5)\ncheckme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("list_append") << "d = []\nd.append(3)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend") << "d = []; q = [int()]\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_tuple") << "d = []; q = (1, 2)\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = []\ncheckme.extend(MyClass())" << "float" << false; QTest::newRow("for_loop") << "d = [3]\nfor item in d:\n checkme = item" << "int" << true; QTest::newRow("for_loop_unsure") << "d = [3, \"foo\"]\nfor item in d:\n checkme = item" << "unsure (int, str)" << true; QTest::newRow("for_loop_tuple_1") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = a" << "int" << true; QTest::newRow("for_loop_tuple_2") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = b" << "float" << true; QTest::newRow("for_loop_tuple_unsure") << "d = [(3, 3.5), (3.5, 3)]\nfor a, b in d:\n checkme = b" << "unsure (float, int)" << true; // Proposed by Nicolás Alvarez; why not? https://bugs.kde.org/show_bug.cgi?id=359915 QTest::newRow("comprehension_messy") << "users = {'a':19, 'b':42, 'c':35}\n" "sorted_list = sorted(users.items(), key=lambda kv: (-kv[1], kv[0]))\n" "checkme = [k for r,(k,v) in enumerate(sorted_list, 1)]" << "list of str" << true; QTest::newRow("comprehension_multiline") << "checkme = [a for\n a in \n (1, 2)]" << "list of int" << true; QTest::newRow("comprehension_multistage") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for bar in nested for foo in bar]" << "list of int" << true; QTest::newRow("comprehension_shadowing_ms") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in nested for foo in foo]" << "list of int" << true; QTest::newRow("comprehension_shadowing_nest1") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in [foo for foo in nested]]" << "list of tuple of (int, int)" << true; QTest::newRow("comprehension_shadowing_nest2") << "nested = (1, 2), (3, 4)\n" "checkme = [[foo for foo in foo] for foo in nested]" << "list of list of int" << true; // From https://bugs.kde.org/show_bug.cgi?id=359912 QTest::newRow("subscript_multi") << "class Middle:\n def __getitem__(self, key):\n return str()\n" "class Outer:\n def __getitem__(self, key):\n return Middle()\n" "aaa = Outer()\ncheckme = aaa[0][0]" << "str" << true; QTest::newRow("subscript_func_call") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "def bar():\n return Foo()\n" "checkme = bar()[0]" << "str" << true; QTest::newRow("subscript_unknown_index") << "a = 1,str()\ncheckme = a[5-4]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure") << "a = 1,2\na=[str()]\ncheckme = a[0]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure_getitem") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "class Bar:\n def __getitem__(self, key):\n return float()\n" "a = Foo()\na=Bar()\na=[1,2]\ncheckme = a[1]" << "unsure (str, float, int)" << true; } void PyDUChainTest::testVariableCreation() { QFETCH(QString, code); QFETCH(QStringList, expected_local_declarations); QFETCH(QStringList, expected_types); ReferencedTopDUContext top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto localDecls = top->localDeclarations(); QVector localDeclNames; for ( const Declaration* d: localDecls ) { localDeclNames.append(d->identifier().toString()); } Q_ASSERT(expected_types.size() == expected_local_declarations.size()); int offset = 0; for ( const QString& expected : expected_local_declarations ) { int index = localDeclNames.indexOf(expected); QVERIFY(index != -1); QVERIFY(localDecls[index]->abstractType()); QCOMPARE(localDecls[index]->abstractType()->toString(), expected_types[offset]); offset++; } } void PyDUChainTest::testVariableCreation_data() { QTest::addColumn("code"); QTest::addColumn("expected_local_declarations"); QTest::addColumn("expected_types"); QTest::newRow("simple") << "a = 3" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("tuple_wrong") << "a, b = 3" << QStringList{"a", "b"} << QStringList{"mixed", "mixed"}; QTest::newRow("tuple_unpack_inplace") << "a, b = 3, 5.5" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_indirect") << "c = 3, 3.5\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_stacked_inplace") << "a, (b, c) = 1, (2, 3.5)" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("tuple_unpack_stacked_indirect") << "d = 3.5, (3, 1)\na, (b, c) = d" << QStringList{"a", "b", "c"} << QStringList{"float", "int", "int"}; QTest::newRow("unpack_from_list_inplace") << "a, b = [1, 2, 3]" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_from_list_indirect") << "c = [1, 2, 3]\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_custom_iterable") << "class Foo:\n" " def __iter__(self): return self\n" " def __next__(self): return 1.5\n" "a, *b = Foo()" << QStringList{"a", "b"} << QStringList {"float", "list of float"}; QTest::newRow("for_loop_simple") << "for i in range(3): pass" << QStringList{"i"} << QStringList{"int"}; QTest::newRow("for_loop_unpack") << "for a, b in [(3, 5.1)]: pass" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("for_loop_stacked") << "for a, (b, c) in [(1, (2, 3.5))]: pass" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("for_loop_tuple") << "for a in 1, 2: pass" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("for_loop_dict") << "for a in {'foo': 1}: pass" << QStringList{"a"} << QStringList{"str"}; +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + QTest::newRow("assignment_expr") << "a = (b := 10)" << QStringList{"a", "b"} << QStringList{"int", "int"}; +#endif } void PyDUChainTest::testCleanupMultiplePasses() { for ( int j = 0; j < 20; j++ ) { ReferencedTopDUContext top = parse("from testCleanupMultiplePasses import foo\ndef fonc(): return 3+2j\nfoo.foo.func = fonc"); } } void PyDUChainTest::testManyDeclarations() { ReferencedTopDUContext top = parse("from testManyDeclarations import test\nk=test.Foo()"); } void PyDUChainTest::testComments() { QFETCH(QString, code); auto top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto decls = top->findDeclarations(QualifiedIdentifier("a")); QCOMPARE(decls.size(), 1); auto a = decls.first(); QCOMPARE(a->comment(), QByteArray("comment")); decls = top->findDeclarations(QualifiedIdentifier("b")); if ( decls.isEmpty() ) { decls = top->childContexts().last()->findDeclarations(QualifiedIdentifier("b")); } auto b = decls.first(); QCOMPARE(b->comment(), QByteArray()); } void PyDUChainTest::testComments_data() { QTest::addColumn("code"); QTest::newRow("variable") << "b=5\n\"\"\"comment\"\"\"\na=5\nb=5"; QTest::newRow("function") << "def a():\n \"\"\"comment\"\"\"\n b=5"; QTest::newRow("class") << "class a:\n \"\"\"comment\"\"\"\n b=5"; } diff --git a/parser/ast.cpp b/parser/ast.cpp index 9892b184..cd570d36 100644 --- a/parser/ast.cpp +++ b/parser/ast.cpp @@ -1,368 +1,373 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2012 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "ast.h" #include "astbuilder.h" #include namespace Python { // We never need actual constructors for AST nodes, but it seems to be required, at least for some platforms // so we provide pseudo implementations // there's nothing happening here, don't bother reading the code Ast::Ast( Ast* parent, Ast::AstType type ) : parent(parent), astType( type ), startCol(0), startLine(-99999), endCol(0), endLine(0), hasUsefulRangeInformation(false), context(nullptr) { } Ast::Ast() : parent(nullptr), startCol(0), startLine(-5), endCol(0), endLine(0), hasUsefulRangeInformation(false), context(nullptr) { } ArgumentsAst::ArgumentsAst(Ast* parent): Ast(parent, Ast::ArgumentsAstType) { } ArgAst::ArgAst(Ast* parent): Ast(parent, Ast::ArgAstType), argumentName(nullptr), annotation(nullptr) { } AssertionAst::AssertionAst(Ast* parent): StatementAst(parent, Ast::AssertionAstType) { } AssignmentAst::AssignmentAst(Ast* parent): StatementAst(parent, Ast::AssignmentAstType), value(nullptr) { } AttributeAst::AttributeAst(Ast* parent): ExpressionAst(parent, Ast::AttributeAstType), value(nullptr), depth(0) { } AugmentedAssignmentAst::AugmentedAssignmentAst(Ast* parent): StatementAst(parent, Ast::AugmentedAssignmentAstType), value(nullptr) { } AnnotationAssignmentAst::AnnotationAssignmentAst(Ast* parent): StatementAst(parent, Ast::AnnotationAssignmentAstType), target(nullptr), value(nullptr), annotation(nullptr) { } BinaryOperationAst::BinaryOperationAst(Ast* parent): ExpressionAst(parent, Ast::BinaryOperationAstType), lhs(nullptr), rhs(nullptr) { } BooleanOperationAst::BooleanOperationAst(Ast* parent): ExpressionAst(parent, Ast::BooleanOperationAstType) { } BreakAst::BreakAst(Ast* parent): StatementAst(parent, Ast::BreakAstType) { } CallAst::CallAst(Ast* parent): ExpressionAst(parent, Ast::CallAstType), function(nullptr) { } ClassDefinitionAst::ClassDefinitionAst(Ast* parent): StatementAst(parent, Ast::ClassDefinitionAstType), name(nullptr) { } CodeAst::CodeAst() : Ast(nullptr, Ast::CodeAstType), name(nullptr) { } CodeAst::~CodeAst() { free_ast_recursive(this); } CompareAst::CompareAst(Ast* parent): ExpressionAst(parent, Ast::CompareAstType), leftmostElement(nullptr) { } ComprehensionAst::ComprehensionAst(Ast* parent): Ast(parent, Ast::ComprehensionAstType), target(nullptr), iterator(nullptr) { } ContinueAst::ContinueAst(Ast* parent): StatementAst(parent, Ast::ContinueAstType) { } DeleteAst::DeleteAst(Ast* parent): StatementAst(parent, Ast::DeleteAstType) { } DictAst::DictAst(Ast* parent): ExpressionAst(parent, Ast::DictAstType) { } IndexAst::IndexAst(Ast* parent): SliceAstBase(parent, Ast::IndexAstType), value(nullptr) { } SliceAst::SliceAst(Ast* parent): SliceAstBase(parent, Ast::SliceAstType), lower(nullptr), upper(nullptr), step(nullptr) { } DictionaryComprehensionAst::DictionaryComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::DictionaryComprehensionAstType), key(nullptr), value(nullptr) { } EllipsisAst::EllipsisAst(Ast* parent): SliceAstBase(parent, Ast::EllipsisAstType) { } ExceptionHandlerAst::ExceptionHandlerAst(Ast* parent): Ast(parent, Ast::ExceptionHandlerAstType), type(nullptr), name(nullptr) { } ListComprehensionAst::ListComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::ListComprehensionAstType), element(nullptr) { } ExpressionAst::ExpressionAst(Ast* parent, AstType type): Ast(parent, type), value(nullptr) { } +AssignmentExpressionAst::AssignmentExpressionAst(Ast* parent): ExpressionAst(parent, Ast::AssignmentExpressionAstType), value(nullptr) +{ + +} + YieldFromAst::YieldFromAst(Ast* parent) : ExpressionAst(parent, Ast::YieldFromAstType) { } ExtendedSliceAst::ExtendedSliceAst(Ast* parent): SliceAstBase(parent, Ast::ExtendedSliceAstType) { } ForAst::ForAst(Ast* parent): StatementAst(parent, Ast::ForAstType), target(nullptr), iterator(nullptr) { } FunctionDefinitionAst::FunctionDefinitionAst(Ast* parent): StatementAst(parent, Ast::FunctionDefinitionAstType), name(nullptr), arguments(nullptr), async(false) { } GeneratorExpressionAst::GeneratorExpressionAst(Ast* parent): ExpressionAst(parent, Ast::GeneratorExpressionAstType), element(nullptr) { } GlobalAst::GlobalAst(Ast* parent): StatementAst(parent, Ast::GlobalAstType) { } Identifier::Identifier(QString value) : Ast(nullptr, Ast::IdentifierAstType), value(value) { } IfAst::IfAst(Ast* parent): StatementAst(parent, Ast::IfAstType), condition(nullptr) { } IfExpressionAst::IfExpressionAst(Ast* parent): ExpressionAst(parent, Ast::IfExpressionAstType), condition(nullptr) { } ImportAst::ImportAst(Ast* parent): StatementAst(parent, Ast::ImportAstType) { } ImportFromAst::ImportFromAst(Ast* parent): StatementAst(parent, Ast::ImportFromAstType), module(nullptr), level(0) { } KeywordAst::KeywordAst(Ast* parent): Ast(parent, Ast::KeywordAstType), argumentName(nullptr), value(nullptr) { } LambdaAst::LambdaAst(Ast* parent): ExpressionAst(parent, Ast::LambdaAstType), arguments(nullptr) { } ListAst::ListAst(Ast* parent): ExpressionAst(parent, Ast::ListAstType) { } NameAst::NameAst(Ast* parent): ExpressionAst(parent, Ast::NameAstType), identifier(nullptr) { } AwaitAst::AwaitAst(Ast* parent): ExpressionAst(parent, Ast::AwaitAstType), value(nullptr) { } NameConstantAst::NameConstantAst(Ast* parent): ExpressionAst(parent, Ast::NameConstantAstType), value(Invalid) { } -NumberAst::NumberAst(Ast* parent): ExpressionAst(parent, Ast::NumberAstType), value(0) +NumberAst::NumberAst(Ast* parent): ExpressionAst(parent, Ast::NumberAstType), value(0), isInt(false) { } PassAst::PassAst(Ast* parent): StatementAst(parent, Ast::PassAstType) { } NonlocalAst::NonlocalAst(Ast* parent): StatementAst(parent, Ast::NonlocalAstType) { } RaiseAst::RaiseAst(Ast* parent): StatementAst(parent, Ast::RaiseAstType), type(nullptr) { } ReturnAst::ReturnAst(Ast* parent): StatementAst(parent, Ast::ReturnAstType), value(nullptr) { } SetAst::SetAst(Ast* parent): ExpressionAst(parent, Ast::SetAstType) { } SetComprehensionAst::SetComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::SetComprehensionAstType), element(nullptr) { } SliceAstBase::SliceAstBase(Ast* parent, AstType type): Ast(parent, type) { } StatementAst::StatementAst(Ast* parent, AstType type): Ast(parent, type) { } StringAst::StringAst(Ast* parent): ExpressionAst(parent, Ast::StringAstType), value(""), usedAsComment(false) { } JoinedStringAst::JoinedStringAst(Ast* parent): ExpressionAst(parent, Ast::JoinedStringAstType), values() { } FormattedValueAst::FormattedValueAst(Ast* parent): ExpressionAst(parent, Ast::FormattedValueAstType), value(nullptr), conversion(0), formatSpec(nullptr) { } BytesAst::BytesAst(Ast* parent): ExpressionAst(parent, Ast::BytesAstType), value("") { } SubscriptAst::SubscriptAst(Ast* parent): ExpressionAst(parent, Ast::SubscriptAstType), value(nullptr), slice(nullptr) { } StarredAst::StarredAst(Ast* parent): ExpressionAst(parent, Ast::StarredAstType) { } TupleAst::TupleAst(Ast* parent): ExpressionAst(parent, Ast::TupleAstType) { } UnaryOperationAst::UnaryOperationAst(Ast* parent): ExpressionAst(parent, Ast::UnaryOperationAstType), operand(nullptr) { } TryAst::TryAst(Ast* parent): StatementAst(parent, Ast::TryAstType) { } WhileAst::WhileAst(Ast* parent): StatementAst(parent, Ast::WhileAstType), condition(nullptr) { } WithAst::WithAst(Ast* parent): StatementAst(parent, Ast::WithAstType) { } WithItemAst::WithItemAst(Ast* parent): Ast(parent, Ast::WithItemAstType) { } YieldAst::YieldAst(Ast* parent): ExpressionAst(parent, Ast::YieldAstType), value(nullptr) { } AliasAst::AliasAst(Ast* parent): Ast(parent, Ast::AliasAstType), name(nullptr), asName(nullptr) { } } diff --git a/parser/ast.h b/parser/ast.h index 378af0bd..65fef1cd 100644 --- a/parser/ast.h +++ b/parser/ast.h @@ -1,783 +1,792 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ // The Python 3.4 Language Reference was used as basis for this AST #ifndef PYTHON_AST_H #define PYTHON_AST_H #include #include #include #include #include "parserexport.h" #include "kdevpythonversion.h" namespace KDevelop { class DUContext; } namespace Python { class StatementAst; class FunctionDefinitionAst; class AssignmentAst; class PassAst; class NonlocalAst; class ExpressionAst; class NameAst; class CallAst; class AttributeAst; class ArgumentsAst; class KeywordAst; class ExpressionAst; class StatementAst; class Ast; class ExceptionHandlerAst; class AliasAst; class ComprehensionAst; class SliceAstBase; class SliceAst; } namespace Python { // Base class for all other Abstract Syntax Tree classes class KDEVPYTHONPARSER_EXPORT Ast { public: enum AstType { StatementAstType, FunctionDefinitionAstType, AssignmentAstType, PassAstType, NonlocalAstType, ArgumentsAstType, ArgAstType, KeywordAstType, ClassDefinitionAstType, ReturnAstType, DeleteAstType, ForAstType, WhileAstType, IfAstType, WithAstType, WithItemAstType, RaiseAstType, TryAstType, ImportAstType, ImportFromAstType, GlobalAstType, BreakAstType, ContinueAstType, AssertionAstType, AugmentedAssignmentAstType, AnnotationAssignmentAstType, LastStatementType, ExpressionAstType, // everything below is an expression AwaitAstType, NameAstType, NameConstantAstType, CallAstType, AttributeAstType, ExtendedSliceAstType, DictionaryComprehensionAstType, BooleanOperationAstType, BinaryOperationAstType, UnaryOperationAstType, LambdaAstType, IfExpressionAstType, // the short one, if a then b else c DictAstType, SetAstType, ListComprehensionAstType, SetComprehensionAstType, GeneratorExpressionAstType, YieldAstType, CompareAstType, NumberAstType, StringAstType, JoinedStringAstType, FormattedValueAstType, BytesAstType, SubscriptAstType, StarredAstType, ListAstType, TupleAstType, YieldFromAstType, ComprehensionAstType, SliceAstType, EllipsisAstType, IndexAstType, + AssignmentExpressionAstType, LastExpressionType, // keep this at the end of the expr ast list CodeAstType, ExceptionHandlerAstType, AliasAstType, // for imports IdentifierAstType, LastAstType // the largest one, not valid! }; enum BooleanOperationTypes { BooleanAnd = 1, BooleanOr, BooleanInvalidOperation }; enum OperatorTypes { OperatorAdd = 1, OperatorSub, OperatorMult, #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) OperatorMatMult, #endif OperatorDiv, OperatorMod, OperatorPow, OperatorLeftShift, OperatorRightShift, OperatorBitwiseOr, OperatorBitwiseXor, OperatorBitwiseAnd, OperatorFloorDivision, OperatorInvalid }; enum UnaryOperatorTypes { UnaryOperatorInvert = 1, UnaryOperatorNot, UnaryOperatorAdd, UnaryOperatorSub, UnaryOperatorInvalid }; enum ComparisonOperatorTypes { ComparisonOperatorEquals = 1, ComparisonOperatorNotEquals, ComparisonOperatorLessThan, ComparisonOperatorLessThanEqual, ComparisonOperatorGreaterThan, ComparisonOperatorGreaterThanEqual, ComparisonOperatorIs, ComparisonOperatorIsNot, ComparisonOperatorIn, ComparisonOperatorNotIn, ComparisonOperatorInvalid }; Ast(Ast* parent, AstType type); Ast(); Ast* parent = nullptr; AstType astType; bool isExpression() const { return astType >= ExpressionAstType && astType <= LastExpressionType; } void copyRange(const Ast* other) { startCol = other->startCol; endCol = other->endCol; startLine = other->startLine; endLine = other->endLine; } bool appearsBefore(const Ast* other) { return startLine < other->startLine || ( startLine == other->startLine && startCol < other->startCol ); } const KTextEditor::Range range() const { return KTextEditor::Range(startLine, startCol, endLine, endCol); }; const KTextEditor::Cursor start() const { return {startLine, startCol}; } const KTextEditor::Cursor end() const { return {endLine, endCol}; } bool isChildOf(Ast* other) const { const Ast* parent = this; while ( parent ) { if ( parent == other ) { return true; } parent = parent->parent; } return false; } int startCol; int startLine; int endCol; int endLine; bool hasUsefulRangeInformation; KDevelop::DUContext* context; }; class KDEVPYTHONPARSER_EXPORT Identifier : public Ast { public: Identifier(QString value); Identifier operator=(const Identifier& other) { value = other.value; startCol = other.startCol; endCol = other.endCol; startLine = other.startLine; endLine = other.endLine; parent = other.parent; hasUsefulRangeInformation = other.hasUsefulRangeInformation; return *this; }; bool operator==(const Identifier& rhs) const { return value == rhs.value; }; bool operator==(const QString& rhs) const { return value == rhs; }; void setEndColumn() { endCol = startCol + value.length() - 1; } operator QString() const { return value; }; QString value; }; // this replaces ModuleAst class KDEVPYTHONPARSER_EXPORT CodeAst : public Ast { public: CodeAst(); ~CodeAst(); typedef QSharedPointer Ptr; QList body; Identifier* name; // module name }; /** Statement classes **/ class KDEVPYTHONPARSER_EXPORT StatementAst : public Ast { public: StatementAst(Ast* parent, AstType type); }; class KDEVPYTHONPARSER_EXPORT FunctionDefinitionAst : public StatementAst { public: FunctionDefinitionAst(Ast* parent); Identifier* name; ArgumentsAst* arguments; QList decorators; QList body; ExpressionAst* returns; bool async; }; class KDEVPYTHONPARSER_EXPORT ClassDefinitionAst : public StatementAst { public: ClassDefinitionAst(Ast* parent); Identifier* name; QList baseClasses; QList body; QList decorators; }; class KDEVPYTHONPARSER_EXPORT ReturnAst : public StatementAst { public: ReturnAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT DeleteAst : public StatementAst { public: DeleteAst(Ast* parent); QList targets; }; class KDEVPYTHONPARSER_EXPORT AssignmentAst : public StatementAst { public: AssignmentAst(Ast* parent); QList targets; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT AugmentedAssignmentAst : public StatementAst { public: AugmentedAssignmentAst(Ast* parent); ExpressionAst* target; Ast::OperatorTypes op; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT AnnotationAssignmentAst : public StatementAst { public: AnnotationAssignmentAst(Ast* parent); ExpressionAst* target; ExpressionAst* value; ExpressionAst* annotation; }; class KDEVPYTHONPARSER_EXPORT ForAst : public StatementAst { public: ForAst(Ast* parent); ExpressionAst* target; ExpressionAst* iterator; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT WhileAst : public StatementAst { public: WhileAst(Ast* parent); ExpressionAst* condition; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT IfAst : public StatementAst { public: IfAst(Ast* parent); ExpressionAst* condition; QList body; QList orelse; }; class KDEVPYTHONPARSER_EXPORT WithItemAst : public Ast { public: WithItemAst(Ast* parent); ExpressionAst* contextExpression; ExpressionAst* optionalVars; }; class KDEVPYTHONPARSER_EXPORT WithAst : public StatementAst { public: WithAst(Ast* parent); QList body; QList items; }; class KDEVPYTHONPARSER_EXPORT RaiseAst : public StatementAst { public: RaiseAst(Ast* parent); ExpressionAst* type; // TODO check what the other things in the grammar actually are and add them }; class KDEVPYTHONPARSER_EXPORT TryAst : public StatementAst { public: TryAst(Ast* parent); QList body; QList handlers; QList orelse; QList finally; }; class KDEVPYTHONPARSER_EXPORT AssertionAst : public StatementAst { public: AssertionAst(Ast* parent); ExpressionAst* condition; ExpressionAst* message; }; class KDEVPYTHONPARSER_EXPORT ImportAst : public StatementAst { public: ImportAst(Ast* parent); QList names; }; class KDEVPYTHONPARSER_EXPORT ImportFromAst : public StatementAst { public: ImportFromAst(Ast* parent); Identifier* module; QList names; int level; }; class KDEVPYTHONPARSER_EXPORT GlobalAst : public StatementAst { public: GlobalAst(Ast* parent); QList names; }; // TODO what's stmt::Expr(expr value) in the grammar and what do we need it for? class KDEVPYTHONPARSER_EXPORT BreakAst : public StatementAst { public: BreakAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT ContinueAst : public StatementAst { public: ContinueAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT PassAst : public StatementAst { public: PassAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT NonlocalAst : public StatementAst { public: NonlocalAst(Ast* parent); }; /** Expression classes **/ class KDEVPYTHONPARSER_EXPORT ExpressionAst : public Ast { public: ExpressionAst(Ast* parent, AstType type = Ast::ExpressionAstType); enum Context { Load = 1, // the object is read Store = 2, // the object is written Delete = 3, // the object is deleted Invalid = -1 }; ExpressionAst* value; // WARNING this is not set in most cases! }; +class KDEVPYTHONPARSER_EXPORT AssignmentExpressionAst : public ExpressionAst { +public: + AssignmentExpressionAst(Ast* parent); + ExpressionAst* target; + ExpressionAst* value; +}; + class KDEVPYTHONPARSER_EXPORT AwaitAst : public ExpressionAst { public: AwaitAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT YieldFromAst : public ExpressionAst { public: YieldFromAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT BooleanOperationAst : public ExpressionAst { public: BooleanOperationAst(Ast* parent); Ast::BooleanOperationTypes type; QList values; }; class KDEVPYTHONPARSER_EXPORT BinaryOperationAst : public ExpressionAst { public: BinaryOperationAst(Ast* parent); Ast::OperatorTypes type; ExpressionAst* lhs; ExpressionAst* rhs; inline QString methodName() const { switch ( type ) { case Python::Ast::OperatorAdd: return QLatin1String("__add__"); case Python::Ast::OperatorBitwiseAnd: return QLatin1String("__and__"); case Python::Ast::OperatorBitwiseOr: return QLatin1String("__or__"); case Python::Ast::OperatorBitwiseXor: return QLatin1String("__xor__"); case Python::Ast::OperatorDiv: return QLatin1String("__div__"); case Python::Ast::OperatorFloorDivision: return QLatin1String("__floordiv__"); case Python::Ast::OperatorLeftShift: return QLatin1String("__lshift__"); case Python::Ast::OperatorMod: return QLatin1String("__mod__"); case Python::Ast::OperatorMult: return QLatin1String("__mul__"); #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Python::Ast::OperatorMatMult: return QLatin1String("__matmul__"); #endif case Python::Ast::OperatorPow: return QLatin1String("__pow__"); case Python::Ast::OperatorRightShift: return QLatin1String("__rshift__"); case Python::Ast::OperatorSub: return QLatin1String("__sub__"); case Python::Ast::OperatorInvalid: // fallthrough default: return QString(); } }; // incremental methods, for e.g. a += 3 inline QString incMethodName() const { QString name = methodName(); if ( name.size() < 3 ) { return name; } name.insert(2, QLatin1Char('i')); return name; } }; class KDEVPYTHONPARSER_EXPORT UnaryOperationAst : public ExpressionAst { public: UnaryOperationAst(Ast* parent); Ast::UnaryOperatorTypes type; ExpressionAst* operand; }; class KDEVPYTHONPARSER_EXPORT LambdaAst : public ExpressionAst { public: LambdaAst(Ast* parent); ArgumentsAst* arguments; ExpressionAst* body; }; class KDEVPYTHONPARSER_EXPORT IfExpressionAst : public ExpressionAst { public: IfExpressionAst(Ast* parent); ExpressionAst* condition; ExpressionAst* body; ExpressionAst* orelse; }; class KDEVPYTHONPARSER_EXPORT DictAst : public ExpressionAst { public: DictAst(Ast* parent); QList keys; // WARNING: Can contain null elements: `{**other}` QList values; }; class KDEVPYTHONPARSER_EXPORT SetAst : public ExpressionAst { public: SetAst(Ast* parent); QList elements; }; class KDEVPYTHONPARSER_EXPORT ListComprehensionAst : public ExpressionAst { public: ListComprehensionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT SetComprehensionAst : public ExpressionAst { public: SetComprehensionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT DictionaryComprehensionAst : public ExpressionAst { public: DictionaryComprehensionAst(Ast* parent); ExpressionAst* key; ExpressionAst* value; QList generators; }; class KDEVPYTHONPARSER_EXPORT GeneratorExpressionAst : public ExpressionAst { public: GeneratorExpressionAst(Ast* parent); ExpressionAst* element; QList generators; }; class KDEVPYTHONPARSER_EXPORT CompareAst : public ExpressionAst { public: CompareAst(Ast* parent); ExpressionAst* leftmostElement; QList operators; QList comparands; }; // TODO whats this exactly? class KDEVPYTHONPARSER_EXPORT ReprAst : public ExpressionAst { public: ReprAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT NumberAst : public ExpressionAst { public: NumberAst(Ast* parent); long value; // only used for ints bool isInt; // otherwise it's a float }; class KDEVPYTHONPARSER_EXPORT StringAst : public ExpressionAst { public: StringAst(Ast* parent); QString value; bool usedAsComment; }; class KDEVPYTHONPARSER_EXPORT JoinedStringAst : public ExpressionAst { public: JoinedStringAst(Ast* parent); QList values; }; class KDEVPYTHONPARSER_EXPORT FormattedValueAst : public ExpressionAst { public: FormattedValueAst(Ast* parent); ExpressionAst* value; int conversion; ExpressionAst* formatSpec; }; class KDEVPYTHONPARSER_EXPORT BytesAst : public ExpressionAst { public: BytesAst(Ast* parent); QString value; }; class KDEVPYTHONPARSER_EXPORT YieldAst : public ExpressionAst { public: YieldAst(Ast* parent); ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT NameAst : public ExpressionAst { public: NameAst(Ast* parent); Identifier* identifier; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT NameConstantAst : public ExpressionAst { public: NameConstantAst(Ast* parent); enum NameConstantTypes { False, True, None, Invalid // should not happen }; NameConstantTypes value; }; class KDEVPYTHONPARSER_EXPORT CallAst : public ExpressionAst { public: CallAst(Ast* parent); ExpressionAst* function; QList arguments; QList keywords; }; class KDEVPYTHONPARSER_EXPORT AttributeAst : public ExpressionAst { public: AttributeAst(Ast* parent); ExpressionAst* value; Identifier* attribute; ExpressionAst::Context context; int depth; }; class KDEVPYTHONPARSER_EXPORT SubscriptAst : public ExpressionAst { public: SubscriptAst(Ast* parent); ExpressionAst* value; SliceAstBase* slice; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT StarredAst : public ExpressionAst { public: StarredAst(Ast* parent); ExpressionAst* value; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT ListAst : public ExpressionAst { public: ListAst(Ast* parent); QList elements; ExpressionAst::Context context; }; class KDEVPYTHONPARSER_EXPORT TupleAst : public ExpressionAst { public: TupleAst(Ast* parent); QList elements; ExpressionAst::Context context; }; /** Slice classes **/ class KDEVPYTHONPARSER_EXPORT SliceAstBase : public Ast { public: SliceAstBase(Ast* parent, AstType type); }; class KDEVPYTHONPARSER_EXPORT EllipsisAst : public SliceAstBase { public: EllipsisAst(Ast* parent); }; class KDEVPYTHONPARSER_EXPORT SliceAst : public SliceAstBase { public: SliceAst(Ast* parent); ExpressionAst* lower; ExpressionAst* upper; ExpressionAst* step; }; class KDEVPYTHONPARSER_EXPORT ExtendedSliceAst : public SliceAstBase { public: ExtendedSliceAst(Ast* parent); QList dims; }; class KDEVPYTHONPARSER_EXPORT IndexAst : public SliceAstBase { public: IndexAst(Ast* parent); ExpressionAst* value; }; /** Independent classes **/ class KDEVPYTHONPARSER_EXPORT ArgAst : public Ast { public: ArgAst(Ast* parent); Identifier* argumentName; ExpressionAst* annotation; }; class KDEVPYTHONPARSER_EXPORT ArgumentsAst : public Ast { public: ArgumentsAst(Ast* parent); QList arguments; QList kwonlyargs; + QList posonlyargs; QList defaultValues; ArgAst* vararg; ArgAst* kwarg; }; class KDEVPYTHONPARSER_EXPORT KeywordAst : public Ast { public: KeywordAst(Ast* parent); Identifier* argumentName; ExpressionAst* value; }; class KDEVPYTHONPARSER_EXPORT ComprehensionAst : public Ast { public: ComprehensionAst(Ast* parent); ExpressionAst* target; ExpressionAst* iterator; QList conditions; }; class KDEVPYTHONPARSER_EXPORT ExceptionHandlerAst : public Ast { public: ExceptionHandlerAst(Ast* parent); ExpressionAst* type; Identifier* name; QList body; }; class KDEVPYTHONPARSER_EXPORT AliasAst : public Ast { public: AliasAst(Ast* parent); Identifier* name; Identifier* asName; }; } #endif diff --git a/parser/astbuilder.cpp b/parser/astbuilder.cpp index a1ccd68d..ad2e0dce 100644 --- a/parser/astbuilder.cpp +++ b/parser/astbuilder.cpp @@ -1,282 +1,288 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astbuilder.h" #include "ast.h" #include #include #include #include #include "python_header.h" #include "astdefaultvisitor.h" #include "cythonsyntaxremover.h" #include "rangefixvisitor.h" #include #include "parserdebug.h" using namespace KDevelop; extern grammar _PyParser_Grammar; namespace Python { #include "generated.h" QMutex AstBuilder::pyInitLock; QString PyUnicodeObjectToQString(PyObject* obj) { auto pyObjectCleanup = [](PyObject* o) { if (o) Py_DECREF(o); }; const auto strOwner = std::unique_ptr(PyObject_Str(obj), pyObjectCleanup); const auto str = strOwner.get(); if (PyUnicode_READY(str) < 0) { qWarning("PyUnicode_READY(%p) returned false!", (void*)str); return QString(); } const auto length = PyUnicode_GET_LENGTH(str); switch(PyUnicode_KIND(str)) { case PyUnicode_1BYTE_KIND: return QString::fromLatin1((const char*)PyUnicode_1BYTE_DATA(str), length); case PyUnicode_2BYTE_KIND: return QString::fromUtf16(PyUnicode_2BYTE_DATA(str), length); case PyUnicode_4BYTE_KIND: return QString::fromUcs4(PyUnicode_4BYTE_DATA(str), length); case PyUnicode_WCHAR_KIND: qWarning("PyUnicode_KIND(%p) returned PyUnicode_WCHAR_KIND, this should not happen!", (void*)str); return QString::fromWCharArray(PyUnicode_AS_UNICODE(str), length); } Q_UNREACHABLE(); } namespace { struct PythonInitializer : private QMutexLocker { PythonInitializer(QMutex& pyInitLock): QMutexLocker(&pyInitLock), arena(nullptr) { Py_InitializeEx(0); Q_ASSERT(Py_IsInitialized()); arena = PyArena_New(); Q_ASSERT(arena); // out of memory } ~PythonInitializer() { if (arena) PyArena_Free(arena); if (Py_IsInitialized()) Py_Finalize(); } PyArena* arena; }; } CodeAst::Ptr AstBuilder::parse(const QUrl& filename, QString &contents) { qCDebug(KDEV_PYTHON_PARSER) << " ====> AST ====> building abstract syntax tree for " << filename.path(); Py_NoSiteFlag = 1; contents.append('\n'); PythonInitializer pyIniter(pyInitLock); PyArena* arena = pyIniter.arena; +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + PyCompilerFlags flags; + flags.cf_flags = PyCF_SOURCE_IS_UTF8 | PyCF_IGNORE_COOKIE | PyCF_ONLY_AST; + flags.cf_feature_version = 7; +#else PyCompilerFlags flags = {PyCF_SOURCE_IS_UTF8 | PyCF_IGNORE_COOKIE}; +#endif CythonSyntaxRemover cythonSyntaxRemover; if (filename.fileName().endsWith(".pyx", Qt::CaseInsensitive)) { qCDebug(KDEV_PYTHON_PARSER) << filename.fileName() << "is probably Cython file."; contents = cythonSyntaxRemover.stripCythonSyntax(contents); } mod_ty syntaxtree = PyParser_ASTFromString(contents.toUtf8().data(), "", file_input, &flags, arena); if ( ! syntaxtree ) { qCDebug(KDEV_PYTHON_PARSER) << " ====< parse error, trying to fix"; PyObject *exception, *value, *backtrace; PyErr_Fetch(&exception, &value, &backtrace); qCDebug(KDEV_PYTHON_PARSER) << "Error objects: " << exception << value << backtrace; if ( ! value ) { qCWarning(KDEV_PYTHON_PARSER) << "Internal parser error: exception value is null, aborting"; return CodeAst::Ptr(); } PyErr_NormalizeException(&exception, &value, &backtrace); if ( ! PyObject_IsInstance(value, PyExc_SyntaxError) ) { qCWarning(KDEV_PYTHON_PARSER) << "Exception was not a SyntaxError, aborting"; return CodeAst::Ptr(); } PyObject* errorMessage_str = PyObject_GetAttrString(value, "msg"); PyObject* linenoobj = PyObject_GetAttrString(value, "lineno"); PyObject* colnoobj = PyObject_GetAttrString(value, "offset"); int lineno = PyLong_AsLong(linenoobj) - 1; int colno = PyLong_AsLong(colnoobj); ProblemPointer p(new Problem()); KTextEditor::Cursor start(lineno, (colno-4 > 0 ? colno-4 : 0)); KTextEditor::Cursor end(lineno, (colno+4 > 4 ? colno+4 : 4)); KTextEditor::Range range(start, end); qCDebug(KDEV_PYTHON_PARSER) << "Problem range: " << range; DocumentRange location(IndexedString(filename.path()), range); p->setFinalLocation(location); p->setDescription(PyUnicodeObjectToQString(errorMessage_str)); p->setSource(IProblem::Parser); m_problems.append(p); // try to recover. // Currently the following is tired: // * If the last non-space char before the error reported was ":", it's most likely an indent error. // The common easy-to-fix and annoying indent error is "for item in foo: ". In that case, just add "pass" after the ":" token. // * If it's not, we will just comment the line with the error, fixing problems like "foo = ". // * If both fails, everything including the first non-empty line before the one with the error will be deleted. int len = contents.length(); int currentLine = 0; QString currentLineContents; QChar c; QChar newline('\n'); int emptySince = 0; int emptySinceLine = 0; int emptyLinesSince = 0; int emptyLinesSinceLine = 0; unsigned short currentLineIndent = 0; bool atLineBeginning = true; QList indents; int errline = qMax(0, lineno); int currentLineBeginning = 0; for ( int i = 0; i < len; i++ ) { c = contents.at(i); if ( ! c.isSpace() ) { emptySince = i; emptySinceLine = currentLine; atLineBeginning = false; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); } else if ( c == newline ) { if ( currentLine == errline ) { atLineBeginning = false; } else { currentLine += 1; currentLineBeginning = i+1; // this line has had content, so reset the "empty lines since" counter if ( ! atLineBeginning ) { // lastNonemptyLineBeginning = emptyLinesSince; emptyLinesSince = i; emptyLinesSinceLine = currentLine; } atLineBeginning = true; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); currentLineIndent = 0; } } else if ( atLineBeginning ) { currentLineIndent += 1; } if ( currentLine == errline && ! atLineBeginning ) { // if the last non-empty char before the error opens a new block, it's likely an "empty block" problem // we can easily fix that by adding in a "pass" statement. However, we want to add that in the next line, if possible // so context ranges for autocompletion stay intact. if ( contents[emptySince] == QChar(':') ) { qCDebug(KDEV_PYTHON_PARSER) << indents.length() << emptySinceLine + 1 << indents; if ( indents.length() > emptySinceLine + 1 && indents.at(emptySinceLine) < indents.at(emptySinceLine + 1) ) { qCDebug(KDEV_PYTHON_PARSER) << indents.at(emptySinceLine) << indents.at(emptySinceLine + 1); contents.insert(emptyLinesSince + 1 + indents.at(emptyLinesSinceLine), "\tpass#"); } else { contents.insert(emptySince + 1, "\tpass#"); } } else if ( indents.length() >= currentLine && currentLine > 0 ) { qCDebug(KDEV_PYTHON_PARSER) << indents << currentLine; contents[i+1+indents.at(currentLine - 1)] = QChar('#'); contents.insert(i+1+indents.at(currentLine - 1), "pass"); } break; } } syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); // 3rd try: discard everything after the last non-empty line, but only until the next block start currentLineBeginning = qMin(contents.length() - 1, currentLineBeginning); errline = qMax(0, qMin(indents.length()-1, errline)); if ( ! syntaxtree ) { qCWarning(KDEV_PYTHON_PARSER) << "Discarding parts of the code to be parsed because of previous errors"; qCDebug(KDEV_PYTHON_PARSER) << indents; int indentAtError = indents.at(errline); QChar c; bool atLineBeginning = true; int currentIndent = -1; int currentLineBeginning_end = currentLineBeginning; int currentLineContentBeginning = currentLineBeginning; for ( int i = currentLineBeginning; i < len; i++ ) { c = contents.at(i); qCDebug(KDEV_PYTHON_PARSER) << c; if ( c == '\n' ) { if ( currentIndent <= indentAtError && currentIndent != -1 ) { qCDebug(KDEV_PYTHON_PARSER) << "Start of error code: " << currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "End of error block (current position): " << currentLineBeginning_end; qCDebug(KDEV_PYTHON_PARSER) << "Length: " << currentLineBeginning_end - currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "indent at error <> current indent:" << indentAtError << "<>" << currentIndent; // contents.remove(currentLineBeginning, currentLineBeginning_end-currentLineBeginning); break; } contents.insert(currentLineContentBeginning - 1, "pass#"); i += 5; i = qMin(i, contents.length()); len = contents.length(); atLineBeginning = true; currentIndent = 0; currentLineBeginning_end = i + 1; currentLineContentBeginning = i + 1; continue; } if ( ! c.isSpace() && atLineBeginning ) { currentLineContentBeginning = i; atLineBeginning = false; } if ( c.isSpace() && atLineBeginning ) currentIndent += 1; } qCDebug(KDEV_PYTHON_PARSER) << "This is what is left: " << contents; syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); } if ( ! syntaxtree ) { return CodeAst::Ptr(); // everything fails, so we abort. } } qCDebug(KDEV_PYTHON_PARSER) << "Got syntax tree from python parser:" << syntaxtree->kind << Module_kind; PythonAstTransformer t; t.run(syntaxtree, filename.fileName().replace(".py", "")); RangeFixVisitor fixVisitor(contents); fixVisitor.visitNode(t.ast); cythonSyntaxRemover.fixAstRanges(t.ast); return CodeAst::Ptr(t.ast); } } diff --git a/parser/astdefaultvisitor.cpp b/parser/astdefaultvisitor.cpp index 5db1f6f1..cdd31bbf 100644 --- a/parser/astdefaultvisitor.cpp +++ b/parser/astdefaultvisitor.cpp @@ -1,368 +1,374 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astdefaultvisitor.h" #include "ast.h" namespace Python { void free_ast_recursive(CodeAst *node) { AstFreeVisitor v; v.visitCode(node); } AstDefaultVisitor::AstDefaultVisitor() { } AstDefaultVisitor::~AstDefaultVisitor() { } // The Ast "ends" here, those don't have child nodes // note that Identifier is not a node in this Ast void AstDefaultVisitor::visitNameConstant(NameConstantAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitPass(PassAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNonlocal(NonlocalAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBreak(BreakAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitContinue(ContinueAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitEllipsis(EllipsisAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitNumber(NumberAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitString(StringAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitBytes(BytesAst* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitIdentifier(Identifier* node) { Q_UNUSED(node); } void AstDefaultVisitor::visitJoinedString(JoinedStringAst* node) { //TODO: Fix range/context/??? bugs, then re-enable this. Q_UNUSED(node); // foreach (Ast* value, node->values) { // visitNode(value); // } } void AstDefaultVisitor::visitFormattedValue(FormattedValueAst* node) { // TODO: Fix range issues, then re-enable (if required) Q_UNUSED(node); // visitNode(node->value); // visitNode(node->formatSpec); } void AstDefaultVisitor::visitStarred(StarredAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitArg(ArgAst* node) { visitNode(node->annotation); visitNode(node->argumentName); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitAlias(AliasAst* node) { visitIdentifier(node->name); visitIdentifier(node->asName); } void AstDefaultVisitor::visitName(NameAst* node) { visitIdentifier(node->identifier); } void AstDefaultVisitor::visitGlobal(GlobalAst* node) { visitNodeList(node->names); } void AstDefaultVisitor::visitCode(CodeAst* node) { visitNodeList(node->body); visitIdentifier(node->name); } void AstDefaultVisitor::visitExpression(ExpressionAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitYieldFrom(YieldFromAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAssertion(AssertionAst* node) { visitNode(node->condition); visitNode(node->message); } void AstDefaultVisitor::visitDelete(DeleteAst* node) { visitNodeList(node->targets); } void AstDefaultVisitor::visitExtendedSlice(ExtendedSliceAst* node) { visitNodeList(node->dims); } void AstDefaultVisitor::visitFor(ForAst* node) { visitNode(node->target); visitNode(node->iterator); visitNodeList(node->body); visitNodeList(node->orelse); } void AstDefaultVisitor::visitGeneratorExpression(GeneratorExpressionAst* node) { visitNode(node->element); visitNodeList(node->generators); } void AstDefaultVisitor::visitIf(IfAst* node) { visitNode(node->condition); visitNodeList(node->body); visitNodeList(node->orelse); } void AstDefaultVisitor::visitIfExpression(IfExpressionAst* node) { visitNode(node->condition); visitNode(node->body); visitNode(node->orelse); } void AstDefaultVisitor::visitImport(ImportAst* node) { visitNodeList(node->names); } void AstDefaultVisitor::visitImportFrom(ImportFromAst* node) { visitNodeList(node->names); visitIdentifier(node->module); } void AstDefaultVisitor::visitIndex(IndexAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitLambda(LambdaAst* node) { visitNode(node->arguments); visitNode(node->body); } void AstDefaultVisitor::visitRaise(RaiseAst* node) { visitNode(node->type); } void AstDefaultVisitor::visitReturn(ReturnAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitAwait(AwaitAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitSet(SetAst* node) { visitNodeList(node->elements); } void AstDefaultVisitor::visitSetComprehension(SetComprehensionAst* node) { visitNode(node->element); visitNodeList(node->generators); } void AstDefaultVisitor::visitSlice(SliceAst* node) { visitNode(node->lower); visitNode(node->upper); visitNode(node->step); } void AstDefaultVisitor::visitSubscript(SubscriptAst* node) { visitNode(node->value); visitNode(node->slice); } void AstDefaultVisitor::visitTry(TryAst* node) { visitNodeList(node->body); visitNodeList(node->handlers); visitNodeList(node->orelse); visitNodeList(node->finally); } void AstDefaultVisitor::visitTuple(TupleAst* node) { visitNodeList(node->elements); } void AstDefaultVisitor::visitUnaryOperation(UnaryOperationAst* node) { visitNode(node->operand); } void AstDefaultVisitor::visitWhile(WhileAst* node) { visitNode(node->condition); visitNodeList(node->body); visitNodeList(node->orelse); } void AstDefaultVisitor::visitWith(WithAst* node) { visitNodeList(node->items); visitNodeList(node->body); } void AstDefaultVisitor::visitWithItem(WithItemAst* node) { visitNode(node->contextExpression); visitNode(node->optionalVars); } void AstDefaultVisitor::visitYield(YieldAst* node) { visitNode(node->value); } void AstDefaultVisitor::visitList(ListAst* node) { visitNodeList(node->elements); } void AstDefaultVisitor::visitListComprehension(ListComprehensionAst* node) { visitNode(node->element); visitNodeList(node->generators); } void AstDefaultVisitor::visitExceptionHandler(ExceptionHandlerAst* node) { visitNode(node->type); visitNode(node->name); visitNodeList(node->body); } void AstDefaultVisitor::visitDict(DictAst* node) { visitNodeList(node->keys); visitNodeList(node->values); } void AstDefaultVisitor::visitDictionaryComprehension(DictionaryComprehensionAst* node) { visitNode(node->key); visitNode(node->value); visitNodeList(node->generators); } void AstDefaultVisitor::visitAugmentedAssignment(AugmentedAssignmentAst* node) { visitNode(node->target); visitNode(node->value); } void AstDefaultVisitor::visitAnnotationAssignment(AnnotationAssignmentAst* node) { visitNode(node->target); visitNode(node->annotation); visitNode(node->value); } +void AstDefaultVisitor::visitAssignmentExpression(AssignmentExpressionAst* node) +{ + visitNode(node->target); + visitNode(node->value); +} + void AstDefaultVisitor::visitBinaryOperation(BinaryOperationAst* node) { visitNode(node->lhs); visitNode(node->rhs); } void AstDefaultVisitor::visitBooleanOperation(BooleanOperationAst* node) { visitNodeList(node->values); } void AstDefaultVisitor::visitClassDefinition(ClassDefinitionAst* node) { visitNodeList(node->baseClasses); visitNodeList(node->body); visitNodeList(node->decorators); visitIdentifier(node->name); } void AstDefaultVisitor::visitCompare(CompareAst* node) { visitNode(node->leftmostElement); visitNodeList(node->comparands); } void AstDefaultVisitor::visitComprehension(ComprehensionAst* node) { visitNode(node->target); visitNode(node->iterator); visitNodeList(node->conditions); } void AstDefaultVisitor::visitAssignment(AssignmentAst* node) { visitNodeList(node->targets); visitNode(node->value); } void AstDefaultVisitor::visitCall(CallAst* node) { visitNode(node->function); visitNodeList(node->arguments); visitNodeList(node->keywords); } void AstDefaultVisitor::visitFunctionDefinition(FunctionDefinitionAst* node) { visitNodeList(node->decorators); visitNode(node->arguments); visitNode(node->returns); visitNodeList(node->body); visitIdentifier(node->name); } void AstDefaultVisitor::visitAttribute(AttributeAst* node) { visitNode(node->value); visitIdentifier(node->attribute); } void AstDefaultVisitor::visitKeyword(KeywordAst* node) { visitNode(node->value); visitIdentifier(node->argumentName); } void AstDefaultVisitor::visitArguments(ArgumentsAst* node) { visitNodeList(node->arguments); visitNodeList(node->defaultValues); } } diff --git a/parser/astdefaultvisitor.h b/parser/astdefaultvisitor.h index e8e0d3de..aa1f540c 100644 --- a/parser/astdefaultvisitor.h +++ b/parser/astdefaultvisitor.h @@ -1,185 +1,187 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2012 Patrick Spendrin * * Copyright 2010-2014 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef PYTHONASTDEFAULTVISITOR_H #define PYTHONASTDEFAULTVISITOR_H #include "astvisitor.h" #include "parserexport.h" /** * Note: This has been generated using utilities/generate.py * but you can modifiy it, it's not regenerated automatically */ namespace Python { class KDEVPYTHONPARSER_EXPORT AstDefaultVisitor : public AstVisitor { public: AstDefaultVisitor(); ~AstDefaultVisitor() override; void visitCode(CodeAst* node) override; void visitFunctionDefinition(FunctionDefinitionAst* node) override; void visitClassDefinition(ClassDefinitionAst* node) override; void visitReturn(ReturnAst* node) override; void visitAwait(AwaitAst* node) override; void visitDelete(DeleteAst* node) override; void visitAssignment(AssignmentAst* node) override; void visitAugmentedAssignment(AugmentedAssignmentAst* node) override; void visitAnnotationAssignment(AnnotationAssignmentAst* node) override; + void visitAssignmentExpression(AssignmentExpressionAst* node) override; void visitFor(ForAst* node) override; void visitWhile(WhileAst* node) override; void visitIf(IfAst* node) override; void visitWith(WithAst* node) override; void visitRaise(RaiseAst* node) override; void visitTry(TryAst* node) override; void visitAssertion(AssertionAst* node) override; void visitImport(ImportAst* node) override; void visitImportFrom(ImportFromAst* node) override; void visitGlobal(GlobalAst* node) override; void visitBreak(BreakAst* node) override; void visitContinue(ContinueAst* node) override; void visitPass(PassAst* node) override; void visitNonlocal(NonlocalAst* node) override; void visitBooleanOperation(BooleanOperationAst* node) override; void visitBinaryOperation(BinaryOperationAst* node) override; void visitUnaryOperation(UnaryOperationAst* node) override; void visitLambda(LambdaAst* node) override; void visitIfExpression(IfExpressionAst* node) override; void visitDict(DictAst* node) override; void visitSet(SetAst* node) override; void visitListComprehension(ListComprehensionAst* node) override; void visitSetComprehension(SetComprehensionAst* node) override; void visitDictionaryComprehension(DictionaryComprehensionAst* node) override; void visitGeneratorExpression(GeneratorExpressionAst* node) override; void visitCompare(CompareAst* node) override; void visitNumber(NumberAst* node) override; void visitString(StringAst* node) override; void visitJoinedString(JoinedStringAst* node) override; void visitFormattedValue(FormattedValueAst* node) override; void visitBytes(BytesAst* node) override; void visitYield(YieldAst* node) override; void visitYieldFrom(YieldFromAst* node) override; void visitName(NameAst* node) override; void visitNameConstant(NameConstantAst* node) override; void visitCall(CallAst* node) override; void visitAttribute(AttributeAst* node) override; void visitSubscript(SubscriptAst* node) override; void visitStarred(StarredAst* node) override; void visitList(ListAst* node) override; void visitTuple(TupleAst* node) override; void visitEllipsis(EllipsisAst* node) override; void visitSlice(SliceAst* node) override; void visitExtendedSlice(ExtendedSliceAst* node) override; void visitIndex(IndexAst* node) override; void visitArguments(ArgumentsAst* node) override; void visitArg(ArgAst* node) override; void visitKeyword(KeywordAst* node) override; void visitComprehension(ComprehensionAst* node) override; void visitExceptionHandler(ExceptionHandlerAst* node) override; void visitAlias(AliasAst* node) override; void visitExpression(ExpressionAst* node) override; void visitWithItem(WithItemAst* node) override; virtual void visitIdentifier(Identifier* node); }; class KDEVPYTHONPARSER_EXPORT AstFreeVisitor : public AstDefaultVisitor { public: /* * lines = open('test.dat', 'r').readlines() * for line in lines: print line.replace(';\n', ' { AstDefaultVisitor::visit'+ line.split('visit')[1] \ * .split('(')[0] +'(node); delete node; }') */ // The CodeAst should not free itself, as this is supposed to be called from ~CodeAst. void visitCode(CodeAst* node) override { AstDefaultVisitor::visitCode(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { AstDefaultVisitor::visitFunctionDefinition(node); delete node; } void visitClassDefinition(ClassDefinitionAst* node) override { AstDefaultVisitor::visitClassDefinition(node); delete node; } void visitReturn(ReturnAst* node) override { AstDefaultVisitor::visitReturn(node); delete node; } void visitDelete(DeleteAst* node) override { AstDefaultVisitor::visitDelete(node); delete node; } void visitAssignment(AssignmentAst* node) override { AstDefaultVisitor::visitAssignment(node); delete node; } void visitAugmentedAssignment(AugmentedAssignmentAst* node) override { AstDefaultVisitor::visitAugmentedAssignment(node); delete node; } void visitAnnotationAssignment(AnnotationAssignmentAst* node) override { AstDefaultVisitor::visitAnnotationAssignment(node); delete node; } + void visitAssignmentExpression(AssignmentExpressionAst* node) override { AstDefaultVisitor::visitAssignmentExpression(node); delete node; } void visitFor(ForAst* node) override { AstDefaultVisitor::visitFor(node); delete node; } void visitWhile(WhileAst* node) override { AstDefaultVisitor::visitWhile(node); delete node; } void visitIf(IfAst* node) override { AstDefaultVisitor::visitIf(node); delete node; } void visitWith(WithAst* node) override { AstDefaultVisitor::visitWith(node); delete node; } void visitRaise(RaiseAst* node) override { AstDefaultVisitor::visitRaise(node); delete node; } void visitTry(TryAst* node) override { AstDefaultVisitor::visitTry(node); delete node; } void visitAssertion(AssertionAst* node) override { AstDefaultVisitor::visitAssertion(node); delete node; } void visitImport(ImportAst* node) override { AstDefaultVisitor::visitImport(node); delete node; } void visitImportFrom(ImportFromAst* node) override { AstDefaultVisitor::visitImportFrom(node); delete node; } void visitGlobal(GlobalAst* node) override { AstDefaultVisitor::visitGlobal(node); delete node; } void visitBreak(BreakAst* node) override { AstDefaultVisitor::visitBreak(node); delete node; } void visitContinue(ContinueAst* node) override { AstDefaultVisitor::visitContinue(node); delete node; } void visitPass(PassAst* node) override { AstDefaultVisitor::visitPass(node); delete node; } void visitNonlocal(NonlocalAst* node) override { AstDefaultVisitor::visitNonlocal(node); delete node; } void visitBooleanOperation(BooleanOperationAst* node) override { AstDefaultVisitor::visitBooleanOperation(node); delete node; } void visitBinaryOperation(BinaryOperationAst* node) override { AstDefaultVisitor::visitBinaryOperation(node); delete node; } void visitUnaryOperation(UnaryOperationAst* node) override { AstDefaultVisitor::visitUnaryOperation(node); delete node; } void visitLambda(LambdaAst* node) override { AstDefaultVisitor::visitLambda(node); delete node; } void visitIfExpression(IfExpressionAst* node) override { AstDefaultVisitor::visitIfExpression(node); delete node; } void visitDict(DictAst* node) override { AstDefaultVisitor::visitDict(node); delete node; } void visitSet(SetAst* node) override { AstDefaultVisitor::visitSet(node); delete node; } void visitListComprehension(ListComprehensionAst* node) override { AstDefaultVisitor::visitListComprehension(node); delete node; } void visitSetComprehension(SetComprehensionAst* node) override { AstDefaultVisitor::visitSetComprehension(node); delete node; } void visitDictionaryComprehension(DictionaryComprehensionAst* node) override { AstDefaultVisitor::visitDictionaryComprehension(node); delete node; } void visitGeneratorExpression(GeneratorExpressionAst* node) override { AstDefaultVisitor::visitGeneratorExpression(node); delete node; } void visitCompare(CompareAst* node) override { AstDefaultVisitor::visitCompare(node); delete node; } void visitNumber(NumberAst* node) override { AstDefaultVisitor::visitNumber(node); delete node; } void visitString(StringAst* node) override { AstDefaultVisitor::visitString(node); delete node; } void visitJoinedString(JoinedStringAst* node) override { AstDefaultVisitor::visitJoinedString(node); delete node; } void visitFormattedValue(FormattedValueAst* node) override { AstDefaultVisitor::visitFormattedValue(node); delete node; } void visitBytes(BytesAst* node) override { AstDefaultVisitor::visitBytes(node); delete node; } void visitYield(YieldAst* node) override { AstDefaultVisitor::visitYield(node); delete node; } void visitYieldFrom(YieldFromAst* node) override { AstDefaultVisitor::visitYieldFrom(node); delete node; } void visitName(NameAst* node) override { AstDefaultVisitor::visitName(node); delete node; } void visitNameConstant(NameConstantAst* node) override { AstDefaultVisitor::visitNameConstant(node); delete node; } void visitCall(CallAst* node) override { AstDefaultVisitor::visitCall(node); delete node; } void visitAttribute(AttributeAst* node) override { AstDefaultVisitor::visitAttribute(node); delete node; } void visitSubscript(SubscriptAst* node) override { AstDefaultVisitor::visitSubscript(node); delete node; } void visitStarred(StarredAst* node) override { AstDefaultVisitor::visitStarred(node); delete node; } void visitList(ListAst* node) override { AstDefaultVisitor::visitList(node); delete node; } void visitTuple(TupleAst* node) override { AstDefaultVisitor::visitTuple(node); delete node; } void visitEllipsis(EllipsisAst* node) override { AstDefaultVisitor::visitEllipsis(node); delete node; } void visitSlice(SliceAst* node) override { AstDefaultVisitor::visitSlice(node); delete node; } void visitExtendedSlice(ExtendedSliceAst* node) override { AstDefaultVisitor::visitExtendedSlice(node); delete node; } void visitIndex(IndexAst* node) override { AstDefaultVisitor::visitIndex(node); delete node; } void visitArguments(ArgumentsAst* node) override { AstDefaultVisitor::visitArguments(node); delete node; } void visitArg(ArgAst* node) override { AstDefaultVisitor::visitArg(node); delete node; } void visitKeyword(KeywordAst* node) override { AstDefaultVisitor::visitKeyword(node); delete node; } void visitComprehension(ComprehensionAst* node) override { AstDefaultVisitor::visitComprehension(node); delete node; } void visitExceptionHandler(ExceptionHandlerAst* node) override { AstDefaultVisitor::visitExceptionHandler(node); delete node; } void visitAlias(AliasAst* node) override { AstDefaultVisitor::visitAlias(node); delete node; } void visitExpression(ExpressionAst* node) override { AstDefaultVisitor::visitExpression(node); delete node; } void visitWithItem(WithItemAst* node) override { AstDefaultVisitor::visitWithItem(node); delete node; } void visitIdentifier(Identifier* node) override { AstDefaultVisitor::visitIdentifier(node); delete node; } }; KDEVPYTHONPARSER_EXPORT void free_ast_recursive(CodeAst* node); } #endif diff --git a/parser/astvisitor.cpp b/parser/astvisitor.cpp index ace7d619..51c5a1c5 100644 --- a/parser/astvisitor.cpp +++ b/parser/astvisitor.cpp @@ -1,111 +1,112 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2011 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astvisitor.h" /** * Note: This has been generated using utilities/generate.py * but you can modifiy it, it's not regenerated automatically */ namespace Python { AstVisitor::AstVisitor() { } AstVisitor::~AstVisitor() { } void AstVisitor::visitNode(Ast* node) { if ( ! node ) return; switch ( node->astType ) { case Ast::CodeAstType: this->visitCode(static_cast(node)); break; case Ast::FunctionDefinitionAstType: this->visitFunctionDefinition(static_cast(node)); break; case Ast::ClassDefinitionAstType: this->visitClassDefinition(static_cast(node)); break; case Ast::ReturnAstType: this->visitReturn(static_cast(node)); break; case Ast::AwaitAstType: this->visitAwait(static_cast(node)); break; case Ast::DeleteAstType: this->visitDelete(static_cast(node)); break; case Ast::AssignmentAstType: this->visitAssignment(static_cast(node)); break; case Ast::AugmentedAssignmentAstType: this->visitAugmentedAssignment(static_cast(node)); break; case Ast::AnnotationAssignmentAstType: this->visitAnnotationAssignment(static_cast(node)); break; + case Ast::AssignmentExpressionAstType: this->visitAssignmentExpression(static_cast(node)); break; case Ast::ForAstType: this->visitFor(static_cast(node)); break; case Ast::WhileAstType: this->visitWhile(static_cast(node)); break; case Ast::IfAstType: this->visitIf(static_cast(node)); break; case Ast::WithAstType: this->visitWith(static_cast(node)); break; case Ast::RaiseAstType: this->visitRaise(static_cast(node)); break; case Ast::TryAstType: this->visitTry(static_cast(node)); break; case Ast::AssertionAstType: this->visitAssertion(static_cast(node)); break; case Ast::ImportAstType: this->visitImport(static_cast(node)); break; case Ast::ImportFromAstType: this->visitImportFrom(static_cast(node)); break; case Ast::GlobalAstType: this->visitGlobal(static_cast(node)); break; case Ast::BreakAstType: this->visitBreak(static_cast(node)); break; case Ast::ContinueAstType: this->visitContinue(static_cast(node)); break; case Ast::PassAstType: this->visitPass(static_cast(node)); break; case Ast::NonlocalAstType: this->visitNonlocal(static_cast(node)); break; case Ast::BooleanOperationAstType: this->visitBooleanOperation(static_cast(node)); break; case Ast::BinaryOperationAstType: this->visitBinaryOperation(static_cast(node)); break; case Ast::UnaryOperationAstType: this->visitUnaryOperation(static_cast(node)); break; case Ast::LambdaAstType: this->visitLambda(static_cast(node)); break; case Ast::IfExpressionAstType: this->visitIfExpression(static_cast(node)); break; case Ast::DictAstType: this->visitDict(static_cast(node)); break; case Ast::SetAstType: this->visitSet(static_cast(node)); break; case Ast::ListComprehensionAstType: this->visitListComprehension(static_cast(node)); break; case Ast::SetComprehensionAstType: this->visitSetComprehension(static_cast(node)); break; case Ast::DictionaryComprehensionAstType: this->visitDictionaryComprehension(static_cast(node)); break; case Ast::GeneratorExpressionAstType: this->visitGeneratorExpression(static_cast(node)); break; case Ast::CompareAstType: this->visitCompare(static_cast(node)); break; case Ast::NumberAstType: this->visitNumber(static_cast(node)); break; case Ast::StringAstType: this->visitString(static_cast(node)); break; case Ast::FormattedValueAstType: this->visitFormattedValue(static_cast(node)); break; case Ast::JoinedStringAstType: this->visitJoinedString(static_cast(node)); break; case Ast::BytesAstType: this->visitBytes(static_cast(node)); break; case Ast::YieldAstType: this->visitYield(static_cast(node)); break; case Ast::NameAstType: this->visitName(static_cast(node)); break; case Ast::NameConstantAstType: this->visitNameConstant(static_cast(node)); break; case Ast::CallAstType: this->visitCall(static_cast(node)); break; case Ast::AttributeAstType: this->visitAttribute(static_cast(node)); break; case Ast::SubscriptAstType: this->visitSubscript(static_cast(node)); break; case Ast::StarredAstType: this->visitStarred(static_cast(node)); break; case Ast::ListAstType: this->visitList(static_cast(node)); break; case Ast::TupleAstType: this->visitTuple(static_cast(node)); break; case Ast::EllipsisAstType: this->visitEllipsis(static_cast(node)); break; case Ast::SliceAstType: this->visitSlice(static_cast(node)); break; case Ast::ExtendedSliceAstType: this->visitExtendedSlice(static_cast(node)); break; case Ast::IndexAstType: this->visitIndex(static_cast(node)); break; case Ast::ArgumentsAstType: this->visitArguments(static_cast(node)); break; case Ast::KeywordAstType: this->visitKeyword(static_cast(node)); break; case Ast::ArgAstType: this->visitArg(static_cast(node)); break; case Ast::ComprehensionAstType: this->visitComprehension(static_cast(node)); break; case Ast::ExceptionHandlerAstType: this->visitExceptionHandler(static_cast(node)); break; case Ast::AliasAstType: this->visitAlias(static_cast(node)); break; case Ast::ExpressionAstType: this->visitExpression(static_cast(node)); break; case Ast::YieldFromAstType: this->visitYieldFrom(static_cast(node)); break; case Ast::WithItemAstType: this->visitWithItem(static_cast(node)); break; case Ast::IdentifierAstType: break; case Ast::StatementAstType: break; case Ast::LastAstType: Q_ASSERT(false); break; case Ast::LastStatementType: Q_ASSERT(false); break; case Ast::LastExpressionType: Q_ASSERT(false); break; } } } diff --git a/parser/astvisitor.h b/parser/astvisitor.h index 1908e9dd..51aa47b4 100644 --- a/parser/astvisitor.h +++ b/parser/astvisitor.h @@ -1,121 +1,122 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2012 Sven Brauch * * Copyright 2012 Patrick Spendrin * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef PYTHONASTVISITOR_H #define PYTHONASTVISITOR_H #include "ast.h" #include "parserexport.h" /** * Note: This has been generated using utilities/generate.py * but you can modifiy it, it's not regenerated automatically */ namespace Python { class KDEVPYTHONPARSER_EXPORT AstVisitor { public: AstVisitor(); virtual ~AstVisitor(); typedef void (AstVisitor::*visitFunc)(Ast *); virtual void visitNode(Ast* node); template void visitNodeList( const QList& l ) { foreach ( T* node, l ) { visitNode(node); } } virtual void visitCode(CodeAst* node) { Q_UNUSED(node); }; virtual void visitStatement(StatementAst* node) { Q_UNUSED(node); }; virtual void visitFunctionDefinition(FunctionDefinitionAst* node) { Q_UNUSED(node); }; virtual void visitClassDefinition(ClassDefinitionAst* node) { Q_UNUSED(node); }; virtual void visitReturn(ReturnAst* node) { Q_UNUSED(node); }; virtual void visitAwait(AwaitAst* node) { Q_UNUSED(node); }; virtual void visitDelete(DeleteAst* node) { Q_UNUSED(node); }; virtual void visitAssignment(AssignmentAst* node) { Q_UNUSED(node); }; virtual void visitAugmentedAssignment(AugmentedAssignmentAst* node) { Q_UNUSED(node); }; virtual void visitAnnotationAssignment(AnnotationAssignmentAst* node) { Q_UNUSED(node); }; + virtual void visitAssignmentExpression(AssignmentExpressionAst* node) { Q_UNUSED(node); }; virtual void visitFor(ForAst* node) { Q_UNUSED(node); }; virtual void visitWhile(WhileAst* node) { Q_UNUSED(node); }; virtual void visitIf(IfAst* node) { Q_UNUSED(node); }; virtual void visitWith(WithAst* node) { Q_UNUSED(node); }; virtual void visitRaise(RaiseAst* node) { Q_UNUSED(node); }; virtual void visitTry(TryAst* node) { Q_UNUSED(node); }; virtual void visitAssertion(AssertionAst* node) { Q_UNUSED(node); }; virtual void visitImport(ImportAst* node) { Q_UNUSED(node); }; virtual void visitImportFrom(ImportFromAst* node) { Q_UNUSED(node); }; virtual void visitGlobal(GlobalAst* node) { Q_UNUSED(node); }; virtual void visitBreak(BreakAst* node) { Q_UNUSED(node); }; virtual void visitContinue(ContinueAst* node) { Q_UNUSED(node); }; virtual void visitPass(PassAst* node) { Q_UNUSED(node); }; virtual void visitNonlocal(NonlocalAst* node) { Q_UNUSED(node); }; virtual void visitExpression(ExpressionAst* node) { Q_UNUSED(node); }; virtual void visitYieldFrom(YieldFromAst* node) { Q_UNUSED(node); }; virtual void visitBooleanOperation(BooleanOperationAst* node) { Q_UNUSED(node); }; virtual void visitBinaryOperation(BinaryOperationAst* node) { Q_UNUSED(node); }; virtual void visitUnaryOperation(UnaryOperationAst* node) { Q_UNUSED(node); }; virtual void visitLambda(LambdaAst* node) { Q_UNUSED(node); }; virtual void visitIfExpression(IfExpressionAst* node) { Q_UNUSED(node); }; virtual void visitDict(DictAst* node) { Q_UNUSED(node); }; virtual void visitSet(SetAst* node) { Q_UNUSED(node); }; virtual void visitListComprehension(ListComprehensionAst* node) { Q_UNUSED(node); }; virtual void visitSetComprehension(SetComprehensionAst* node) { Q_UNUSED(node); }; virtual void visitDictionaryComprehension(DictionaryComprehensionAst* node) { Q_UNUSED(node); }; virtual void visitGeneratorExpression(GeneratorExpressionAst* node) { Q_UNUSED(node); }; virtual void visitCompare(CompareAst* node) { Q_UNUSED(node); }; virtual void visitNumber(NumberAst* node) { Q_UNUSED(node); }; virtual void visitString(StringAst* node) { Q_UNUSED(node); }; virtual void visitFormattedValue(FormattedValueAst* node) { Q_UNUSED(node); }; virtual void visitJoinedString(JoinedStringAst* node) { Q_UNUSED(node); }; virtual void visitBytes(BytesAst* node) { Q_UNUSED(node); }; virtual void visitYield(YieldAst* node) { Q_UNUSED(node); }; virtual void visitName(NameAst* node) { Q_UNUSED(node); }; virtual void visitNameConstant(NameConstantAst* node) { Q_UNUSED(node); }; virtual void visitCall(CallAst* node) { Q_UNUSED(node); }; virtual void visitAttribute(AttributeAst* node) { Q_UNUSED(node); }; virtual void visitSubscript(SubscriptAst* node) { Q_UNUSED(node); }; virtual void visitStarred(StarredAst* node) { Q_UNUSED(node); }; virtual void visitList(ListAst* node) { Q_UNUSED(node); }; virtual void visitTuple(TupleAst* node) { Q_UNUSED(node); }; virtual void visitEllipsis(EllipsisAst* node) { Q_UNUSED(node); }; virtual void visitSlice(SliceAst* node) { Q_UNUSED(node); }; virtual void visitExtendedSlice(ExtendedSliceAst* node) { Q_UNUSED(node); }; virtual void visitIndex(IndexAst* node) { Q_UNUSED(node); }; virtual void visitArguments(ArgumentsAst* node) { Q_UNUSED(node); }; virtual void visitKeyword(KeywordAst* node) { Q_UNUSED(node); }; virtual void visitArg(ArgAst* node) { Q_UNUSED(node); }; virtual void visitComprehension(ComprehensionAst* node) { Q_UNUSED(node); }; virtual void visitExceptionHandler(ExceptionHandlerAst* node) { Q_UNUSED(node); }; virtual void visitAlias(AliasAst* node) { Q_UNUSED(node); }; virtual void visitWithItem(WithItemAst* node) { Q_UNUSED(node); }; }; } #endif diff --git a/parser/conversionGenerator.py b/parser/conversionGenerator.py index 9d65cc0d..66055953 100644 --- a/parser/conversionGenerator.py +++ b/parser/conversionGenerator.py @@ -1,320 +1,323 @@ #!/usr/bin/env python # Copyright 2014 by Sven Brauch # License: GPL v2+ # Transforms a conversion definition file (.sdef) into C++ code. To be copied over manually. :) # sdef example line: # RULE_FOR _stmt;KIND Expr_kind;ACTIONS create|ExpressionAst set|value->ExpressionAst,value;CODE;; import sys -contents = open('python36.sdef').read().replace("\n", "").split(';;') +contents = open('python38.sdef').read().replace("\n", "").split(';;') func_structure = ''' Ast* visitNode(%{RULE_FOR}* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { %{SWITCH_LINES} default: qWarning() << "Unsupported %{RULE_FOR} AST type: " << node->kind; Q_ASSERT(false); } %{APPENDIX} if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } ''' simple_func_structure = ''' Ast* visitNode(%{RULE_FOR}* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; %{SWITCH_LINES} return v; } ''' switch_line = ''' case %{KIND}: { %{ACTIONS} - result = v; break; }''' create_ast_line = ''' %{AST_TYPE}* v = new %{AST_TYPE}(parent());''' create_identifier_line = ''' v->%{TARGET} = node->v.%{KIND_W/O_SUFFIX}.%{VALUE} ? new Python::Identifier(PyUnicodeObjectToQString(node->v.%{KIND_W/O_SUFFIX}.%{VALUE})) : nullptr;''' set_attribute_line = ''' nodeStack.push(v); v->%{TARGET} = static_cast<%{AST_TYPE}*>(visitNode(node->v.%{KIND_W/O_SUFFIX}.%{VALUE})); nodeStack.pop();''' resolve_list_line = ''' nodeStack.push(v); v->%{TARGET} = visitNodeList<%{PYTHON_AST_TYPE}, %{AST_TYPE}>(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}); nodeStack.pop();''' create_identifier_line_any = ''' v->%{TARGET} = node->%{VALUE} ? new Python::Identifier(PyUnicodeObjectToQString(node->%{VALUE})) : nullptr;''' set_attribute_line_any = ''' nodeStack.push(v); v->%{TARGET} = static_cast<%{AST_TYPE}*>(visitNode(node->%{VALUE})); nodeStack.pop();''' resolve_list_line_any = ''' nodeStack.push(v); v->%{TARGET} = visitNodeList<%{PYTHON_AST_TYPE}, %{AST_TYPE}>(node->%{VALUE}); nodeStack.pop();''' direct_assignment_line = ''' v->%{TARGET} = node->v.%{KIND_W/O_SUFFIX}.%{VALUE};''' direct_assignment_line_any = ''' v->%{TARGET} = node->v.%{VALUE};''' cast_operator_line = ''' v->%{TARGET} = (ExpressionAst::%{AST_TYPE}) node->v.%{KIND_W/O_SUFFIX}.%{VALUE};''' resolve_string = ''' v->%{TARGET} = PyUnicodeObjectToQString(node->v.%{KIND_W/O_SUFFIX}.%{VALUE});''' assign_mindless = ''' v->%{TARGET} = node->%{VALUE};''' assign_linetransform = ''' v->%{TARGET} = tline(node->%{VALUE} - 1);''' singleton_convert_line = ''' v->%{TARGET} = node->v.NameConstant.value == Py_None ? NameConstantAst::None : node->v.NameConstant.value == Py_False ? NameConstantAst::False : NameConstantAst::True;''' resolve_oplist_block = ''' for ( int _i = 0; _i < node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->size; _i++ ) { v->%{TARGET}.append((ExpressionAst::%{AST_TYPE}) node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->elements[_i]); } ''' resolve_identifier_block = ''' for ( int _i = 0; _i < node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyUnicodeObjectToQString( static_cast(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->elements[_i]) )); v->%{TARGET}.append(id); } ''' copy_ident_ranges = ''' if ( v->%{TARGET} ) { v->%{TARGET}->startCol = node->col_offset; v->startCol = v->%{TARGET}->startCol; v->%{TARGET}->startLine = tline(node->lineno - 1); v->startLine = v->%{TARGET}->startLine; v->%{TARGET}->endCol = node->col_offset + v->%{TARGET}->value.length() - 1; v->endCol = v->%{TARGET}->endCol; v->%{TARGET}->endLine = tline(node->lineno - 1); v->endLine = v->%{TARGET}->endLine; ranges_copied = true; }''' results = dict() does_match_any = dict() def pluginAstToPythonAstType(plugintypestr): if plugintypestr == 'ExpressionAst': return '_expr' if plugintypestr == 'StatementAst' : return '_stmt' if plugintypestr == 'NameAst': return '_expr' if plugintypestr == 'ExceptionHandlerAst': return '_excepthandler' if plugintypestr == 'ComprehensionAst': return '_comprehension' if plugintypestr == 'KeywordAst': return '_keyword' if plugintypestr == 'ArgumentsAst': return '_arguments' if plugintypestr == 'AliasAst': return '_alias' if plugintypestr == 'SliceAst': return '_slice' if plugintypestr == 'Ast': return '_stmt' # not sure about this if plugintypestr == 'GeneratorExpressionAst': return '_expr' if plugintypestr == 'ArgAst': return '_arg' if plugintypestr == 'WithItemAst': return '_withitem' else: sys.stderr.write("W: Could not decode name %s\n" % plugintypestr) return '' for rule in contents: outline = rule.split(';') command = outline[0] if command[:7] == 'COMMENT' or command == '': continue elif command[:7] != 'RULE_FO': raise SyntaxError('Invalid syntax in sdef file, line: ' + rule) rule_for = outline[0].split(' ')[1] kind = outline[1].split(' ')[1] kind_wo_suffix = kind.replace('_kind', '') actions = outline[2].split(' ')[1:] code = None since_version = None before_version = None if len(outline) > 3: if outline[3].startswith('BEFORE'): before_version = [int(n) for n in outline[3][7:].split('.')] elif outline[3].startswith('SINCE'): since_version = [int(n) for n in outline[3][6:].split('.')] elif outline[3].startswith('CODE'): code = ' '.join(';'.join(outline[3:]).split('CODE')[1:]) + ";" else: raise SyntaxError('Invalid syntax in sdef file, line: ' + rule) if len(outline) > 4 and outline[4].startswith('CODE'): code = ' '.join(';'.join(outline[4:]).split('CODE')[1:]) + ";" if rule_for not in results: results[rule_for] = list() current_actions = list() + created_v = False for action in actions: command = action.split('|')[0] try: arguments = action.split('|')[1] except IndexError: continue action = '' if command == 'set': s = arguments.split('>') commandType = s[0][-1] # -, ~, =, : , *, # target = s[0][:-1] s = s[1].split(',') raw = False if kind == 'any': any = True else: any = False # commands with one argument if commandType in ['~', ':', '$', '+', 'l', '_']: if commandType == '_': raw = singleton_convert_line if commandType == ':': raw = direct_assignment_line if not any else direct_assignment_line_any if commandType == '~': raw = create_identifier_line if not any else create_identifier_line_any if rule_for in ['_expr', '_stmt', '_excepthandler', '_arg']: raw += copy_ident_ranges if commandType == '$': raw = resolve_string if commandType == '+': raw = assign_mindless; if commandType == 'l': raw = assign_linetransform; value = s[0] # commands with two arguments else: astType = s[0] try: value = s[1] except IndexError: raise SyntaxError('Missing argument for operator ' + commandType + ' in rule: ' + rule) if commandType == '=': if astType == 'Identifier': raw = resolve_identifier_block else: raw = resolve_list_line if not any else resolve_list_line_any if commandType == '-': raw = set_attribute_line if not any else set_attribute_line_any if commandType == '*': raw = cast_operator_line if commandType == '#': raw = resolve_oplist_block if raw: command = raw.replace('%{AST_TYPE}', astType).replace('%{TARGET}', target) \ .replace('%{PYTHON_AST_TYPE}', pluginAstToPythonAstType(astType)) \ .replace('%{KIND_W/O_SUFFIX}', kind_wo_suffix).replace('%{VALUE}', value) else: command = '' current_actions.append(command) elif command == 'create': astType = arguments current_actions.append(create_ast_line.replace('%{AST_TYPE}', astType)) - + created_v = True + if code: current_actions.append(code); - + current_actions = "\n".join(current_actions) if kind == 'any': current_stmt = current_actions else: + if created_v: + current_actions += "\n result = v;" current_stmt = switch_line.replace('%{KIND}', kind).replace('%{ACTIONS}', current_actions) if before_version: version_cpp_if = ("#if PYTHON_VERSION < QT_VERSION_CHECK(%d, %d, 0)\n" %(before_version[0], before_version[1])) current_stmt = version_cpp_if + current_stmt + "\n#endif" if since_version: version_cpp_if = ("#if PYTHON_VERSION >= QT_VERSION_CHECK(%d, %d, 0)\n" %(since_version[0], since_version[1])) current_stmt = version_cpp_if + current_stmt + "\n#endif" results[rule_for].append(current_stmt) does_match_any[rule_for] = any print('''/* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ #include #include "kdevpythonversion.h" class PythonAstTransformer { public: CodeAst* ast; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } // Shift lines by some fixed amount inline int tline(int line) { if ( line == -99999 ) { // don't touch the marker return -99999; } return line; }; private: QStack nodeStack; Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } ''') for index, lines in sorted(results.items()): current_switch_lines = "\n".join(lines) appendix = '' if index == '_expr' or index == '_stmt': appendix = ''' if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } ''' appendix += ''' // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } ''' if not does_match_any[index]: func = func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines).replace('%{APPENDIX}', appendix) else: func = simple_func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines) print(func) print('''}; /* * End generated code */ ''') diff --git a/parser/generated.h b/parser/generated.h index 9061deb0..e1136f84 100644 --- a/parser/generated.h +++ b/parser/generated.h @@ -1,792 +1,828 @@ /* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ #include #include "kdevpythonversion.h" class PythonAstTransformer { public: CodeAst* ast; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } // Shift lines by some fixed amount inline int tline(int line) { if ( line == -99999 ) { // don't touch the marker return -99999; } return line; }; private: QStack nodeStack; Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } Ast* visitNode(_alias* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; AliasAst* v = new AliasAst(parent()); v->name = node->name ? new Python::Identifier(PyUnicodeObjectToQString(node->name)) : nullptr; v->asName = node->asname ? new Python::Identifier(PyUnicodeObjectToQString(node->asname)) : nullptr; return v; } Ast* visitNode(_arg* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgAst* v = new ArgAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; if ( v->argumentName ) { v->argumentName->startCol = node->col_offset; v->startCol = v->argumentName->startCol; v->argumentName->startLine = tline(node->lineno - 1); v->startLine = v->argumentName->startLine; v->argumentName->endCol = node->col_offset + v->argumentName->value.length() - 1; v->endCol = v->argumentName->endCol; v->argumentName->endLine = tline(node->lineno - 1); v->endLine = v->argumentName->endLine; ranges_copied = true; } nodeStack.push(v); v->annotation = static_cast(visitNode(node->annotation)); nodeStack.pop(); return v; } Ast* visitNode(_arguments* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) ArgumentsAst* v = new ArgumentsAst(parent()); nodeStack.push(v); v->vararg = static_cast(visitNode(node->vararg)); nodeStack.pop(); nodeStack.push(v); v->kwarg = static_cast(visitNode(node->kwarg)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_arg, ArgAst>(node->args); nodeStack.pop(); nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); nodeStack.push(v); v->kwonlyargs = visitNodeList<_arg, ArgAst>(node->kwonlyargs); nodeStack.pop(); +#endif +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + ArgumentsAst* v = new ArgumentsAst(parent()); + nodeStack.push(v); v->vararg = static_cast(visitNode(node->vararg)); nodeStack.pop(); + nodeStack.push(v); v->kwarg = static_cast(visitNode(node->kwarg)); nodeStack.pop(); + nodeStack.push(v); v->arguments = visitNodeList<_arg, ArgAst>(node->args); nodeStack.pop(); + nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); + nodeStack.push(v); v->kwonlyargs = visitNodeList<_arg, ArgAst>(node->kwonlyargs); nodeStack.pop(); + nodeStack.push(v); v->posonlyargs = visitNodeList<_arg, ArgAst>(node->posonlyargs); nodeStack.pop(); +#endif return v; } Ast* visitNode(_comprehension* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ComprehensionAst* v = new ComprehensionAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->iter)); nodeStack.pop(); nodeStack.push(v); v->conditions = visitNodeList<_expr, ExpressionAst>(node->ifs); nodeStack.pop(); return v; } Ast* visitNode(_excepthandler* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case ExceptHandler_kind: { ExceptionHandlerAst* v = new ExceptionHandlerAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.ExceptHandler.type)); nodeStack.pop(); v->name = node->v.ExceptHandler.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ExceptHandler.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ExceptHandler.body); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _excepthandler AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_expr* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Await_kind: { AwaitAst* v = new AwaitAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Await.value)); nodeStack.pop(); result = v; break; } #endif case BoolOp_kind: { BooleanOperationAst* v = new BooleanOperationAst(parent()); v->type = (ExpressionAst::BooleanOperationTypes) node->v.BoolOp.op; nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.BoolOp.values); nodeStack.pop(); result = v; break; } case BinOp_kind: { BinaryOperationAst* v = new BinaryOperationAst(parent()); v->type = (ExpressionAst::OperatorTypes) node->v.BinOp.op; nodeStack.push(v); v->lhs = static_cast(visitNode(node->v.BinOp.left)); nodeStack.pop(); nodeStack.push(v); v->rhs = static_cast(visitNode(node->v.BinOp.right)); nodeStack.pop(); result = v; break; } case UnaryOp_kind: { UnaryOperationAst* v = new UnaryOperationAst(parent()); v->type = (ExpressionAst::UnaryOperatorTypes) node->v.UnaryOp.op; nodeStack.push(v); v->operand = static_cast(visitNode(node->v.UnaryOp.operand)); nodeStack.pop(); result = v; break; } case Lambda_kind: { LambdaAst* v = new LambdaAst(parent()); nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.Lambda.args)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Lambda.body)); nodeStack.pop(); result = v; break; } case IfExp_kind: { IfExpressionAst* v = new IfExpressionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.IfExp.test)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.IfExp.body)); nodeStack.pop(); nodeStack.push(v); v->orelse = static_cast(visitNode(node->v.IfExp.orelse)); nodeStack.pop(); result = v; break; } case Dict_kind: { DictAst* v = new DictAst(parent()); nodeStack.push(v); v->keys = visitNodeList<_expr, ExpressionAst>(node->v.Dict.keys); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Dict.values); nodeStack.pop(); result = v; break; } case Set_kind: { SetAst* v = new SetAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Set.elts); nodeStack.pop(); result = v; break; } case ListComp_kind: { ListComprehensionAst* v = new ListComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.ListComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.ListComp.generators); nodeStack.pop(); result = v; break; } case SetComp_kind: { SetComprehensionAst* v = new SetComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.SetComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.SetComp.generators); nodeStack.pop(); result = v; break; } case DictComp_kind: { DictionaryComprehensionAst* v = new DictionaryComprehensionAst(parent()); nodeStack.push(v); v->key = static_cast(visitNode(node->v.DictComp.key)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.DictComp.value)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.DictComp.generators); nodeStack.pop(); result = v; break; } case GeneratorExp_kind: { GeneratorExpressionAst* v = new GeneratorExpressionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.GeneratorExp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.GeneratorExp.generators); nodeStack.pop(); result = v; break; } case Yield_kind: { YieldAst* v = new YieldAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Yield.value)); nodeStack.pop(); result = v; break; } case Compare_kind: { CompareAst* v = new CompareAst(parent()); nodeStack.push(v); v->leftmostElement = static_cast(visitNode(node->v.Compare.left)); nodeStack.pop(); for ( int _i = 0; _i < node->v.Compare.ops->size; _i++ ) { v->operators.append((ExpressionAst::ComparisonOperatorTypes) node->v.Compare.ops->elements[_i]); } nodeStack.push(v); v->comparands = visitNodeList<_expr, ExpressionAst>(node->v.Compare.comparators); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION < QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop();};if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop();}; result = v; break; } #endif +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) case Num_kind: { NumberAst* v = new NumberAst(parent()); v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n); result = v; break; } +#endif +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) case Str_kind: { StringAst* v = new StringAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Str.s); result = v; break; } +#endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case JoinedStr_kind: { JoinedStringAst* v = new JoinedStringAst(parent()); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.JoinedStr.values); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case FormattedValue_kind: { FormattedValueAst* v = new FormattedValueAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.FormattedValue.value)); nodeStack.pop(); v->conversion = node->v.FormattedValue.conversion; nodeStack.push(v); v->formatSpec = static_cast(visitNode(node->v.FormattedValue.format_spec)); nodeStack.pop(); result = v; break; } #endif +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) case Bytes_kind: { BytesAst* v = new BytesAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Bytes.s); result = v; break; } +#endif case Attribute_kind: { AttributeAst* v = new AttributeAst(parent()); v->attribute = node->v.Attribute.attr ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Attribute.attr)) : nullptr; if ( v->attribute ) { v->attribute->startCol = node->col_offset; v->startCol = v->attribute->startCol; v->attribute->startLine = tline(node->lineno - 1); v->startLine = v->attribute->startLine; v->attribute->endCol = node->col_offset + v->attribute->value.length() - 1; v->endCol = v->attribute->endCol; v->attribute->endLine = tline(node->lineno - 1); v->endLine = v->attribute->endLine; ranges_copied = true; } nodeStack.push(v); v->value = static_cast(visitNode(node->v.Attribute.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Attribute.ctx; result = v; break; } case Subscript_kind: { SubscriptAst* v = new SubscriptAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Subscript.value)); nodeStack.pop(); nodeStack.push(v); v->slice = static_cast(visitNode(node->v.Subscript.slice)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Subscript.ctx; result = v; break; } case Starred_kind: { StarredAst* v = new StarredAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Starred.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Starred.ctx; result = v; break; } case Name_kind: { NameAst* v = new NameAst(parent()); v->identifier = node->v.Name.id ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Name.id)) : nullptr; if ( v->identifier ) { v->identifier->startCol = node->col_offset; v->startCol = v->identifier->startCol; v->identifier->startLine = tline(node->lineno - 1); v->startLine = v->identifier->startLine; v->identifier->endCol = node->col_offset + v->identifier->value.length() - 1; v->endCol = v->identifier->endCol; v->identifier->endLine = tline(node->lineno - 1); v->endLine = v->identifier->endLine; ranges_copied = true; } v->context = (ExpressionAst::Context) node->v.Name.ctx; result = v; break; } case List_kind: { ListAst* v = new ListAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.List.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.List.ctx; result = v; break; } case Tuple_kind: { TupleAst* v = new TupleAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Tuple.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Tuple.ctx; result = v; break; } +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) case Ellipsis_kind: { EllipsisAst* v = new EllipsisAst(parent()); result = v; break; } +#endif +#if PYTHON_VERSION < QT_VERSION_CHECK(3, 8, 0) case NameConstant_kind: { NameConstantAst* v = new NameConstantAst(parent()); v->value = node->v.NameConstant.value == Py_None ? NameConstantAst::None : node->v.NameConstant.value == Py_False ? NameConstantAst::False : NameConstantAst::True; result = v; break; } +#endif case YieldFrom_kind: { YieldFromAst* v = new YieldFromAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.YieldFrom.value)); nodeStack.pop(); result = v; break; } +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + case Constant_kind: { +PyObject *value = node->v.Constant.value;if (value == Py_None) { NameConstantAst* v = new NameConstantAst(parent()); v->value = NameConstantAst::None; result = v;}else if (value == Py_True) { NameConstantAst* v = new NameConstantAst(parent()); v->value = NameConstantAst::True; result = v;}else if (value == Py_False) { NameConstantAst* v = new NameConstantAst(parent()); v->value = NameConstantAst::False; result = v;}else if (value->ob_type == &PyLong_Type) { NumberAst* v = new NumberAst(parent()); v->isInt = true; v->value = PyLong_AsLong(value); result = v;}else if (value->ob_type == &PyFloat_Type || value->ob_type == &PyComplex_Type) { result = new NumberAst(parent());}else if (value->ob_type == &PyUnicode_Type) { StringAst* v = new StringAst(parent()); v->value = PyUnicodeObjectToQString(value); result = v;}else if (value->ob_type == &PyBytes_Type) { result = new BytesAst(parent());}else if (value->ob_type == &PyEllipsis_Type) { result = new EllipsisAst(parent());}else { qWarning() << "Unhandled constant type: " << value->ob_type->tp_name; Q_ASSERT(false);}; + break; + } +#endif +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + case NamedExpr_kind: { + AssignmentExpressionAst* v = new AssignmentExpressionAst(parent()); + nodeStack.push(v); v->target = static_cast(visitNode(node->v.NamedExpr.target)); nodeStack.pop(); + nodeStack.push(v); v->value = static_cast(visitNode(node->v.NamedExpr.value)); nodeStack.pop(); + result = v; + break; + } +#endif default: qWarning() << "Unsupported _expr AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_keyword* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; KeywordAst* v = new KeywordAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; nodeStack.push(v); v->value = static_cast(visitNode(node->value)); nodeStack.pop(); return v; } Ast* visitNode(_slice* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Slice_kind: { SliceAst* v = new SliceAst(parent()); nodeStack.push(v); v->lower = static_cast(visitNode(node->v.Slice.lower)); nodeStack.pop(); nodeStack.push(v); v->upper = static_cast(visitNode(node->v.Slice.upper)); nodeStack.pop(); nodeStack.push(v); v->step = static_cast(visitNode(node->v.Slice.step)); nodeStack.pop(); result = v; break; } case ExtSlice_kind: { ExtendedSliceAst* v = new ExtendedSliceAst(parent()); nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop(); result = v; break; } case Index_kind: { IndexAst* v = new IndexAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Index.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _slice AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_stmt* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Expr_kind: { ExpressionAst* v = new ExpressionAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Expr.value)); nodeStack.pop(); result = v; break; } case FunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.FunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.FunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.FunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.FunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.FunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.FunctionDef.returns)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.AsyncFunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.AsyncFunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.AsyncFunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.AsyncFunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.AsyncFunctionDef.returns)); nodeStack.pop(); v->async = true; result = v; break; } #endif case ClassDef_kind: { ClassDefinitionAst* v = new ClassDefinitionAst(parent()); v->name = node->v.ClassDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ClassDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->baseClasses = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.bases); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ClassDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.decorator_list); nodeStack.pop(); result = v; break; } case Return_kind: { ReturnAst* v = new ReturnAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Return.value)); nodeStack.pop(); result = v; break; } case Delete_kind: { DeleteAst* v = new DeleteAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Delete.targets); nodeStack.pop(); result = v; break; } case Assign_kind: { AssignmentAst* v = new AssignmentAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Assign.targets); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Assign.value)); nodeStack.pop(); result = v; break; } case AugAssign_kind: { AugmentedAssignmentAst* v = new AugmentedAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AugAssign.target)); nodeStack.pop(); v->op = (ExpressionAst::OperatorTypes) node->v.AugAssign.op; nodeStack.push(v); v->value = static_cast(visitNode(node->v.AugAssign.value)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case AnnAssign_kind: { AnnotationAssignmentAst* v = new AnnotationAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AnnAssign.target)); nodeStack.pop(); nodeStack.push(v); v->annotation = static_cast(visitNode(node->v.AnnAssign.annotation)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.AnnAssign.value)); nodeStack.pop(); result = v; break; } #endif case For_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.For.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.For.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.For.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.For.orelse); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFor_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AsyncFor.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.AsyncFor.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFor.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.AsyncFor.orelse); nodeStack.pop(); result = v; break; } #endif case While_kind: { WhileAst* v = new WhileAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.While.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.While.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.While.orelse); nodeStack.pop(); result = v; break; } case If_kind: { IfAst* v = new IfAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.If.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.If.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.If.orelse); nodeStack.pop(); result = v; break; } case With_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.With.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.With.items); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncWith_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncWith.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.AsyncWith.items); nodeStack.pop(); result = v; break; } #endif case Raise_kind: { RaiseAst* v = new RaiseAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.Raise.exc)); nodeStack.pop(); result = v; break; } case Try_kind: { TryAst* v = new TryAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.Try.body); nodeStack.pop(); nodeStack.push(v); v->handlers = visitNodeList<_excepthandler, ExceptionHandlerAst>(node->v.Try.handlers); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.Try.orelse); nodeStack.pop(); nodeStack.push(v); v->finally = visitNodeList<_stmt, Ast>(node->v.Try.finalbody); nodeStack.pop(); result = v; break; } case Assert_kind: { AssertionAst* v = new AssertionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.Assert.test)); nodeStack.pop(); nodeStack.push(v); v->message = static_cast(visitNode(node->v.Assert.msg)); nodeStack.pop(); result = v; break; } case Import_kind: { ImportAst* v = new ImportAst(parent()); nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.Import.names); nodeStack.pop(); result = v; break; } case ImportFrom_kind: { ImportFromAst* v = new ImportFromAst(parent()); v->module = node->v.ImportFrom.module ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ImportFrom.module)) : nullptr; if ( v->module ) { v->module->startCol = node->col_offset; v->startCol = v->module->startCol; v->module->startLine = tline(node->lineno - 1); v->startLine = v->module->startLine; v->module->endCol = node->col_offset + v->module->value.length() - 1; v->endCol = v->module->endCol; v->module->endLine = tline(node->lineno - 1); v->endLine = v->module->endLine; ranges_copied = true; } nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.ImportFrom.names); nodeStack.pop(); v->level = node->v.ImportFrom.level; result = v; break; } case Global_kind: { GlobalAst* v = new GlobalAst(parent()); for ( int _i = 0; _i < node->v.Global.names->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyUnicodeObjectToQString( static_cast(node->v.Global.names->elements[_i]) )); v->names.append(id); } result = v; break; } case Break_kind: { BreakAst* v = new BreakAst(parent()); result = v; break; } case Continue_kind: { ContinueAst* v = new ContinueAst(parent()); result = v; break; } case Pass_kind: { PassAst* v = new PassAst(parent()); result = v; break; } case Nonlocal_kind: { NonlocalAst* v = new NonlocalAst(parent()); result = v; break; } default: qWarning() << "Unsupported _stmt AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_withitem* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; WithItemAst* v = new WithItemAst(parent()); nodeStack.push(v); v->contextExpression = static_cast(visitNode(node->context_expr)); nodeStack.pop(); nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->optional_vars)); nodeStack.pop(); return v; } }; /* * End generated code */ diff --git a/parser/python36.sdef b/parser/python38.sdef similarity index 82% rename from parser/python36.sdef rename to parser/python38.sdef index f53ff1c7..30d9f95a 100644 --- a/parser/python36.sdef +++ b/parser/python38.sdef @@ -1,102 +1,145 @@ COMMENT;This file specifies rules for a conversion of a python (C) to a plugin-internal (C++) syntax tree;; COMMENT;All lines are terminated by a double semicolon token, newlines are ignored.;; COMMENT;-> stands for "convert AST", => for "convert AST list", ~> for "make an identifier", :> is an assignment w/o conversion, and *> is an enum cast.;; COMMENT;$> is a string assignment, +> is a "mindless assignment, like, it just does v->arg = node->arg. COMMENT;_> is a Singleton Assignment (converts Py_True, Py_False, Py_None to enum) COMMENT;the CODE statement can be used to add custom code.;; RULE_FOR _stmt;KIND Expr_kind;ACTIONS create|ExpressionAst set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND FunctionDef_kind;ACTIONS create|FunctionDefinitionAst set|name~>name set|arguments->ArgumentsAst,args set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list set|returns->ExpressionAst,returns;; RULE_FOR _stmt;KIND AsyncFunctionDef_kind;ACTIONS create|FunctionDefinitionAst set|name~>name set|arguments->ArgumentsAst,args set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list set|returns->ExpressionAst,returns;SINCE 3.5;CODE v->async = true;; RULE_FOR _stmt;KIND ClassDef_kind;ACTIONS create|ClassDefinitionAst set|name~>name set|baseClasses=>ExpressionAst,bases set|body=>Ast,body set|decorators=>ExpressionAst,decorator_list;; RULE_FOR _stmt;KIND Return_kind;ACTIONS create|ReturnAst set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND Delete_kind;ACTIONS create|DeleteAst set|targets=>ExpressionAst,targets;; RULE_FOR _stmt;KIND Assign_kind;ACTIONS create|AssignmentAst set|targets=>ExpressionAst,targets set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND AugAssign_kind;ACTIONS create|AugmentedAssignmentAst set|target->ExpressionAst,target set|op*>OperatorTypes,op set|value->ExpressionAst,value;; RULE_FOR _stmt;KIND AnnAssign_kind;ACTIONS create|AnnotationAssignmentAst set|target->ExpressionAst,target set|annotation->ExpressionAst,annotation set|value->ExpressionAst,value;SINCE 3.6;; RULE_FOR _stmt;KIND For_kind;ACTIONS create|ForAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND AsyncFor_kind;ACTIONS create|ForAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|body=>Ast,body set|orelse=>Ast,orelse;SINCE 3.5;; RULE_FOR _stmt;KIND While_kind;ACTIONS create|WhileAst set|condition->ExpressionAst,test set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND If_kind;ACTIONS create|IfAst set|condition->ExpressionAst,test set|body=>Ast,body set|orelse=>Ast,orelse;; RULE_FOR _stmt;KIND With_kind;ACTIONS create|WithAst set|body=>Ast,body set|items=>WithItemAst,items;; RULE_FOR _stmt;KIND AsyncWith_kind;ACTIONS create|WithAst set|body=>Ast,body set|items=>WithItemAst,items;SINCE 3.5;; COMMENT;FIXME: the struct Raise has changed, so the following line is likely wrong;; RULE_FOR _stmt;KIND Raise_kind;ACTIONS create|RaiseAst set|type->ExpressionAst,exc;; RULE_FOR _stmt;KIND Try_kind;ACTIONS create|TryAst set|body=>Ast,body set|handlers=>ExceptionHandlerAst,handlers set|orelse=>Ast,orelse set|finally=>Ast,finalbody;; RULE_FOR _stmt;KIND Assert_kind;ACTIONS create|AssertionAst set|condition->ExpressionAst,test set|message->ExpressionAst,msg;; RULE_FOR _stmt;KIND Import_kind;ACTIONS create|ImportAst set|names=>AliasAst,names;; RULE_FOR _stmt;KIND ImportFrom_kind;ACTIONS create|ImportFromAst set|module~>module set|names=>AliasAst,names set|level:>level;; RULE_FOR _stmt;KIND Global_kind;ACTIONS create|GlobalAst set|names=>Identifier,names;; RULE_FOR _stmt;KIND Break_kind;ACTIONS create|BreakAst;; RULE_FOR _stmt;KIND Continue_kind;ACTIONS create|ContinueAst;; RULE_FOR _stmt;KIND Pass_kind;ACTIONS create|PassAst;; RULE_FOR _stmt;KIND Nonlocal_kind;ACTIONS create|NonlocalAst;; RULE_FOR _expr;KIND Await_kind;ACTIONS create|AwaitAst set|value->ExpressionAst,value;SINCE 3.5;; RULE_FOR _expr;KIND BoolOp_kind;ACTIONS create|BooleanOperationAst set|type*>BooleanOperationTypes,op set|values=>ExpressionAst,values;; RULE_FOR _expr;KIND BinOp_kind;ACTIONS create|BinaryOperationAst set|type*>OperatorTypes,op set|lhs->ExpressionAst,left set|rhs->ExpressionAst,right;; RULE_FOR _expr;KIND UnaryOp_kind;ACTIONS create|UnaryOperationAst set|type*>UnaryOperatorTypes,op set|operand->ExpressionAst,operand;; RULE_FOR _expr;KIND Lambda_kind;ACTIONS create|LambdaAst set|arguments->ArgumentsAst,args set|body->ExpressionAst,body;; RULE_FOR _expr;KIND IfExp_kind;ACTIONS create|IfExpressionAst set|condition->ExpressionAst,test set|body->ExpressionAst,body set|orelse->ExpressionAst,orelse;; RULE_FOR _expr;KIND Dict_kind;ACTIONS create|DictAst set|keys=>ExpressionAst,keys set|values=>ExpressionAst,values;; RULE_FOR _expr;KIND Set_kind;ACTIONS create|SetAst set|elements=>ExpressionAst,elts;; RULE_FOR _expr;KIND ListComp_kind;ACTIONS create|ListComprehensionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND SetComp_kind;ACTIONS create|SetComprehensionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND DictComp_kind;ACTIONS create|DictionaryComprehensionAst set|key->ExpressionAst,key set|value->ExpressionAst,value set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND GeneratorExp_kind;ACTIONS create|GeneratorExpressionAst set|element->ExpressionAst,elt set|generators=>ComprehensionAst,generators;; RULE_FOR _expr;KIND Yield_kind;ACTIONS create|YieldAst set|value->ExpressionAst,value;; RULE_FOR _expr;KIND Compare_kind;ACTIONS create|CompareAst set|leftmostElement->ExpressionAst,left set|operators#>ComparisonOperatorTypes,ops set|comparands=>ExpressionAst,comparators;; RULE_FOR _expr;KIND Call_kind;ACTIONS create|CallAst set|function->ExpressionAst,func set|arguments=>ExpressionAst,args set|keywords=>KeywordAst,keywords;SINCE 3.5;; RULE_FOR _expr;KIND Call_kind;ACTIONS create|CallAst set|function->ExpressionAst,func set|arguments=>ExpressionAst,args set|keywords=>KeywordAst,keywords;BEFORE 3.5; CODE /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */ if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop(); }; if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop(); };; -RULE_FOR _expr;KIND Num_kind;ACTIONS create|NumberAst;CODE v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n);; -RULE_FOR _expr;KIND Str_kind;ACTIONS create|StringAst set|value$>s;; +RULE_FOR _expr;KIND Num_kind;ACTIONS create|NumberAst;BEFORE 3.8;CODE v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n);; +RULE_FOR _expr;KIND Str_kind;ACTIONS create|StringAst set|value$>s;BEFORE 3.8;; RULE_FOR _expr;KIND JoinedStr_kind;ACTIONS create|JoinedStringAst set|values=>ExpressionAst,values;SINCE 3.6;; RULE_FOR _expr;KIND FormattedValue_kind;ACTIONS create|FormattedValueAst set|value->ExpressionAst,value set|conversion:>conversion set|formatSpec->ExpressionAst,format_spec;SINCE 3.6;; -RULE_FOR _expr;KIND Bytes_kind;ACTIONS create|BytesAst set|value$>s;; +RULE_FOR _expr;KIND Bytes_kind;ACTIONS create|BytesAst set|value$>s;BEFORE 3.8;; RULE_FOR _expr;KIND Attribute_kind;ACTIONS create|AttributeAst set|attribute~>attr set|value->ExpressionAst,value set|context*>Context,ctx;; RULE_FOR _expr;KIND Subscript_kind;ACTIONS create|SubscriptAst set|value->ExpressionAst,value set|slice->SliceAst,slice set|context*>Context,ctx;; RULE_FOR _expr;KIND Starred_kind;ACTIONS create|StarredAst set|value->ExpressionAst,value set|context*>Context,ctx;; RULE_FOR _expr;KIND Name_kind;ACTIONS create|NameAst set|identifier~>id set|context*>Context,ctx;; RULE_FOR _expr;KIND List_kind;ACTIONS create|ListAst set|elements=>ExpressionAst,elts set|context*>Context,ctx;; RULE_FOR _expr;KIND Tuple_kind;ACTIONS create|TupleAst set|elements=>ExpressionAst,elts set|context*>Context,ctx;; -RULE_FOR _expr;KIND Ellipsis_kind;ACTIONS create|EllipsisAst;; -RULE_FOR _expr;KIND NameConstant_kind;ACTIONS create|NameConstantAst set|value_>value;; +RULE_FOR _expr;KIND Ellipsis_kind;ACTIONS create|EllipsisAst;BEFORE 3.8;; +RULE_FOR _expr;KIND NameConstant_kind;ACTIONS create|NameConstantAst set|value_>value;BEFORE 3.8;; RULE_FOR _expr;KIND YieldFrom_kind;ACTIONS create|YieldFromAst set|value->ExpressionAst,value;; +RULE_FOR _expr;KIND Constant_kind;ACTIONS;SINCE 3.8;CODE +PyObject *value = node->v.Constant.value; +if (value == Py_None) { + NameConstantAst* v = new NameConstantAst(parent()); + v->value = NameConstantAst::None; + result = v; +} +else if (value == Py_True) { + NameConstantAst* v = new NameConstantAst(parent()); + v->value = NameConstantAst::True; + result = v; +} +else if (value == Py_False) { + NameConstantAst* v = new NameConstantAst(parent()); + v->value = NameConstantAst::False; + result = v; +} +else if (value->ob_type == &PyLong_Type) { + NumberAst* v = new NumberAst(parent()); + v->isInt = true; + v->value = PyLong_AsLong(value); + result = v; +} +else if (value->ob_type == &PyFloat_Type || value->ob_type == &PyComplex_Type) { + result = new NumberAst(parent()); +} +else if (value->ob_type == &PyUnicode_Type) { + StringAst* v = new StringAst(parent()); + v->value = PyUnicodeObjectToQString(value); + result = v; +} +else if (value->ob_type == &PyBytes_Type) { + result = new BytesAst(parent()); +} +else if (value->ob_type == &PyEllipsis_Type) { + result = new EllipsisAst(parent()); +} +else { + qWarning() << "Unhandled constant type: " << value->ob_type->tp_name; + Q_ASSERT(false); +};; +RULE_FOR _expr;KIND NamedExpr_kind;ACTIONS create|AssignmentExpressionAst set|target->ExpressionAst,target set|value->ExpressionAst,value;SINCE 3.8;; RULE_FOR _slice;KIND Slice_kind;ACTIONS create|SliceAst set|lower->ExpressionAst,lower set|upper->ExpressionAst,upper set|step->ExpressionAst,step;; RULE_FOR _slice;KIND ExtSlice_kind;ACTIONS create|ExtendedSliceAst set|dims=>SliceAst,dims;; RULE_FOR _slice;KIND Index_kind;ACTIONS create|IndexAst set|value->ExpressionAst,value;; RULE_FOR _comprehension;KIND any;ACTIONS create|ComprehensionAst set|target->ExpressionAst,target set|iterator->ExpressionAst,iter set|conditions=>ExpressionAst,ifs;; RULE_FOR _excepthandler;KIND ExceptHandler_kind;ACTIONS create|ExceptionHandlerAst set|type->ExpressionAst,type set|name~>name set|body=>Ast,body;; -RULE_FOR _arguments;KIND any;ACTIONS create|ArgumentsAst set|vararg->ArgAst,vararg set|kwarg->ArgAst,kwarg set|arguments=>ArgAst,args set|defaultValues=>ExpressionAst,defaults set|kwonlyargs=>ArgAst,kwonlyargs;; +RULE_FOR _arguments;KIND any;ACTIONS create|ArgumentsAst set|vararg->ArgAst,vararg set|kwarg->ArgAst,kwarg set|arguments=>ArgAst,args set|defaultValues=>ExpressionAst,defaults set|kwonlyargs=>ArgAst,kwonlyargs;BEFORE 3.8;; +RULE_FOR _arguments;KIND any;ACTIONS create|ArgumentsAst set|vararg->ArgAst,vararg set|kwarg->ArgAst,kwarg set|arguments=>ArgAst,args set|defaultValues=>ExpressionAst,defaults set|kwonlyargs=>ArgAst,kwonlyargs set|posonlyargs=>ArgAst,posonlyargs;SINCE 3.8;; RULE_FOR _arg;KIND any;ACTIONS create|ArgAst set|argumentName~>arg set|annotation->ExpressionAst,annotation;; RULE_FOR _keyword;KIND any;ACTIONS create|KeywordAst set|argumentName~>arg set|value->ExpressionAst,value;; RULE_FOR _alias;KIND any;ACTIONS create|AliasAst set|name~>name set|asName~>asname;; RULE_FOR _withitem;KIND any; ACTIONS create|WithItemAst set|contextExpression->ExpressionAst,context_expr set|optionalVars->ExpressionAst,optional_vars;; diff --git a/parser/tests/pyasttest.cpp b/parser/tests/pyasttest.cpp index 4fd5f4eb..c28f6fce 100644 --- a/parser/tests/pyasttest.cpp +++ b/parser/tests/pyasttest.cpp @@ -1,296 +1,301 @@ /* This file is part of kdev-python, the python language plugin for KDevelop Copyright (C) 2012 Sven Brauch svenbrauch@googlemail.com This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "kdevpythonversion.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" #include "pyasttest.h" #include "../astbuilder.h" #include "../parserdebug.h" #include #include #include using namespace Python; QTEST_MAIN(PyAstTest) PyAstTest::PyAstTest(QObject* parent): QObject(parent) { initShell(); } void PyAstTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } CodeAst::Ptr PyAstTest::getAst(QString code) { QSharedPointer builder(new AstBuilder); CodeAst::Ptr result = builder->parse(QUrl(""), code); return result; } class VerifyVisitor : public AstDefaultVisitor { public: VerifyVisitor() : AstDefaultVisitor(), m_nodecount(0) { }; void visitNode(Ast* node) override { m_nodecount += 1; QVERIFY(! node || node->astType < Ast::LastAstType); AstDefaultVisitor::visitNode(node); }; void visitName(NameAst* node) override { QVERIFY(! node->identifier->value.isNull()); AstDefaultVisitor::visitName(node); }; void visitCode(CodeAst* node) override { AstDefaultVisitor::visitCode(node); qDebug() << "done, nodes visited:" << m_nodecount; }; int m_nodecount; }; void PyAstTest::testCode(QString code) { CodeAst::Ptr ast = getAst(code); VerifyVisitor v; v.visitCode(ast.data()); } void PyAstTest::testExceptionHandlers() { QString code = "try: pass\n" "except FooBar as baz: pass\n" "except Cat as baz: pass\n" "except Dawg as asdf: pass\n"; CodeAst::Ptr ast = getAst(code); VerifyVisitor v; v.visitCode(ast.data()); QCOMPARE(ast->body.size(), 1); QVERIFY(ast->body.first()->astType == Ast::TryAstType); TryAst* try_ = static_cast(ast->body.first()); QCOMPARE(try_->handlers.size(), 3); foreach ( ExceptionHandlerAst* handler, try_->handlers ) { QVERIFY(handler->name); QCOMPARE(handler->name->astType, Ast::IdentifierAstType); } } void PyAstTest::testStatements() { QFETCH(QString, code); testCode(code); } void PyAstTest::testStatements_data() { QTest::addColumn("code"); QTest::newRow("assign_int") << "a = 3"; QTest::newRow("funcdef") << "def myfun(): pass"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("asyncfuncdef") << "async def myfun(): pass"; QTest::newRow("asyncfuncdef2") << "async def myfun(): pass"; QTest::newRow("asyncfuncdef_await") << "async def myfun(): await 3"; #endif QTest::newRow("funcdef_args") << "def myfun(arg1, arg2): pass"; QTest::newRow("funcdef_vararg") << "def myfun(arg1, *arg): pass"; QTest::newRow("funcdef_kwarg") << "def myfun(**arg): pass"; QTest::newRow("classdef_inheritance") << "class myclass(parent): pass"; QTest::newRow("return") << "return 3"; QTest::newRow("for") << "for i in 1, 2, 3: pass"; QTest::newRow("while") << "while True: pass"; QTest::newRow("if") << "if True: pass"; QTest::newRow("ifElse") << "if True: pass\nelse:pass"; QTest::newRow("with") << "with x as y: pass"; QTest::newRow("raise") << "raise Exception"; QTest::newRow("tryexcept") << "try:pass\nexcept:pass"; QTest::newRow("tryexceptfinally") << "try:pass\nexcept:pass\nfinally:pass"; QTest::newRow("assert") << "assert false"; QTest::newRow("import") << "import foobar"; QTest::newRow("importfrom") << "from foobar import bazbang"; QTest::newRow("global") << "global x"; QTest::newRow("break") << "while True: break"; QTest::newRow("continue") << "while True: continue"; QTest::newRow("pass") << "pass"; QTest::newRow("nonlocal") << "nonlocal x"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("varannotation1") << "primes: List[int] = []"; QTest::newRow("varannotation2") << "captain: str # Note: no initial value!"; #endif } void PyAstTest::testSlices() { QFETCH(QString, code); testCode(code); } void PyAstTest::testSlices_data() { QTest::addColumn("code"); QTest::newRow("slice1") << "x[1]"; QTest::newRow("slice2") << "x[2:3]"; QTest::newRow("slice3") << "x[::]"; QTest::newRow("slice4") << "x[1:2:3]"; } void PyAstTest::testOther() { QFETCH(QString, code); testCode(code); } void PyAstTest::testOther_data() { QTest::addColumn("code"); QTest::newRow("args") << "foo(bar, baz, *bang, **baa)"; QTest::newRow("kws") << "foo(bar=baz, bang=3)"; QTest::newRow("excpthandler") << "try:pass\nexcept foo as bar: pass"; QTest::newRow("alias") << "import foo as bar"; } void PyAstTest::testExpressions() { QFETCH(QString, code); testCode(code); } void PyAstTest::testExpressions_data() { QTest::addColumn("code"); QTest::newRow("boolop") << "b or c"; QTest::newRow("binop") << "b ^ c"; QTest::newRow("unop") << "not a"; QTest::newRow("lambda") << "lambda x: y"; QTest::newRow("ifexpr") << "3 if 4 else 5"; QTest::newRow("dict") << "{}"; QTest::newRow("set") << "(3, 5)"; QTest::newRow("listcomp") << "[x for x in y]"; QTest::newRow("setcomp") << "(x for x in y)"; QTest::newRow("dictcomp") << "{x:y for x, y in z}"; QTest::newRow("comp") << "x < y"; QTest::newRow("number") << "3"; QTest::newRow("string") << "\"foo\""; QTest::newRow("bytes") << "b\"bytes\""; QTest::newRow("yield") << "yield x"; QTest::newRow("name") << "foo"; QTest::newRow("call") << "foo()"; QTest::newRow("attribute") << "foo.bar"; QTest::newRow("attribute_nontoplevel") << "while True: foo.bar"; QTest::newRow("subscript") << "foo[3]"; QTest::newRow("starred") << "*[1, 2, 3 ,4]"; QTest::newRow("list") << "[]"; QTest::newRow("tuple") << "()"; QTest::newRow("None") << "None"; QTest::newRow("False") << "False"; QTest::newRow("True") << "True"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("async_generator") << "async def foo(): result = [i async for i in aiter() if i % 2]"; QTest::newRow("await_generator") << "async def foo(): result = [await fun() for fun in funcs]"; QTest::newRow("underscore_literals") << "0x_FF_FF_FF_FF"; QTest::newRow("formatted_string_literal") << "f\"He said his name is {name}.\""; #endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("dstar_unpack") << "ext_map = {\n" " **{ext: self.obj_extension for ext in self.src_extensions},\n" " **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions},\n" "}"; #endif +#if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0) + QTest::newRow("assignment_expr_1") << "a = (b := 10)"; + QTest::newRow("assignment_expr_2") << "a = [q for z in (1, 2, 3) if (q := 2*z)]"; + QTest::newRow("positional_params") << "def foo(a, b, /, c, d, *, e): pass"; +#endif } void PyAstTest::testCorrectedFuncRanges() { QFETCH(QString, code); QFETCH(KTextEditor::Range, range); CodeAst::Ptr ast = getAst(code); QVERIFY(ast); foreach ( Ast* node, ast->body ) { if ( node->astType != Ast::FunctionDefinitionAstType ) { continue; } FunctionDefinitionAst* func = static_cast(node); QVERIFY(func->name); qCDebug(KDEV_PYTHON_PARSER) << func->name->range() << range; QCOMPARE(func->name->range(), range); } } void PyAstTest::testCorrectedFuncRanges_data() { QTest::addColumn("code"); QTest::addColumn("range"); QTest::newRow("decorator") << "@decorate\ndef func(arg): pass" << KTextEditor::Range(1, 4, 1, 7); QTest::newRow("decorator_arg") << "@decorate(yomama=3)\ndef func(arg): pass" << KTextEditor::Range(1, 4, 1, 7); QTest::newRow("two_decorators") << "@decorate2\n@decorate\ndef func(arg): pass" << KTextEditor::Range(2, 4, 2, 7); QTest::newRow("decorate_class") << "class foo:\n @decorate2\n @decorate\n def func(arg): pass" << KTextEditor::Range(3, 5, 3, 8); } void PyAstTest::testNewPython3() { QFETCH(QString, code); testCode(code); } void PyAstTest::testNewPython3_data() { QTest::addColumn("code"); QTest::newRow("funcannotation1") << "def haul(item: Haulable, *vargs: PackAnimal) -> Distance: pass"; QTest::newRow("funcannotation2") << "def foo() -> expr: pass"; QTest::newRow("funcannotation3") << "def foo(a: 'x', b: 5 + 6, c: list) -> max(2, 9): pass"; QTest::newRow("kwonly1") << "def compare(a, b, *, key=None): pass"; QTest::newRow("kwonly2") << "def foo(a: 'x', b: 5 + 6, c: list) -> max(2, 9): pass"; QTest::newRow("listunpack") << "(a, *rest, b) = range(5)"; QTest::newRow("metaclass") << "class C(metaclass=M): pass"; QTest::newRow("exception_chain") << "raise SecondaryException() from primary_exception"; QTest::newRow("yield_from") << "def foo(): yield from [1, 2, 3]"; } void PyAstTest::testClass() { testCode("class c: pass"); }