diff --git a/CMakeLists.txt b/CMakeLists.txt index 13a76f2e..884bef0d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,143 +1,143 @@ cmake_minimum_required(VERSION 3.0) project(kdevpython) # write the plugin version to a file set(KDEVPYTHON_VERSION_MAJOR 5) set(KDEVPYTHON_VERSION_MINOR 1) -set(KDEVPYTHON_VERSION_PATCH 40) +set(KDEVPYTHON_VERSION_PATCH 80) # KDevplatform dependency version set( KDEVPLATFORM_VERSION "${KDEVPYTHON_VERSION_MAJOR}.${KDEVPYTHON_VERSION_MINOR}" ) find_package (ECM "5.14.0" REQUIRED NO_MODULE) set(CMAKE_MODULE_PATH ${kdevpython_SOURCE_DIR}/cmake/modules ${CMAKE_MODULE_PATH} ${ECM_MODULE_PATH}) include(KDECompilerSettings NO_POLICY_SCOPE) include(GenerateExportHeader) include(CMakePackageConfigHelpers) include(ECMAddTests) include(ECMOptionalAddSubdirectory) include(ECMSetupVersion) include(ECMQtDeclareLoggingCategory) include(KDEInstallDirs) include(KDECMakeSettings) if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wdocumentation") endif() add_definitions( -DTRANSLATION_DOMAIN=\"kdevpython\" ) # CMake looks for exactly the specified version first and ignores newer versions. # To avoid that, start looking for the newest supported version and work down. set(Python_ADDITIONAL_VERSIONS 3.6 3.5 3.4) foreach(_PYTHON_V ${Python_ADDITIONAL_VERSIONS}) find_package(PythonInterp ${_PYTHON_V}) if ( PYTHONINTERP_FOUND ) break() endif() endforeach() # Must unset before searching for libs, otherwise these are checked before the required version... unset(Python_ADDITIONAL_VERSIONS) if ( PYTHONINTERP_FOUND AND PYTHON_VERSION_STRING VERSION_GREATER "3.4" ) # Find libraries that match the found interpreter (mismatched versions not supported). # This assumes libs are available for the newest Python version on the system. # KDevelop should _always_ be built against the newest possible version, so notabug. find_package(PythonLibs "${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}" REQUIRED EXACT) endif() if ( NOT PYTHONLIBS_FOUND OR PYTHONLIBS_VERSION_STRING VERSION_LESS "3.4.3" ) message(FATAL_ERROR "Python >= 3.4.3 but < 3.7 with --enable-shared is required to build kdev-python") endif() configure_file( "${kdevpython_SOURCE_DIR}/kdevpythonversion.h.cmake" "${kdevpython_BINARY_DIR}/kdevpythonversion.h" @ONLY ) set(QT_MIN_VERSION "5.5.0") find_package(Qt5 ${QT_MIN_VERSION} CONFIG REQUIRED Core Widgets Test) set(KF5_DEP_VERSION "5.15.0") find_package(KF5 ${KF5_DEP_VERSION} REQUIRED I18n NewStuff ItemModels ThreadWeaver TextEditor KCMUtils) find_package(KDevPlatform ${KDEVPLATFORM_VERSION} REQUIRED) find_package(KDevelop ${KDEVPLATFORM_VERSION} REQUIRED) enable_testing() if ( NOT WIN32 ) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wfatal-errors -Wall") endif ( NOT WIN32 ) # then, build the plugin include_directories( ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/duchain ${CMAKE_CURRENT_SOURCE_DIR}/parser ${CMAKE_CURRENT_BINARY_DIR}/parser ) add_subdirectory(app_templates) add_subdirectory(parser) add_subdirectory(duchain) add_subdirectory(codecompletion) add_subdirectory(debugger) add_subdirectory(docfilekcm) set(kdevpythonlanguagesupport_PART_SRCS codegen/correctionfilegenerator.cpp codegen/refactoring.cpp pythonlanguagesupport.cpp pythonparsejob.cpp pythonhighlighting.cpp pythonstylechecking.cpp # config pages: docfilekcm/docfilewizard.cpp docfilekcm/docfilemanagerwidget.cpp docfilekcm/kcm_docfiles.cpp pep8kcm/kcm_pep8.cpp projectconfig/projectconfigpage.cpp ) ecm_qt_declare_logging_category(kdevpythonlanguagesupport_PART_SRCS HEADER codegendebug.h IDENTIFIER KDEV_PYTHON_CODEGEN CATEGORY_NAME "kdevelop.languages.python.codegen" ) ecm_qt_declare_logging_category(kdevpythonlanguagesupport_PART_SRCS HEADER pythondebug.h IDENTIFIER KDEV_PYTHON CATEGORY_NAME "kdevelop.languages.python" ) ki18n_wrap_ui(kdevpythonlanguagesupport_PART_SRCS codegen/correctionwidget.ui projectconfig/projectconfig.ui pep8kcm/pep8.ui ) kdevplatform_add_plugin(kdevpythonlanguagesupport JSON kdevpythonsupport.json SOURCES ${kdevpythonlanguagesupport_PART_SRCS}) target_link_libraries(kdevpythonlanguagesupport KDev::Interfaces KDev::Language KDev::Util KF5::ThreadWeaver KF5::TextEditor KF5::NewStuff kdevpythoncompletion kdevpythonparser kdevpythonduchain ) get_target_property(DEFINESANDINCLUDES_INCLUDE_DIRS KDev::DefinesAndIncludesManager INTERFACE_INCLUDE_DIRECTORIES) include_directories(${DEFINESANDINCLUDES_INCLUDE_DIRS}) install(DIRECTORY documentation_files DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) install(DIRECTORY correction_files DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) install(FILES codestyle.py DESTINATION ${DATA_INSTALL_DIR}/kdevpythonsupport) # kdebugsettings file install(FILES kdevpythonsupport.categories DESTINATION ${KDE_INSTALL_CONFDIR}) feature_summary(WHAT ALL FATAL_ON_MISSING_REQUIRED_PACKAGES) diff --git a/README.packagers b/README.packagers index 9d6b1b43..aa204c42 100644 --- a/README.packagers +++ b/README.packagers @@ -1,38 +1,47 @@ This information is intended for package mantainers. First of all, thanks for packaging this software! If you encounter any issues which you think should be fixed upstream, please report a bug or send a patch, I'll be happy to accept it if it makes sense. Notes ===== This is a collection of notes about what might be of interest, related to packaging. Supported Python versions ------------------------- Python 2 support was dropped, and we just support the latest 2 to 3 minor releases of Python 3 right now. +NOTE: Since we use the upstream CPython parser, compiling against an old + version will restrict kdev-python's syntax support to that version. +This will cause false-positive warnings to the user and failed type-inference + for newer Python code. Then we get bug reports upstream for features that + were already implemented. + +Please compile against the newest supported version unless there is no other + possible alternative. + Python fork ----------- This branch, which contains the python3 version of kdev-python, does no longer contain a fork of python. Instead, it links against your system's python. Python is both a build- and a runtime dependency of this program. documentation_data directory ---------------------------- This directory does NOT contain user documentation for the plugin. It MUST be packaged together with the program, as it contains runtime data which is necessary for the program to work correctly (such as representations of python's built-in data types, which are then read by the parser etc.). Licensing notes --------------- The following files are not copyrighted: Everything in duchain/tests/data/ Everything in documentation_files/ Everything in correction_files/ Everything in app_templates/ example_ast.py diff --git a/duchain/contextbuilder.cpp b/duchain/contextbuilder.cpp index 8941107a..a02cfb0c 100644 --- a/duchain/contextbuilder.cpp +++ b/duchain/contextbuilder.cpp @@ -1,483 +1,489 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "pythonlanguagesupport.h" #include "pythoneditorintegrator.h" #include "dumpchain.h" #include "usebuilder.h" #include "contextbuilder.h" #include "pythonducontext.h" #include "pythonparsejob.h" #include "declarationbuilder.h" #include "helpers.h" #include "duchaindebug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace KDevelop; using namespace KTextEditor; namespace Python { ReferencedTopDUContext ContextBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { if (!updateContext) { DUChainReadLocker lock(DUChain::lock()); updateContext = DUChain::self()->chainForDocument(url); if ( updateContext ) { Q_ASSERT(updateContext->type() == DUContext::Global); } } if (updateContext) { qCDebug(KDEV_PYTHON_DUCHAIN) << " ====> DUCHAIN ====> rebuilding duchain for" << url.str() << "(was built before)"; DUChainWriteLocker lock(DUChain::lock()); Q_ASSERT(updateContext->type() == DUContext::Global); updateContext->clearImportedParentContexts(); updateContext->parsingEnvironmentFile()->clearModificationRevisions(); updateContext->clearProblems(); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << " ====> DUCHAIN ====> building duchain for" << url.str(); } return ContextBuilderBase::build(url, node, updateContext); } PythonEditorIntegrator* ContextBuilder::editor() const { return m_editor; } IndexedString ContextBuilder::currentlyParsedDocument() const { return m_currentlyParsedDocument; } void ContextBuilder::setCurrentlyParsedDocument(const IndexedString &document) { m_currentlyParsedDocument = document; } void ContextBuilder::setFutureModificationRevision(const ModificationRevision &rev) { m_futureModificationRevision = rev; } RangeInRevision ContextBuilder::rangeForNode(Ast* node, bool moveRight) { return RangeInRevision(node->startLine, node->startCol, node->endLine, node->endCol + (int) moveRight); } RangeInRevision ContextBuilder::rangeForNode(Identifier* node, bool moveRight) { return rangeForNode(static_cast(node), moveRight); } TopDUContext* ContextBuilder::newTopContext(const RangeInRevision& range, ParsingEnvironmentFile* file) { IndexedString currentDocumentUrl = currentlyParsedDocument(); if ( !file ) { file = new ParsingEnvironmentFile(currentDocumentUrl); file->setLanguage(IndexedString("python")); } TopDUContext* top = new PythonTopDUContext(currentDocumentUrl, range, file); ReferencedTopDUContext ref(top); m_topContext = ref; return top; } DUContext* ContextBuilder::newContext(const RangeInRevision& range) { return new PythonNormalDUContext(range, currentContext()); } void ContextBuilder::addUnresolvedImport(const IndexedString &module) { m_unresolvedImports.append(module); } QList ContextBuilder::unresolvedImports() const { return m_unresolvedImports; } void ContextBuilder::setEditor(PythonEditorIntegrator* editor) { m_editor = editor; } void ContextBuilder::startVisiting(Ast* node) { visitNode(node); } void ContextBuilder::setContextOnNode(Ast* node, DUContext* context) { node->context = context; } DUContext* ContextBuilder::contextFromNode(Ast* node) { return node->context; } RangeInRevision ContextBuilder::editorFindRange(Ast* fromNode, Ast* toNode) { return editor()->findRange(fromNode, toNode); } CursorInRevision ContextBuilder::editorFindPositionSafe(Ast* node) { if ( !node ) { return CursorInRevision::invalid(); } return editor()->findPosition(node); } CursorInRevision ContextBuilder::startPos( Ast* node ) { return m_editor->findPosition(node, PythonEditorIntegrator::FrontEdge); } QualifiedIdentifier ContextBuilder::identifierForNode( Python::Identifier* node ) { return QualifiedIdentifier(node->value); } void ContextBuilder::addImportedContexts() { if ( compilingContexts() && !m_importedParentContexts.isEmpty() ) { DUChainWriteLocker lock( DUChain::lock() ); foreach( DUContext* imported, m_importedParentContexts ) currentContext()->addImportedParentContext( imported ); m_importedParentContexts.clear(); } } void ContextBuilder::closeAlreadyOpenedContext(DUContextPointer context) { Q_ASSERT(currentContext() == context.data()); while ( ! m_temporarilyClosedContexts.isEmpty() ) { openContext(m_temporarilyClosedContexts.last().data()); m_temporarilyClosedContexts.removeLast(); } } void ContextBuilder::activateAlreadyOpenedContext(DUContextPointer context) { Q_ASSERT(m_temporarilyClosedContexts.isEmpty()); Q_ASSERT(contextAlreadyOpen(context)); DUContext* current = currentContext(); bool reallyCompilingContexts = compilingContexts(); setCompilingContexts(false); // TODO this is very hackish. while ( current ) { if ( current == context.data() ) { setCompilingContexts(reallyCompilingContexts); return; } m_temporarilyClosedContexts.append(DUContextPointer(current)); closeContext(); current = currentContext(); } setCompilingContexts(reallyCompilingContexts); } bool ContextBuilder::contextAlreadyOpen(DUContextPointer context) { DUContext* current = currentContext(); while ( current ) { if ( context.data() == current ) return true; current = current->parentContext(); } return false; } void ContextBuilder::visitListComprehension(ListComprehensionAst* node) { visitComprehensionCommon(node); } void ContextBuilder::visitDictionaryComprehension(DictionaryComprehensionAst* node) { visitComprehensionCommon(node); } void ContextBuilder::visitGeneratorExpression(GeneratorExpressionAst* node) { visitComprehensionCommon(node); } RangeInRevision ContextBuilder::comprehensionRange(Ast* node) { RangeInRevision range = editorFindRange(node, node); return range; } void ContextBuilder::visitComprehensionCommon(Ast* node) { RangeInRevision range = comprehensionRange(node); Q_ASSERT(range.isValid()); if ( range.isValid() ) { DUChainWriteLocker lock; openContext(node, range, KDevelop::DUContext::Other); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating comprehension context" << node << range; Q_ASSERT(currentContext()); // currentContext()->setLocalScopeIdentifier(QualifiedIdentifier("")); lock.unlock(); if ( node->astType == Ast::DictionaryComprehensionAstType ) Python::AstDefaultVisitor::visitDictionaryComprehension(static_cast(node)); if ( node->astType == Ast::ListComprehensionAstType ) Python::AstDefaultVisitor::visitListComprehension(static_cast(node)); if ( node->astType == Ast::GeneratorExpressionAstType ) Python::AstDefaultVisitor::visitGeneratorExpression(static_cast(node)); if ( node->astType == Ast::SetComprehensionAstType ) Python::AstDefaultVisitor::visitSetComprehension(static_cast(node)); lock.lock(); closeContext(); } } void ContextBuilder::openContextForClassDefinition(ClassDefinitionAst* node) { // make sure the contexts ends at the next DEDENT token, not at the last statement. // also, make the context begin *after* the parent list and class name. int endLine = editor()->indent()->nextChange(node->endLine, FileIndentInformation::Dedent); CursorInRevision start = CursorInRevision(node->body.first()->startLine, node->body.first()->startCol); if ( start.line > node->startLine ) { start = CursorInRevision(node->startLine + 1, 0); } RangeInRevision range(start, CursorInRevision(endLine, 0)); DUChainWriteLocker lock; openContext(node, range, DUContext::Class, node->name); currentContext()->setLocalScopeIdentifier(identifierForNode(node->name)); lock.unlock(); addImportedContexts(); } void ContextBuilder::visitClassDefinition( ClassDefinitionAst* node ) { openContextForClassDefinition(node); Python::AstDefaultVisitor::visitClassDefinition(node); closeContext(); } void ContextBuilder::visitCode(CodeAst* node) { - auto doc_url = Helper::getDocumentationFile(); - IndexedString doc = IndexedString(doc_url); + IndexedString doc = Helper::getDocumentationFile(); Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); if ( currentlyParsedDocument() != doc ) { // Search for the python built-in functions file, and dump its contents into the current file. auto internal = Helper::getDocumentationFileContext(); if ( ! internal ) { // If the built-in functions file is not yet parsed, schedule it with a high priority. m_unresolvedImports.append(doc); KDevelop::ICore::self()->languageController()->backgroundParser() ->addDocument(doc, KDevelop::TopDUContext::ForceUpdate, BackgroundParser::BestPriority*2, nullptr, ParseJob::FullSequentialProcessing); // This must NOT be called from parse threads! It's only meant to be used from the foreground thread, and will // cause thread starvation if called from here. // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); } else { DUChainWriteLocker wlock; currentContext()->addImportedParentContext(internal); } } AstDefaultVisitor::visitCode(node); } QPair ContextBuilder::findModulePath(const QString& name, const QUrl& currentDocument) { QStringList nameComponents = name.split("."); QVector searchPaths; if ( name.startsWith('.') ) { /* To take care for imports like "from ....xxxx.yyy import zzz" * we need to take current doc path and run "cd .." enough times */ nameComponents.removeFirst(); QString tname = name.mid(1); // remove first dot QDir curPathDir = QDir(currentDocument.adjusted(QUrl::RemoveFilename).toLocalFile()); foreach(QString c, tname) { if (c != ".") break; curPathDir.cdUp(); nameComponents.removeFirst(); } searchPaths << QUrl::fromLocalFile(curPathDir.path()); } else { // If this is not a relative import, use the project directory, // the current directory, and all system include paths. // FIXME: If absolute imports enabled, don't add curently parsed doc path searchPaths = Helper::getSearchPaths(currentDocument); } // Loop over all the name components, and find matching folders or files. QDir tmp; QStringList leftNameComponents; foreach ( const QUrl& currentPath, searchPaths ) { tmp.setPath(currentPath.toLocalFile()); leftNameComponents = nameComponents; foreach ( QString component, nameComponents ) { if ( component == "*" ) { // For "from ... import *", if "..." is a directory, use the "__init__.py" file component = QStringLiteral("__init__"); } else { // only empty the list if not importing *, this is convenient later on leftNameComponents.removeFirst(); } QString testFilename = tmp.path() + "/" + component; bool can_continue = tmp.cd(component); QFileInfo sourcedir(testFilename); const bool dir_exists = sourcedir.exists() && sourcedir.isDir(); // we can only parse those, so we don't care about anything else for now. // Any C modules (.so, .dll) will be ignored, and highlighted as "not found". TODO fix this static QStringList valid_extensions{".py", ".pyx"}; foreach ( const auto& extension, valid_extensions ) { QFile sourcefile(testFilename + extension); if ( ! dir_exists || leftNameComponents.isEmpty() ) { // If the search cannot continue further down into a hierarchy of directories, // the file matching the next name component will be returned, // toegether with a list of names which must be resolved inside that file. if ( sourcefile.exists() ) { auto sourceUrl = QUrl::fromLocalFile(testFilename + extension); // TODO QUrl: cleanPath? return qMakePair(sourceUrl, leftNameComponents); } else if ( dir_exists ) { auto path = QUrl::fromLocalFile(testFilename + "/__init__.py"); // TODO QUrl: cleanPath? return qMakePair(path, leftNameComponents); } } } if ( ! can_continue ) { // if not returned yet and the cd into the next component failed, // abort and try the next search path. break; } } } return {}; } +void ContextBuilder::visitLambda(LambdaAst* node) +{ + openContext(node, editorFindRange(node, node->body), DUContext::Other); + AstDefaultVisitor::visitLambda(node); + closeContext(); +} + RangeInRevision ContextBuilder::rangeForArgumentsContext(FunctionDefinitionAst* node) { auto start = node->name->range().end(); auto end = start; auto args = node->arguments; if ( args->kwarg ) { end = args->kwarg->range().end(); } else if ( args->vararg && ( args->arguments.isEmpty() || ! args->vararg->appearsBefore(args->arguments.last())) ) { end = args->vararg->range().end(); } else if ( ! args->arguments.isEmpty() ) { end = args->arguments.last()->range().end(); } if ( ! args->defaultValues.isEmpty() ) { end = qMax(args->defaultValues.last()->range().end(), end); } RangeInRevision range(start.line(), start.column(), end.line(), end.column()); range.start.column += 1; // Don't include end of name. range.end.column += 1; // Include end parenthesis (unless spaces...) return range; } void ContextBuilder::visitFunctionArguments(FunctionDefinitionAst* node) { RangeInRevision range = rangeForArgumentsContext(node); // The DUChain expects the context containing a function's arguments to be of type Function. // The function body will have DUContext::Other as type, as it contains only code. DUContext* funcctx = openContext(node->arguments, range, DUContext::Function, node->name); AstDefaultVisitor::visitArguments(node->arguments); visitArguments(node->arguments); closeContext(); // the parameters should be visible in the function body, so import that context there m_importedParentContexts.append(funcctx); } void ContextBuilder::visitFunctionDefinition(FunctionDefinitionAst* node) { visitNodeList(node->decorators); visitNode(node->returns); visitFunctionArguments(node); visitFunctionBody(node); } void ContextBuilder::visitFunctionBody(FunctionDefinitionAst* node) { // The function should end at the next DEDENT token, not at the body's last statement int endLine = node->endLine; if ( ! node->body.isEmpty() ) { endLine = node->body.last()->startLine; } if ( node->endLine != node->startLine ) { endLine = editor()->indent()->nextChange(endLine, FileIndentInformation::Dedent); if ( ! node->body.isEmpty() ) { endLine = qMax(endLine, node->body.last()->endLine + 1); } } CursorInRevision end = CursorInRevision(endLine, node->startLine == node->endLine ? INT_MAX : 0); CursorInRevision start = rangeForArgumentsContext(node).end; if ( start.line < node->body.first()->startLine ) { start = CursorInRevision(node->startLine + 1, 0); } RangeInRevision range(start, end); // Open the context for the function body (the list of statements) // It's of type Other, as it contains only code openContext(node, range, DUContext::Other, identifierForNode(node->name)); { DUChainWriteLocker lock; currentContext()->setLocalScopeIdentifier(identifierForNode(node->name)); } // import the parameters into the function body addImportedContexts(); visitNodeList(node->body); closeContext(); } } diff --git a/duchain/contextbuilder.h b/duchain/contextbuilder.h index b0542e44..466246d2 100644 --- a/duchain/contextbuilder.h +++ b/duchain/contextbuilder.h @@ -1,208 +1,209 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright (c) 2010-2014 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef PYTHON_CONTEXTBUILDER_H #define PYTHON_CONTEXTBUILDER_H #include "astdefaultvisitor.h" #include #include #include #include "pythonduchainexport.h" using namespace KDevelop; namespace Python { class PythonEditorIntegrator; class FileIndentInformation; typedef KDevelop::AbstractContextBuilder ContextBuilderBase; /** * @brief The context builder, which calculates the scopes in a file. * * For practical reasons, some building of scopes also happens * in the declaration builder. */ class KDEVPYTHONDUCHAIN_EXPORT ContextBuilder: public ContextBuilderBase, public Python::AstDefaultVisitor { public: ContextBuilder() = default; /** * @brief Entry function called by KDevPlatform API. */ ReferencedTopDUContext build(const KDevelop::IndexedString& url, Ast* node, ReferencedTopDUContext updateContext = ReferencedTopDUContext()) override; /** * @brief Set the editor integrator. */ void setEditor(PythonEditorIntegrator* editor); /** * @brief Set the modification revision which will be created by this builder. */ void setFutureModificationRevision(const ModificationRevision& rev); /** * @brief Get the editor integrator. */ PythonEditorIntegrator* editor() const; /** * @brief Find the URL which would be imported by the dotted name @p name. * * @param name a dotted name, such as PyQt4.QtCore.QWidget * @param currentDocument the current document, for resolving relative imports * @return QPair< QUrl, QStringList > the URL if found, and a list of components from * the end of the name which were not yet consumed */ static QPair findModulePath(const QString& name, const QUrl& currentDocument); /** * @brief Get the range which encompasses the given @p node. * @param moveRight true to make the range longer by one character */ static RangeInRevision rangeForNode(Ast* node, bool moveRight); /** * @brief Get the range of @p identifier. * @param moveRight true to make the range longer by one character */ static RangeInRevision rangeForNode(Identifier* identifier, bool moveRight); /** * @brief Find the range of a comprehension. * @param node Comprehension to find the range of, e.g. a ListComprehensionAst. */ RangeInRevision comprehensionRange(Ast* node); /** * @brief Calculate the range of the arguments context of the given @p node. * @return Range the argument list of this function encompasses. */ RangeInRevision rangeForArgumentsContext(Python::FunctionDefinitionAst* node); /** * @brief Add @p module to the list of unresolved imports in this builder. */ void addUnresolvedImport(const IndexedString& module); /** * @brief Retrieve a list of imports not resolved by this builder pass. */ QList unresolvedImports() const; public: // ugly because this collides with currentDocument(), but we have to use it; // for some reason the UseBuilder does not have m_url set, and it's private (not even protected) to AbstractContextBuilder. // so at least keep this consistent within the plugin and use this everywhere. // maybe we can remove this hack later. TODO maybe change something in kdevplatform, or maybe we're doing something wrong here? IndexedString currentlyParsedDocument() const; void setCurrentlyParsedDocument(const IndexedString& document); protected: /** * @brief Create a new top context and set it as this builder's active context. * * @param range Range to encompass * @return KDevelop::TopDUContext* weak pointer to the created top context. */ TopDUContext* newTopContext(const RangeInRevision& range, ParsingEnvironmentFile* file) override; /** * @brief Create a new context. * Overridden to create instances of Python's specialized DUContext. */ KDevelop::DUContext* newContext(const KDevelop::RangeInRevision& range) override; protected: // AST visitor functions + void visitLambda(Python::LambdaAst * node) override; void visitFunctionDefinition( FunctionDefinitionAst* ) override; void visitClassDefinition( ClassDefinitionAst* ) override; void visitCode(CodeAst* node) override; void visitListComprehension(ListComprehensionAst* node) override; void visitDictionaryComprehension(DictionaryComprehensionAst* node) override; void visitGeneratorExpression(GeneratorExpressionAst* node) override; void visitComprehensionCommon(Ast* node); void startVisiting(Ast* node) override; KDevelop::RangeInRevision editorFindRange(Ast* fromNode, Ast* toNode) override; virtual KDevelop::CursorInRevision editorFindPositionSafe(Ast* node); virtual KDevelop::CursorInRevision startPos(Ast* node); KDevelop::QualifiedIdentifier identifierForNode(Identifier* node) override; /** * @brief Set @p context as the context of @p node. * The context is stored inside the AST itself. */ void setContextOnNode(Ast* node, KDevelop::DUContext* context) override; /** * @brief Get the context set on @p node as previously set by @ref setContextOnNode. */ KDevelop::DUContext* contextFromNode(Ast* node) override; /** * @brief Add the saved list of contexts to import to the current context, and clear it. */ void addImportedContexts(); // helpers which need to be called separately from DeclarationBuilder virtual void visitFunctionArguments(FunctionDefinitionAst* node); virtual void visitFunctionBody(FunctionDefinitionAst* node); void openContextForClassDefinition(ClassDefinitionAst* node); protected: // those functions can be used if you want to do something to a context you are in, // but which is not the current one. Example: You want to add a variable to a class context, // but the current context is inside that class context (method declaration, ...) bool contextAlreadyOpen(DUContextPointer context); void activateAlreadyOpenedContext(DUContextPointer context); void closeAlreadyOpenedContext(DUContextPointer context); QList m_temporarilyClosedContexts; protected: // true if the first of the two performed passes is currently active bool m_prebuilding = false; // List of imports which were encountered, but could not be resolved QList m_unresolvedImports; // The ModificationRevision this context will be valid for ModificationRevision m_futureModificationRevision; IndexedString m_currentlyParsedDocument; private: // The top-context being built. ReferencedTopDUContext m_topContext; PythonEditorIntegrator* m_editor = nullptr; QList m_importedParentContexts; QSharedPointer m_indentInformationCache; }; } #endif diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index 1e91c3d6..3447daf3 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1875 +1,1876 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2016 Sven Brauch * * Copyright 2016 Francis Herne * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "declarationbuilder.h" #include "duchain/declarations/functiondeclaration.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/nonetype.h" #include "types/indexedcontainer.h" #include "contextbuilder.h" #include "expressionvisitor.h" #include "pythoneditorintegrator.h" #include "helpers.h" #include "assistants/missingincludeassistant.h" #include "correctionhelper.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchaindebug.h" #include using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder(Python::PythonEditorIntegrator* editor, int ownPriority) : DeclarationBuilderBase() , m_ownPriority(ownPriority) { setEditor(editor); } DeclarationBuilder:: ~DeclarationBuilder() { if ( ! m_scheduledForDeletion.isEmpty() ) { DUChainWriteLocker lock; foreach ( DUChainBase* d, m_scheduledForDeletion ) { delete d; } m_scheduledForDeletion.clear(); } } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { m_correctionHelper.reset(new CorrectionHelper(url, this)); // The declaration builder needs to run twice, so it can resolve uses of e.g. functions // which are called before they are defined (which is easily possible, due to python's dynamic nature). if ( ! m_prebuilding ) { DeclarationBuilder* prebuilder = new DeclarationBuilder(editor(), m_ownPriority); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); delete prebuilder; qCDebug(KDEV_PYTHON_DUCHAIN) << "Second declarationbuilder pass"; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Prebuilding declarations"; } return DeclarationBuilderBase::build(url, node, updateContext); } int DeclarationBuilder::jobPriority() const { return m_ownPriority; } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock(DUChain::lock()); currentDeclaration()->setKind(Declaration::Type); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::eventuallyReopenDeclaration(Identifier* name, FitDeclarationType mustFitType) { QList existingDeclarations = existingDeclarationsForNode(name); Declaration* dec = nullptr; reopenFittingDeclaration(existingDeclarations, mustFitType, editorFindRange(name, name), &dec); bool declarationOpened = (bool) dec; if ( ! declarationOpened ) { dec = openDeclaration(name); } Q_ASSERT(dynamic_cast(dec)); return static_cast(dec); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { if ( node->astType == Ast::NameAstType ) { NameAst* currentVariableDefinition = static_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" // This is just a sanity check, the code should never request creation of a variable // in such cases. if ( currentVariableDefinition->context != ExpressionAst::Context::Store ) { return nullptr; } return visitVariableDeclaration(currentVariableDefinition->identifier, previous, type, flags); } else if ( node->astType == Ast::IdentifierAstType ) { return visitVariableDeclaration(static_cast(node), previous, type, flags); } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "cannot create variable declaration for non-(name|identifier) AST, this is a programming error"; return static_cast(nullptr); } } QList< Declaration* > DeclarationBuilder::existingDeclarationsForNode(Identifier* node) { return currentContext()->findDeclarations( identifierForNode(node).last(), CursorInRevision::invalid(), nullptr, (DUContext::SearchFlag) (DUContext::DontSearchInParent | DUContext::DontResolveAliases) ); } DeclarationBuilder::FitDeclarationType DeclarationBuilder::kindForType(AbstractType::Ptr type, bool isAlias) { if ( type ) { if ( type->whichType() == AbstractType::TypeFunction ) { return FunctionDeclarationType; } } if ( isAlias ) { return AliasDeclarationType; } return InstanceDeclarationType; } template QList DeclarationBuilder::reopenFittingDeclaration( QList declarations, FitDeclarationType mustFitType, RangeInRevision updateRangeTo, Declaration** ok ) { // Search for a declaration from a previous parse pass which should be re-used QList remainingDeclarations; *ok = nullptr; foreach ( Declaration* d, declarations ) { Declaration* fitting = dynamic_cast(d); if ( ! fitting ) { // Only use a declaration if the type matches qCDebug(KDEV_PYTHON_DUCHAIN) << "skipping" << d->toString() << "which could not be cast to the requested type"; continue; } // Do not use declarations which have been encountered previously; // this function only handles declarations from previous parser passes which have not // been encountered yet in this pass bool reallyEncountered = wasEncountered(d) && ! m_scheduledForDeletion.contains(d); bool invalidType = false; if ( d->abstractType() && mustFitType != NoTypeRequired ) { invalidType = ( ( d->isFunctionDeclaration() ) != ( mustFitType == FunctionDeclarationType ) ); if ( ! invalidType ) { invalidType = ( ( dynamic_cast(d) != nullptr ) != ( mustFitType == AliasDeclarationType ) ); } } if ( fitting && ! reallyEncountered && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { openDeclarationInternal(d); d->setRange(updateRangeTo); *ok = d; setEncountered(d); break; } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Not opening previously existing declaration because it's in another top context"; } } else if ( ! invalidType ) { remainingDeclarations << d; } } return remainingDeclarations; } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Declaration* previous, AbstractType::Ptr type, VisitVariableFlags flags) { DUChainWriteLocker lock; RangeInRevision range = editorFindRange(node, node); // ask the correction file library if there's a user-specified type for this object if ( AbstractType::Ptr hint = m_correctionHelper->hintForLocal(node->value) ) { type = hint; } // If no type is known, display "mixed". if ( ! type ) { type = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); } QList existingDeclarations; if ( previous ) { existingDeclarations << previous; } else { // declarations declared at an earlier range in this top-context existingDeclarations = existingDeclarationsForNode(node); } // declaration existing in a previous version of this top-context Declaration* dec = nullptr; existingDeclarations = reopenFittingDeclaration(existingDeclarations, kindForType(type), range, &dec); bool declarationOpened = (bool) dec; if ( flags & AbortIfReopenMismatch && previous && ! declarationOpened ) { return nullptr; } // tells whether the declaration found for updating is in the same top context bool inSameTopContext = true; // tells whether there's fitting declarations to update (update is not the same as re-open! one is for // code which uses the same variable twice, the other is for multiple passes of the parser) bool haveFittingDeclaration = false; if ( ! existingDeclarations.isEmpty() && existingDeclarations.last() ) { Declaration* d = Helper::resolveAliasDeclaration(existingDeclarations.last()); DUChainReadLocker lock; if ( d && d->topContext() != topContext() ) { inSameTopContext = false; } if ( dynamic_cast(existingDeclarations.last()) ) { haveFittingDeclaration = true; } } if ( currentContext() && currentContext()->type() == DUContext::Class && ! haveFittingDeclaration ) { // If the current context is a class, then this is a class member variable. if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(AbstractType::Ptr(type)); dec->setKind(KDevelop::Declaration::Instance); } else if ( ! haveFittingDeclaration ) { // This name did not previously appear in the user code, so a new variable is declared // check whether a declaration from a previous parser pass must be updated if ( ! dec ) { dec = openDeclaration(node); Q_ASSERT(! declarationOpened); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } AbstractType::Ptr newType; if ( currentContext()->type() == DUContext::Function ) { // check for argument type hints (those are created when calling functions) AbstractType::Ptr hints = Helper::extractTypeHints(dec->abstractType()); if ( hints.cast() || hints.cast() ) { // This only happens when the type hint is a tuple, which means the vararg/kwarg of a function is being processed. newType = hints; } else { newType = Helper::mergeTypes(hints, type); } } else { newType = type; } dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); } else if ( inSameTopContext ) { // The name appeared previously in the user code, so no new variable is declared, but just // the type is modified accordingly. dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = type; if ( newType ) { if ( currentType && currentType->indexed() != newType->indexed() ) { // If the previous and new type are different, use an unsure type dec->setType(Helper::mergeTypes(currentType, newType)); } else { // If no type was set previously, use only the new one. dec->setType(AbstractType::Ptr(type)); } } } T* result = dynamic_cast(dec); if ( ! result ) qCWarning(KDEV_PYTHON_DUCHAIN) << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_unresolvedImports.clear(); DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( node->name ) { // Python allows to assign the caught exception to a variable; create that variable if required. ExpressionVisitor v(currentContext()); v.visitNode(node->type); visitVariableDeclaration(node->name, nullptr, v.lastType()); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWithItem(WithItemAst* node) { if ( node->optionalVars ) { // For statements like "with open(f) as x", a new variable must be created; do this here. ExpressionVisitor v(currentContext()); v.visitNode(node->contextExpression); visitVariableDeclaration(node->optionalVars, nullptr, v.lastType()); } Python::AstDefaultVisitor::visitWithItem(node); } void DeclarationBuilder::visitFor(ForAst* node) { if ( node->iterator ) { ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; // TODO make this a bit faster, it wastes time Declaration* lastAccessedDeclaration = nullptr; int i = 0; int identifierCount = dottedNameIdentifier.length(); foreach ( const QString& currentIdentifier, dottedNameIdentifier ) { Q_ASSERT(currentContext); i++; QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), nullptr, DUContext::NoFiltering); // break if the list of identifiers is not yet totally worked through and no // declaration with an internal context was found if ( declarations.isEmpty() || ( !declarations.last()->internalContext() && identifierCount != i ) ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return nullptr; } else { lastAccessedDeclaration = declarations.last(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } QString DeclarationBuilder::buildModuleNameFromNode(ImportFromAst* node, AliasAst* alias, const QString& intermediate) const { QString moduleName = alias->name->value; if ( ! intermediate.isEmpty() ) { moduleName.prepend('.').prepend(intermediate); } if ( node->module ) { moduleName.prepend('.').prepend(node->module->value); } // To handle relative imports correctly, add node level in the beginning of the path // This will allow findModulePath to deduce module search direcotry properly moduleName.prepend(QString(node->level, '.')); return moduleName; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; QString declarationName; foreach ( AliasAst* name, node->names ) { // iterate over all the names that are imported, like "from foo import bar as baz, bang as asdf" Identifier* declarationIdentifier = nullptr; declarationName.clear(); if ( name->asName ) { // use either the alias ("as foo"), or the object name itself if no "as" is given declarationIdentifier = name->asName; declarationName = name->asName->value; } else { declarationIdentifier = name->name; declarationName = name->name->value; } // This is a bit hackish, it tries to find the specified object twice twice -- once it tries to // import the name from a module's __init__.py file, and once from a "real" python file // TODO improve this code-wise ProblemPointer problem(nullptr); QString intermediate; moduleName = buildModuleNameFromNode(node, name, intermediate); Declaration* success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem); if ( ! success && (node->module || node->level) ) { ProblemPointer problem_init(nullptr); intermediate = QString("__init__"); moduleName = buildModuleNameFromNode(node, name, intermediate); success = createModuleImportDeclaration(moduleName, declarationName, declarationIdentifier, problem_init); } if ( ! success && problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::visitComprehension(ComprehensionAst* node) { Python::AstDefaultVisitor::visitComprehension(node); ExpressionVisitor v(currentContext()); v.visitNode(node->iterator); assignToUnknown(node->target, Helper::contentOfIterable(v.lastType(), topContext())); } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock; foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; ProblemPointer problem(nullptr); createModuleImportDeclaration(moduleName, declarationIdentifier->value, declarationIdentifier, problem); if ( problem ) { DUChainWriteLocker lock; topContext()->addProblem(problem); } } } void DeclarationBuilder::scheduleForDeletion(DUChainBase* d, bool doschedule) { if ( doschedule ) { m_scheduledForDeletion.append(d); } else { m_scheduledForDeletion.removeAll(d); } } Declaration* DeclarationBuilder::createDeclarationTree(const QStringList& nameComponents, Identifier* declarationIdentifier, const ReferencedTopDUContext& innerCtx, Declaration* aliasDeclaration, const RangeInRevision& range) { // This actually handles two use cases which are very similar -- thus this check: // There might be either one declaration which should be imported from another module, // or there might be a whole context. In "import foo.bar", the "bar" might be either // a single class/function/whatever, or a whole file to import. // NOTE: The former case can't actually happen in python, it's not allowed. However, // it is still handled here, because it's very useful for documentation files (pyQt for example // makes heavy use of that feature). Q_ASSERT( ( innerCtx.data() || aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); Q_ASSERT( ( !innerCtx.data() || !aliasDeclaration ) && "exactly one of innerCtx or aliasDeclaration must be provided"); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating declaration tree for" << nameComponents; Declaration* lastDeclaration = nullptr; int depth = 0; // check for already existing trees to update for ( int i = nameComponents.length() - 1; i >= 0; i-- ) { QStringList currentName; for ( int j = 0; j < i; j++ ) { currentName.append(nameComponents.at(j)); } lastDeclaration = findDeclarationInContext(currentName, topContext()); if ( lastDeclaration && (!range.isValid() || lastDeclaration->range() < range) ) { depth = i; break; } } DUContext* extendingPreviousImportCtx = nullptr; QStringList remainingNameComponents; bool injectingContext = false; if ( lastDeclaration && lastDeclaration->internalContext() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Found existing import statement while creating declaration for " << declarationIdentifier->value; for ( int i = depth; i < nameComponents.length(); i++ ) { remainingNameComponents.append(nameComponents.at(i)); } extendingPreviousImportCtx = lastDeclaration->internalContext(); injectContext(extendingPreviousImportCtx); injectingContext = true; qCDebug(KDEV_PYTHON_DUCHAIN) << "remaining identifiers:" << remainingNameComponents; } else { remainingNameComponents = nameComponents; extendingPreviousImportCtx = topContext(); } // now, proceed in creating the declaration tree with whatever context QList openedDeclarations; QList openedTypes; QList openedContexts; RangeInRevision displayRange = RangeInRevision::invalid(); DUChainWriteLocker lock; for ( int i = 0; i < remainingNameComponents.length(); i++ ) { // Iterate over all the names, and create a declaration + sub-context for each of them const QString& component = remainingNameComponents.at(i); Identifier temporaryIdentifier(component); Declaration* d = nullptr; temporaryIdentifier.copyRange(declarationIdentifier); temporaryIdentifier.endCol = temporaryIdentifier.startCol; temporaryIdentifier.startCol += 1; displayRange = editorFindRange(&temporaryIdentifier, &temporaryIdentifier); // TODO fixme bool done = false; if ( aliasDeclaration && i == remainingNameComponents.length() - 1 ) { // it's the last level, so if we have an alias declaration create it and stop if ( aliasDeclaration->isFunctionDeclaration() || dynamic_cast(aliasDeclaration) || dynamic_cast(aliasDeclaration) ) { aliasDeclaration = Helper::resolveAliasDeclaration(aliasDeclaration); AliasDeclaration* adecl = eventuallyReopenDeclaration(&temporaryIdentifier, AliasDeclarationType); if ( adecl ) { adecl->setAliasedDeclaration(aliasDeclaration); } d = adecl; closeDeclaration(); } else { d = visitVariableDeclaration(&temporaryIdentifier); d->setAbstractType(aliasDeclaration->abstractType()); } openedDeclarations.append(d); done = true; } if ( ! done ) { // create the next level of the tree hierarchy if not done yet. d = visitVariableDeclaration(&temporaryIdentifier); } if ( d ) { if ( topContext() != currentContext() ) { d->setRange(RangeInRevision(currentContext()->range().start, currentContext()->range().start)); } else { d->setRange(displayRange); } d->setAutoDeclaration(true); currentContext()->createUse(d->ownIndex(), d->range()); qCDebug(KDEV_PYTHON_DUCHAIN) << "really encountered:" << d << "; scheduled:" << m_scheduledForDeletion; qCDebug(KDEV_PYTHON_DUCHAIN) << d->toString(); scheduleForDeletion(d, false); qCDebug(KDEV_PYTHON_DUCHAIN) << "scheduled:" << m_scheduledForDeletion; } if ( done ) break; qCDebug(KDEV_PYTHON_DUCHAIN) << "creating context for " << component; // otherwise, create a new "level" entry (a pseudo type + context + declaration which contains all imported items) StructureType::Ptr moduleType = StructureType::Ptr(new StructureType()); openType(moduleType); // the identifier is needed so the context does not get re-opened if // more contexts are opened for other files with the same range Python::Identifier contextIdentifier(component); auto moduleContext = openContext(declarationIdentifier, KDevelop::DUContext::Other, &contextIdentifier); openedContexts.append(moduleContext); foreach ( Declaration* local, currentContext()->localDeclarations() ) { // keep all the declarations until the builder finished // kdevelop would otherwise delete them as soon as the context is closed if ( ! wasEncountered(local) ) { setEncountered(local); scheduleForDeletion(local, true); } } openedDeclarations.append(d); openedTypes.append(moduleType); if ( i == remainingNameComponents.length() - 1 ) { if ( innerCtx ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "adding imported context to inner declaration"; currentContext()->addImportedParentContext(innerCtx); } else if ( aliasDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "setting alias declaration on inner declaration"; } } } for ( int i = openedContexts.length() - 1; i >= 0; i-- ) { // Close all the declarations and contexts opened previosly, and assign the types. qCDebug(KDEV_PYTHON_DUCHAIN) << "closing context"; closeType(); closeContext(); auto d = openedDeclarations.at(i); // because no context will be opened for an alias declaration, this will not happen if there's one if ( d ) { openedTypes[i]->setDeclaration(d); d->setType(openedTypes.at(i)); d->setInternalContext(openedContexts.at(i)); } } if ( injectingContext ) { closeInjectedContext(); } if ( ! openedDeclarations.isEmpty() ) { // return the lowest-level element in the tree, for the caller to do stuff with return openedDeclarations.last(); } else return nullptr; } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString moduleName, QString declarationName, Identifier* declarationIdentifier, ProblemPointer& problemEncountered, Ast* rangeNode) { // Search the disk for a python file which contains the requested declaration auto moduleInfo = findModulePath(moduleName, currentlyParsedDocument().toUrl()); RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, false); } Q_ASSERT(range.isValid()); qCDebug(KDEV_PYTHON_DUCHAIN) << "Found module path [path/path in file]: " << moduleInfo; qCDebug(KDEV_PYTHON_DUCHAIN) << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock; const IndexedString modulePath = IndexedString(moduleInfo.first); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(modulePath); lock.unlock(); Declaration* resultingDeclaration = nullptr; if ( ! moduleInfo.first.isValid() ) { // The file was not found -- this is either an error in the user's code, // a missing module, or a C module (.so) which is unreadable for kdevelop // TODO imrpove error handling in case the module exists as a shared object or .pyc file only qCDebug(KDEV_PYTHON_DUCHAIN) << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Module \"%1\" not found", moduleName)); m_missingModules.append(IndexedString(moduleName)); problemEncountered = p; return nullptr; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done qCDebug(KDEV_PYTHON_DUCHAIN) << "No module context, recompiling"; m_unresolvedImports.append(modulePath); Helper::scheduleDependency(modulePath, m_ownPriority); // parseDocuments() must *not* be called from a background thread! // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); return nullptr; } if ( moduleInfo.second.isEmpty() ) { // import the whole module resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, moduleContext, nullptr, range); auto initFile = QStringLiteral("/__init__.py"); auto path = moduleInfo.first.path(); if ( path.endsWith(initFile) ) { // if the __init__ file is imported, import all the other files in that directory as well QDir dir(path.left(path.size() - initFile.size())); dir.setNameFilters({"*.py"}); dir.setFilter(QDir::Files); auto files = dir.entryList(); foreach ( const auto& file, files ) { if ( file == QStringLiteral("__init__.py") ) { continue; } const auto filePath = declarationName.split(".") << file.left(file.lastIndexOf(".py")); const auto fileUrl = QUrl::fromLocalFile(dir.path() + "/" + file); ReferencedTopDUContext fileContext; { DUChainReadLocker lock; fileContext = DUChain::self()->chainForDocument(IndexedString(fileUrl)); } if ( fileContext ) { Identifier id = *declarationIdentifier; id.value.append(".").append(filePath.last()); createDeclarationTree(filePath, &id, fileContext, nullptr); } else { m_unresolvedImports.append(IndexedString(fileUrl)); Helper::scheduleDependency(IndexedString(fileUrl), m_ownPriority); } } } } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { qCDebug(KDEV_PYTHON_DUCHAIN) << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = createDeclarationTree(declarationName.split("."), declarationIdentifier, ReferencedTopDUContext(nullptr), originalDeclaration, editorFindRange(declarationIdentifier, declarationIdentifier)); } else { KDevelop::Problem *p = new Python::MissingIncludeProblem(moduleName, currentlyParsedDocument()); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); problemEncountered = p; } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { // Functions containing "yield" statements will return lists in our abstraction. // The content type of that list can be guessed from the yield statements. AstDefaultVisitor::visitYield(node); // Determine the type of the argument to "yield", like "int" in "yield 3" ExpressionVisitor v(currentContext()); v.visitNode(node->value); AbstractType::Ptr encountered = v.lastType(); // In some obscure (or wrong) cases, "yield" might appear outside of a function body, // so check for that here. if ( ! node->value || ! hasCurrentType() ) { return; } TypePtr t = currentType(); if ( ! t ) { return; } if ( auto previous = t->returnType().cast() ) { // If the return type of the function already is set to a list, *add* the encountered type // to its possible content types. DUChainWriteLocker lock; previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { // Otherwise, create a new container type, and set it as the function's return type. DUChainWriteLocker lock; auto container = ExpressionVisitor::typeObjectForIntegralType("list"); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } void DeclarationBuilder::visitLambda(LambdaAst* node) { DUChainWriteLocker lock; // A context must be opened, because the lamdba's arguments are local to the lambda: // d = lambda x: x*2; print x # <- gives an error openContext(node, editorFindRange(node, node->body), DUContext::Other); foreach ( ArgAst* argument, node->arguments->arguments ) { visitVariableDeclaration(argument->argumentName); } + visitNodeList(node->arguments->defaultValues); if (node->arguments->vararg) { visitVariableDeclaration(node->arguments->vararg->argumentName); } if (node->arguments->kwarg) { visitVariableDeclaration(node->arguments->kwarg->argumentName); } visitNode(node->body); closeContext(); } void DeclarationBuilder::applyDocstringHints(CallAst* node, FunctionDeclaration::Ptr function) { ExpressionVisitor v(currentContext()); v.visitNode(static_cast(node->function)->value); // Don't do anything if the object the function is being called on is not a container. auto container = v.lastType().cast(); if ( ! container || ! function ) { return; } // Don't do updates to pre-defined functions. - if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { + if ( ! v.lastDeclaration() || v.lastDeclaration()->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Check for the different types of modifiers such a function can have QStringList args; QHash< QString, std::function > items; items["addsTypeOfArg"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } // Check which type should be added to the list ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); // Actually add that type if ( ! argVisitor.lastType() ) { return; } DUChainWriteLocker wlock; qCDebug(KDEV_PYTHON_DUCHAIN) << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); }; items["addsTypeOfArgContent"] = [&]() { const int offset = ! args.isEmpty() ? (int) args.at(0).toUInt() : 0; if ( node->arguments.length() <= offset ) { return; } ExpressionVisitor argVisitor(currentContext()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker wlock; auto contentType = Helper::contentOfIterable(argVisitor.lastType(), topContext()); container->addContentType(contentType); v.lastDeclaration()->setType(container); } }; auto docstring = function->comment(); if ( ! docstring.isEmpty() ) { foreach ( const auto& key, items.keys() ) { if ( Helper::docstringContainsHint(docstring, key, &args) ) { items[key](); } } } } void DeclarationBuilder::addArgumentTypeHints(CallAst* node, DeclarationPointer called) { DUChainReadLocker lock; auto funcInfo = Helper::functionForCalled(called.data()); auto function = funcInfo.declaration; if ( ! function ) { return; } - if ( function->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { + if ( function->topContext()->url() == Helper::getDocumentationFile() ) { return; } // Note: within this function: // - 'parameters' refers to the parameters of the function definition. // - 'arguments' refers to the arguments of the function call. DUContext* parameterContext = DUChainUtils::getArgumentContext(function); FunctionType::Ptr functionType = function->type(); if ( ! parameterContext || ! functionType ) { return; } QVector parameters = parameterContext->localDeclarations(); if ( parameters.isEmpty() ) { return; } const int specialParamsCount = (function->vararg() != -1) + (function->kwarg() != -1); // Look for the "self" in the argument list, the type of that should not be updated. bool hasSelfParam = false; if ( ( function->context()->type() == DUContext::Class || funcInfo.isConstructor ) && ! function->isStatic() ) { // ... unless for some reason the function only has *vararg, **kwarg as parameters // (this could happen for example if the method is static but kdev-python does not know, // or if the user just made a mistake in his code) if ( specialParamsCount < parameters.size() ) { hasSelfParam = true; } } lock.unlock(); bool explicitSelfArgument = false; if ( hasSelfParam && ! function->isClassMethod() && node->function->astType == Ast::AttributeAstType ) { // Calling an attribute, e.g. `instance.foo(arg)` or `MyClass.foo(instance, arg)`. ExpressionVisitor valueVisitor(currentContext()); valueVisitor.visitNode(static_cast(node->function)->value); if ( valueVisitor.lastDeclaration().dynamicCast() && valueVisitor.isAlias() ) { // Function is attribute of a class _type_ (not instance), so first arg is used as `self`. explicitSelfArgument = true; } } int currentParamIndex = hasSelfParam; int currentArgumentIndex = explicitSelfArgument; int indexInVararg = -1; int paramsAvailable = qMin(functionType->arguments().length(), parameters.size()); int argsAvailable = node->arguments.size(); bool atVararg = false; // Iterate over all the arguments, trying to guess the type of the object being // passed as an argument, and update the parameter accordingly. // Stop if more parameters supplied than possible, and we're not at the vararg. for ( ; ( atVararg || currentParamIndex < paramsAvailable ) && currentArgumentIndex < argsAvailable; currentArgumentIndex++ ) { atVararg = atVararg || currentParamIndex == function->vararg(); // Not >=, nonexistent vararg is -1. ExpressionAst* arg = node->arguments.at(currentArgumentIndex); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(arg); AbstractType::Ptr argumentType = argumentVisitor.lastType(); // Update the parameter type: change both the type of the function argument, // and the type of the declaration which belongs to that argument HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); DUChainWriteLocker wlock; if ( atVararg ) { indexInVararg++; Declaration* parameter = parameters.at(function->vararg()); IndexedContainer::Ptr varargContainer = parameter->type(); if ( ! varargContainer ) continue; if ( varargContainer->typesCount() > indexInVararg ) { AbstractType::Ptr oldType = varargContainer->typeAt(indexInVararg).abstractType(); AbstractType::Ptr newType = Helper::mergeTypes(oldType, addType.cast()); varargContainer->replaceType(indexInVararg, newType); } else { varargContainer->addEntry(addType.cast()); } parameter->setAbstractType(varargContainer.cast()); } else { if ( ! argumentType ) continue; AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(currentParamIndex)->abstractType(), addType.cast()); // TODO this does not correctly update the types in quickopen! Investigate why. functionType->removeArgument(currentParamIndex); functionType->addArgument(newType, currentParamIndex); function->setAbstractType(functionType.cast()); parameters.at(currentParamIndex)->setType(newType); currentParamIndex++; } } // **kwargs is always the last parameter MapType::Ptr kwargsDict; if ( function->kwarg() != -1 ) { kwargsDict = parameters.last()->abstractType().cast(); } lock.unlock(); DUChainWriteLocker wlock; foreach ( KeywordAst* keyword, node->keywords ) { wlock.unlock(); ExpressionVisitor argumentVisitor(currentContext()); argumentVisitor.visitNode(keyword->value); if ( ! argumentVisitor.lastType() ) { continue; } wlock.lock(); bool matchedNamedParam = false; HintedType::Ptr addType = HintedType::Ptr(new HintedType()); if ( keyword->argumentName ) { openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); for (int ip = currentParamIndex; ip < paramsAvailable; ++ip ) { if ( parameters.at(ip)->identifier().toString() != keyword->argumentName->value ) { continue; } matchedNamedParam = true; auto newType = Helper::mergeTypes(parameters.at(ip)->abstractType(), addType); functionType->removeArgument(ip); functionType->addArgument(newType, ip); parameters.at(ip)->setType(newType); } } else if ( auto unpackedDict = argumentVisitor.lastType().cast() ) { // 'keyword is actually an unpacked dict: `foo(**{'a': 12}). openType(addType); addType->setType(unpackedDict->contentType().abstractType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); } else { // Maybe the dict type wasn't loaded yet, or something else happened. continue; } if ( ! matchedNamedParam && kwargsDict ) { DUChainWriteLocker lock; kwargsDict->addContentType(addType); parameters.last()->setAbstractType(kwargsDict); } } function->setAbstractType(functionType); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); // Find the function being called; this code also handles cases where non-names // are called, for example: // class myclass(): // def myfun(self): return 3 // l = [myclass()] // x = l[0].myfun() # the called object is actually l[0].myfun // In the above example, this call will be evaluated to "myclass.myfun" in the following statement. ExpressionVisitor functionVisitor(currentContext()); functionVisitor.visitNode(node); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastDeclaration() ) { // Some special functions, like "append", update the content of the object they operate on. // Find the object the function is called on, like for d = [1, 2, 3]; d.append(5), this will give "d" FunctionDeclaration::Ptr function = functionVisitor.lastDeclaration().dynamicCast(); applyDocstringHints(node, function); } if ( ! m_prebuilding ) { return; } // The following code will try to update types of function parameters based on what is passed // for those when the function is used. // In case of this code: // def foo(arg): print arg // foo(3) // the following will change the type of "arg" to be "int" when it processes the second line. addArgumentTypeHints(node, functionVisitor.lastDeclaration()); } void DeclarationBuilder::assignToName(NameAst* target, const DeclarationBuilder::SourceType& element) { if ( element.isAlias ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(target->identifier, AliasDeclarationType); decl->setAliasedDeclaration(element.declaration.data()); closeDeclaration(); } else { DUChainWriteLocker lock; Declaration* dec = visitVariableDeclaration(target, nullptr, element.type); if ( dec && m_lastComment && ! m_lastComment->usedAsComment ) { dec->setComment(m_lastComment->value); m_lastComment->usedAsComment = true; } /** DEBUG **/ if ( element.type && dec ) { Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } void DeclarationBuilder::assignToSubscript(SubscriptAst* subscript, const DeclarationBuilder::SourceType& element) { ExpressionAst* v = subscript->value; if ( ! element.type ) { return; } ExpressionVisitor targetVisitor(currentContext()); targetVisitor.visitNode(v); auto list = ListType::Ptr::dynamicCast(targetVisitor.lastType()); if ( list ) { DUChainWriteLocker lock; list->addContentType(element.type); } auto map = MapType::Ptr::dynamicCast(list); if ( map ) { if ( subscript->slice && subscript->slice->astType == Ast::IndexAstType ) { ExpressionVisitor keyVisitor(currentContext()); keyVisitor.visitNode(static_cast(subscript->slice)->value); AbstractType::Ptr key = keyVisitor.lastType(); if ( key ) { map->addKeyType(key); } } } DeclarationPointer lastDecl = targetVisitor.lastDeclaration(); if ( list && lastDecl ) { DUChainWriteLocker lock; lastDecl->setAbstractType(list.cast()); } } void DeclarationBuilder::assignToAttribute(AttributeAst* attrib, const DeclarationBuilder::SourceType& element) { // visit the base expression before the dot ExpressionVisitor checkPreviousAttributes(currentContext()); checkPreviousAttributes.visitNode(attrib->value); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); DUContextPointer internal(nullptr); if ( ! parentObjectDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration for attribute base, aborting creation of attribute"; return; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { DUChainReadLocker lock; StructureType::Ptr structure(parentObjectDeclaration->abstractType().cast()); if ( ! structure || ! structure->declaration(topContext()) ) { return; } parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); } if ( ! internal ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No internal context for structure type, aborting creation of attribute declaration"; return; } Declaration* attributeDeclaration = nullptr; { DUChainReadLocker lock; attributeDeclaration = Helper::accessAttribute(parentObjectDeclaration->abstractType(), attrib->attribute->value, topContext()); } if ( ! attributeDeclaration || ! wasEncountered(attributeDeclaration) ) { // inject a new attribute into the class type DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreadyOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration( attrib->attribute, attributeDeclaration, element.type, AbortIfReopenMismatch ); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock; previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); } closeInjectedContext(); } } else { DUChainWriteLocker lock; // the declaration is already there, just update the type if ( ! attributeDeclaration->type() ) { auto newType = Helper::mergeTypes(attributeDeclaration->abstractType(), element.type); attributeDeclaration->setAbstractType(newType); } } } void DeclarationBuilder::tryUnpackType(AbstractType::Ptr sourceType, QVector& outTypes, int starred) { if ( const auto indexed = sourceType.cast() ) { int spare = indexed->typesCount() - outTypes.length(); if ( spare < -1 || (starred == -1 && spare != 0) ) { return; // Wrong number of elements to unpack. } for ( int i_out = 0, i_in = 0; i_out < outTypes.length(); ++i_out ) { if ( i_out == starred ) { // PEP-3132. Made into list in assignToTuple(). for (; spare >= 0; --spare, ++i_in ) { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); } } else { auto content = indexed->typeAt(i_in).abstractType(); outTypes[i_out] = Helper::mergeTypes(outTypes.at(i_out), content); ++i_in; } } } else { auto content = Helper::contentOfIterable(sourceType, topContext()); if ( !Helper::isUsefulType(content) ) { return; } for (auto out = outTypes.begin(); out != outTypes.end(); ++out) { *out = Helper::mergeTypes(*out, content); } } } void DeclarationBuilder::assignToTuple(TupleAst* tuple, const SourceType& element) { int starred = -1; // Index (if any) of PEP-3132 starred assignment. for (int ii = 0; ii < tuple->elements.length(); ++ii) { if (tuple->elements.at(ii)->astType == Ast::StarredAstType) { starred = ii; break; } } QVector outTypes(tuple->elements.length()); if ( auto unsure = element.type.cast() ) { FOREACH_FUNCTION ( const auto& type, unsure->types ) { tryUnpackType(type.abstractType(), outTypes, starred); } } else { tryUnpackType(element.type, outTypes, starred); } for (int ii = 0; ii < outTypes.length(); ++ii) { const auto sourceType = outTypes.at(ii); auto target = tuple->elements.at(ii); if ( target->astType == Ast::StarredAstType ) { DUChainReadLocker lock; auto listType = ExpressionVisitor::typeObjectForIntegralType("list"); lock.unlock(); if (listType) { listType->addContentType(sourceType); assignToUnknown(static_cast(target)->value, listType); } } else { assignToUnknown(target, sourceType); } } } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const AbstractType::Ptr type) { auto source = SourceType{ type, DeclarationPointer(), false }; assignToUnknown(target, source); } void DeclarationBuilder::assignToUnknown(ExpressionAst* target, const DeclarationBuilder::SourceType& element) { // Must be a nicer way to do this. if ( target->astType == Ast::TupleAstType ) { // Assignments of the form "a, b = 1, 2" or "a, b = c" assignToTuple(static_cast(target), element); } else if ( target->astType == Ast::NameAstType ) { // Assignments of the form "a = 3" assignToName(static_cast(target), element); } else if ( target->astType == Ast::SubscriptAstType ) { // Assignments of the form "a[0] = 3" assignToSubscript(static_cast(target), element); } else if ( target->astType == Ast::AttributeAstType ) { // Assignments of the form "a.b = 3" assignToAttribute(static_cast(target), element); } } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { AstDefaultVisitor::visitAssignment(node); ExpressionVisitor v(currentContext()); v.visitNode(node->value); auto sourceType = SourceType{ v.lastType(), DeclarationPointer(Helper::resolveAliasDeclaration(v.lastDeclaration().data())), v.isAlias() }; foreach(ExpressionAst* target, node->targets) { assignToUnknown(target, sourceType); } } void DeclarationBuilder::visitAnnotationAssignment(AnnotationAssignmentAst* node) { ExpressionVisitor v(currentContext()); v.visitNode(node->target); v.visitNode(node->value); auto assignType = v.lastType(); // Never mind aliasing, why annotate that? v.visitNode(node->annotation); assignType = Helper::mergeTypes(assignType, v.lastType()); assignToUnknown(node->target, assignType); } void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { visitNodeList(node->decorators); const CorrectionHelper::Recursion r(m_correctionHelper->enterClass(node->name->value)); StructureType::Ptr type(new StructureType()); DUChainWriteLocker lock; ClassDeclaration* dec = eventuallyReopenDeclaration(node->name, NoTypeRequired); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); auto docstring = getDocstring(node->body); dec->setComment(docstring); if ( ! docstring.isEmpty() ) { // check whether this is a type container (list, dict, ...) or just a "normal" class if ( Helper::docstringContainsHint(docstring, "TypeContainer") ) { ListType* container = nullptr; if ( Helper::docstringContainsHint(docstring, "hasTypedKeys") ) { container = new MapType(); } else { container = new ListType(); } type = StructureType::Ptr(container); } if ( Helper::docstringContainsHint(docstring, "IndexedTypeContainer") ) { IndexedContainer* container = new IndexedContainer(); type = StructureType::Ptr(container); } } lock.unlock(); foreach ( ExpressionAst* c, node->baseClasses ) { // Iterate over all the base classes, and add them to the duchain. ExpressionVisitor v(currentContext()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; lock.lock(); dec->addBaseClass(base); lock.unlock(); } } lock.lock(); // every python class inherits from "object". // We use this to add all the __str__, __get__, ... methods. if ( dec->baseClassesSize() == 0 && node->name->value != "object" ) { DUChainWriteLocker wlock; ReferencedTopDUContext docContext = Helper::getDocumentationFileContext(); if ( docContext ) { QList object = docContext->findDeclarations( QualifiedIdentifier("object") ); if ( ! object.isEmpty() && object.first()->abstractType() ) { Declaration* objDecl = object.first(); BaseClassInstance base; base.baseClass = objDecl->abstractType()->indexed(); // this can be queried from autocompletion or elsewhere to hide the items, if required; // of course, it's not private strictly speaking base.access = KDevelop::Declaration::Private; dec->addBaseClass(base); } } } type->setDeclaration(dec); dec->setType(type); openType(type); m_currentClassTypes.append(type); // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); dec->setInternalContext(currentContext()); lock.unlock(); visitNodeList(node->body); lock.lock(); closeContext(); m_currentClassTypes.removeLast(); closeType(); closeDeclaration(); } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { const CorrectionHelper::Recursion r(m_correctionHelper->enterFunction(node->name->value)); // Search for an eventual containing class declaration; // if that exists, then this function is a member function DeclarationPointer eventualParentDeclaration(currentDeclaration()); FunctionType::Ptr type(new FunctionType()); DUChainWriteLocker lock; FunctionDeclaration* dec = eventuallyReopenDeclaration(node->name, FunctionDeclarationType); Q_ASSERT(dec->isFunctionDeclaration()); // check for documentation dec->setComment(getDocstring(node->body)); openType(type); dec->setInSymbolTable(false); dec->setType(type); lock.unlock(); dec->setStatic(false); dec->setClassMethod(false); dec->setProperty(false); foreach ( auto decorator, node->decorators) { visitNode(decorator); switch (decorator->astType) { case Ast::AttributeAstType: { auto attr = static_cast(decorator)->attribute->value; if ( attr == QStringLiteral("setter") || attr == QStringLiteral("getter") || attr == QStringLiteral("deleter") ) dec->setProperty(true); break; } case Ast::NameAstType: { auto name = static_cast(decorator)->identifier->value; if ( name == QStringLiteral("staticmethod") ) dec->setStatic(true); else if ( name == QStringLiteral("classmethod") ) dec->setClassMethod(true); else if ( name == QStringLiteral("property") ) dec->setProperty(true); break; } default: {} } } visitFunctionArguments(node); visitFunctionBody(node); lock.lock(); closeDeclaration(); eventuallyAssignInternalContext(); closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { static IndexedString constructorName("__init__"); DUChainWriteLocker lock(DUChain::lock()); if ( dec->identifier().identifier() == constructorName ) { // the constructor returns an instance of the object, // nice to display it in tooltips etc. type->setReturnType(currentType()); } if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new NoneType())); } dec->setType(type); } if ( ! dec->isStatic() ) { DUContext* args = DUChainUtils::getArgumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); static IndexedString newMethodName("__new__"); static IndexedString selfArgumentName("self"); static IndexedString clsArgumentName("cls"); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() && ! dec->isClassMethod() ) { QString description; if ( dec->identifier().identifier() == newMethodName && parameters[0]->identifier().identifier() != clsArgumentName ) { description = i18n("First argument of __new__ method is not called cls, this is deprecated"); } else if ( dec->identifier().identifier() != newMethodName && parameters[0]->identifier().identifier() != selfArgumentName ) { description = i18n("First argument of class method is not called self, this is deprecated"); } if ( ! description.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setDescription(description); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), parameters[0]->range().castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); // only mark first line p->setFinalLocation(DocumentRange(currentlyParsedDocument(), KTextEditor::Range(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } if ( AbstractType::Ptr hint = m_correctionHelper->returnTypeHint() ) { type->setReturnType(hint); dec->setType(type); } // check for (python3) function annotations if ( node->returns ) { lock.unlock(); ExpressionVisitor v(currentContext()); v.visitNode(node->returns); lock.lock(); if ( v.lastType() && v.isAlias() ) { type->setReturnType(Helper::mergeTypes(type->returnType(), v.lastType())); dec->setType(type); } else if ( ! v.isAlias()) { qCDebug(KDEV_PYTHON_DUCHAIN) << "not updating function return type because expression is not a type object"; } } lock.lock(); dec->setInSymbolTable(true); } QString DeclarationBuilder::getDocstring(QList< Python::Ast* > body) const { if ( ! body.isEmpty() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { // If the first statement in a function/class body is a string, then that is the docstring. StringAst* docstring = static_cast(static_cast(body.first())->value); docstring->usedAsComment = true; return docstring->value.trimmed(); } return QString(); } void DeclarationBuilder::visitAssertion(AssertionAst* node) { adjustForTypecheck(node->condition, false); Python::AstDefaultVisitor::visitAssertion(node); } void DeclarationBuilder::visitIf(IfAst* node) { adjustForTypecheck(node->condition, true); Python::AstDefaultVisitor::visitIf(node); } void DeclarationBuilder::adjustForTypecheck(Python::ExpressionAst* check, bool useUnsure) { if ( ! check ) return; if ( check->astType == Ast::UnaryOperationAstType && static_cast(check)->type == Ast::UnaryOperatorNot ) { // It could be something like " if not isinstance(foo, Bar): return None ". check = static_cast(check)->operand; } if ( check->astType == Ast::CallAstType ) { // Is this a call of the form "isinstance(foo, bar)"? CallAst* call = static_cast(check); if ( ! call->function ) { return; } if ( call->function->astType != Ast::NameAstType ) { return; } const QString functionName = static_cast(call->function)->identifier->value; if ( functionName != QLatin1String("isinstance") ) { return; } if ( call->arguments.length() != 2 ) { return; } adjustExpressionsForTypecheck(call->arguments.at(0), call->arguments.at(1), useUnsure); } else if ( check->astType == Ast::CompareAstType ) { // Is this a call of the form "type(ainstance) == a"? CompareAst* compare = static_cast(check); if ( compare->operators.size() != 1 || compare->comparands.size() != 1 ) { return; } if ( compare->operators.first() != Ast::ComparisonOperatorEquals ) { return; } ExpressionAst* c1 = compare->comparands.first(); ExpressionAst* c2 = compare->leftmostElement; if ( ! ( (c1->astType == Ast::CallAstType) ^ (c2->astType == Ast::CallAstType) ) ) { // Exactly one of the two must be a call. TODO: support adjusting function return types return; } CallAst* typecall = static_cast(c1->astType == Ast::CallAstType ? c1 : c2); if ( ! typecall->function || typecall->function->astType != Ast::NameAstType || typecall->arguments.length() != 1 ) { return; } const QString functionName = static_cast(typecall->function)->identifier->value; if ( functionName != QLatin1String("type") ) { return; } adjustExpressionsForTypecheck(typecall->arguments.at(0), c1->astType == Ast::CallAstType ? c2 : c1, useUnsure); } } void DeclarationBuilder::adjustExpressionsForTypecheck(Python::ExpressionAst* adjustExpr, Python::ExpressionAst* from, bool useUnsure) { // Find types of the two arguments ExpressionVisitor first(currentContext()); ExpressionVisitor second(currentContext()); first.visitNode(adjustExpr); second.visitNode(from); AbstractType::Ptr hint; DeclarationPointer adjust; if ( second.isAlias() && second.lastType() ) { hint = second.lastType(); adjust = first.lastDeclaration(); } if ( ! adjust || adjust->isFunctionDeclaration() ) { // no declaration for the thing to verify, can't adjust it. return; } else if ( adjust->topContext() == Helper::getDocumentationFileContext() ) { // do not motify types in the doc context return; } DUChainWriteLocker lock; if ( useUnsure ) { adjust->setAbstractType(Helper::mergeTypes(adjust->abstractType(), hint)); } else { adjust->setAbstractType(hint); } } void DeclarationBuilder::visitReturn(ReturnAst* node) { static auto noneType = AbstractType::Ptr(new NoneType()); if ( auto function = currentType() ) { // Statements with no explicit value return `None`. auto encountered = noneType; if ( node->value ) { // Find the type of the object being "return"ed ExpressionVisitor v(currentContext()); v.visitNode(node->value); encountered = v.lastType(); } // Update the containing function's return type DUChainWriteLocker lock; function->setReturnType(Helper::mergeTypes(function->returnType(), encountered)); } else { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), node->range())); // only mark first line p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } DeclarationBuilderBase::visitReturn(node); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { if ( ! currentDeclaration() || ! currentDeclaration()->isFunctionDeclaration() ) { return; } FunctionDeclaration* workingOnDeclaration = static_cast(Helper::resolveAliasDeclaration(currentDeclaration())); workingOnDeclaration->clearDefaultParameters(); if ( ! hasCurrentType() || ! currentType() ) { return; } FunctionType::Ptr type = currentType(); bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; foreach ( ArgAst* arg, node->arguments + node->kwonlyargs ) { // Iterate over all the function's arguments, create declarations, and add the arguments // to the functions FunctionType. currentIndex += 1; if ( ! arg->argumentName ) { continue; } // Create a variable declaration for the parameter, to be used in the function body. Declaration* paramDeclaration = nullptr; if ( currentIndex == 1 && workingOnDeclaration->isClassMethod() ) { DUChainWriteLocker lock; AliasDeclaration* decl = eventuallyReopenDeclaration(arg->argumentName, AliasDeclarationType); if ( ! m_currentClassTypes.isEmpty() ) { auto classDecl = m_currentClassTypes.last()->declaration(topContext()); decl->setAliasedDeclaration(classDecl); } closeDeclaration(); paramDeclaration = decl; } else { paramDeclaration = visitVariableDeclaration(arg->argumentName); } if ( ! paramDeclaration ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "could not create parameter declaration!"; continue; } AbstractType::Ptr argumentType(new IntegralType(IntegralType::TypeMixed)); if ( arg->annotation ) { ExpressionVisitor v(currentContext()); v.visitNode(arg->annotation); if ( v.lastType() && v.isAlias() ) { DUChainWriteLocker lock; argumentType = Helper::mergeTypes(paramDeclaration->abstractType(), v.lastType()); } } else if ( currentIndex > firstDefaultParameterOffset && currentIndex <= node->arguments.size() ) { // Handle arguments with default values, like def foo(bar = 3): pass // Find type of given default value, and assign it to the declaration ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { argumentType = v.lastType(); } // TODO add the real expression from the document here as default value workingOnDeclaration->addDefaultParameter(IndexedString("...")); } if ( isFirst && ! workingOnDeclaration->isStatic() && currentContext() && currentContext()->parentContext() ) { DUChainReadLocker lock; if ( currentContext()->parentContext()->type() == DUContext::Class ) { argumentType = m_currentClassTypes.last().cast(); isFirst = false; } } DUChainWriteLocker lock; paramDeclaration->setAbstractType(Helper::mergeTypes(paramDeclaration->abstractType(), argumentType)); type->addArgument(argumentType); } // Handle *args, **kwargs, and assign them a list / dictionary type. if ( node->vararg ) { // inject the vararg at the correct place int atIndex = 0; int useIndex = -1; foreach ( ArgAst* arg, node->arguments ) { if ( node->vararg && workingOnDeclaration->vararg() == -1 && node->vararg->appearsBefore(arg) ) { useIndex = atIndex; } atIndex += 1; } if ( useIndex == -1 ) { // if the vararg does not appear in the middle of the params, place it at the end. // this is new in python3, you can do like def fun(a, b, *c, z): pass useIndex = type->arguments().size(); } DUChainReadLocker lock; IndexedContainer::Ptr tupleType = ExpressionVisitor::typeObjectForIntegralType("tuple"); lock.unlock(); if ( tupleType ) { visitVariableDeclaration(node->vararg->argumentName, nullptr, tupleType.cast()); workingOnDeclaration->setVararg(atIndex); type->addArgument(tupleType.cast(), useIndex); } } if ( node->kwarg ) { DUChainReadLocker lock; AbstractType::Ptr stringType = ExpressionVisitor::typeObjectForIntegralType("str"); auto dictType = ExpressionVisitor::typeObjectForIntegralType("dict"); lock.unlock(); if ( dictType && stringType ) { dictType->addKeyType(stringType); visitVariableDeclaration(node->kwarg->argumentName, nullptr, dictType.cast()); type->addArgument(dictType.cast()); workingOnDeclaration->setKwarg(type->arguments().size() - 1); } } } void DeclarationBuilder::visitString(StringAst* node) { if ( node->parent && node->parent->astType == Ast::ExpressionAstType ) { m_lastComment = node; } DeclarationBuilderBase::visitString(node); } void DeclarationBuilder::visitNode(Ast* node) { DeclarationBuilderBase::visitNode(node); if ( node && node->astType >= Ast::StatementAstType && node->astType <= Ast::LastStatementType) { m_lastComment = nullptr; } } void DeclarationBuilder::visitGlobal(GlobalAst* node) { TopDUContext* top = topContext(); foreach ( Identifier *id, node->names ) { QualifiedIdentifier qid = identifierForNode(id); DUChainWriteLocker lock; QList< Declaration* > existing = top->findLocalDeclarations(qid.first()); if ( ! existing.empty() ) { AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(existing.first()); closeDeclaration(); } else { injectContext(top); Declaration* dec = visitVariableDeclaration(id); dec->setRange(editorFindRange(id, id)); dec->setAutoDeclaration(true); closeContext(); AliasDeclaration* ndec = openDeclaration(id); ndec->setAliasedDeclaration(dec); closeDeclaration(); } } } } diff --git a/duchain/helpers.cpp b/duchain/helpers.cpp index 5831c16e..ff2f2312 100644 --- a/duchain/helpers.cpp +++ b/duchain/helpers.cpp @@ -1,558 +1,560 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2011-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "helpers.h" #include #include #include #include #include #include "duchaindebug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ast.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "types/indexedcontainer.h" #include "kdevpythonversion.h" #include "expressionvisitor.h" using namespace KDevelop; namespace Python { QMap> Helper::cachedCustomIncludes; QMap> Helper::cachedSearchPaths; QVector Helper::projectSearchPaths; QStringList Helper::dataDirs; -QString Helper::documentationFile; +IndexedString Helper::documentationFile; DUChainPointer Helper::documentationFileContext = DUChainPointer(nullptr); QStringList Helper::correctionFileDirs; QString Helper::localCorrectionFileDir; QMutex Helper::cacheMutex; QMutex Helper::projectPathLock; void Helper::scheduleDependency(const IndexedString& dependency, int betterThanPriority) { BackgroundParser* bgparser = KDevelop::ICore::self()->languageController()->backgroundParser(); bool needsReschedule = true; if ( bgparser->isQueued(dependency) ) { const auto priority= bgparser->priorityForDocument(dependency); if ( priority > betterThanPriority - 1 ) { bgparser->removeDocument(dependency); } else { needsReschedule = false; } } if ( needsReschedule ) { bgparser->addDocument(dependency, TopDUContext::ForceUpdate, betterThanPriority - 1, nullptr, ParseJob::FullSequentialProcessing); } } IndexedDeclaration Helper::declarationUnderCursor(bool allowUse) { KDevelop::IDocument* doc = ICore::self()->documentController()->activeDocument(); const auto view = static_cast(ICore::self()->partController())->activeView(); if ( doc && doc->textDocument() && view ) { DUChainReadLocker lock; const auto cursor = view->cursorPosition(); if ( allowUse ) { return IndexedDeclaration(DUChainUtils::itemUnderCursor(doc->url(), cursor).declaration); } else { return DUChainUtils::declarationInLine(cursor, DUChainUtils::standardContextForUrl(doc->url())); } } return KDevelop::IndexedDeclaration(); } Declaration* Helper::accessAttribute(const AbstractType::Ptr accessed, const IndexedIdentifier& attribute, const TopDUContext* topContext) { if ( ! accessed ) { return nullptr; } // if the type is unsure, search all the possibilities (but return the first match) auto structureTypes = Helper::filterType(accessed, [](AbstractType::Ptr toFilter) { auto type = Helper::resolveAliasType(toFilter); return type && type->whichType() == AbstractType::TypeStructure; }, [](AbstractType::Ptr toMap) { return StructureType::Ptr::staticCast(Helper::resolveAliasType(toMap)); } ); auto docFileContext = Helper::getDocumentationFileContext(); for ( const auto& type: structureTypes ) { auto searchContexts = Helper::internalContextsForClass(type, topContext); for ( const auto ctx: searchContexts ) { auto found = ctx->findDeclarations(attribute, CursorInRevision::invalid(), topContext, DUContext::DontSearchInParent); if ( !found.isEmpty() && ( found.last()->topContext() != docFileContext || ctx->topContext() == docFileContext) ) { // never consider decls from the builtins return found.last(); } } } return nullptr; } AbstractType::Ptr Helper::resolveAliasType(const AbstractType::Ptr eventualAlias) { return TypeUtils::resolveAliasType(eventualAlias); } AbstractType::Ptr Helper::extractTypeHints(AbstractType::Ptr type) { return Helper::foldTypes(Helper::filterType(type, [](AbstractType::Ptr t) -> bool { auto hint = t.cast(); return !hint || hint->isValid(); })); } Helper::FuncInfo Helper::functionForCalled(Declaration* called, bool isAlias) { if ( ! called ) { return { nullptr, false }; } else if ( called->isFunctionDeclaration() ) { return { static_cast( called ), false }; } // If we're calling a type object (isAlias == true), look for a constructor. static const IndexedIdentifier initId(KDevelop::Identifier("__init__")); // Otherwise look for a `__call__()` method. static const IndexedIdentifier callId(KDevelop::Identifier("__call__")); auto attr = accessAttribute(called->abstractType(), (isAlias ? initId : callId), called->topContext()); return { dynamic_cast(attr), isAlias }; } Declaration* Helper::declarationForName(const QString& name, const CursorInRevision& location, KDevelop::DUChainPointer context) { DUChainReadLocker lock(DUChain::lock()); auto identifier = KDevelop::Identifier(name); auto localDeclarations = context->findLocalDeclarations(identifier, location, 0, AbstractType::Ptr(0), DUContext::DontResolveAliases); if ( !localDeclarations.isEmpty() ) { return localDeclarations.last(); } QList declarations; const DUContext* currentContext = context.data(); bool findInNext = true, findBeyondUse = false; do { if (findInNext) { CursorInRevision findUntil = findBeyondUse ? currentContext->topContext()->range().end : location; declarations = currentContext->findDeclarations(identifier, findUntil); for (Declaration* declaration: declarations) { if (declaration->context()->type() != DUContext::Class || (currentContext->type() == DUContext::Function && declaration->context() == currentContext->parentContext())) { // Declarations from class decls must be referenced through `self.`, except // in their local scope (handled above) or when used as default arguments for methods of the same class. // Otherwise, we're done! return declaration; } } if (!declarations.isEmpty()) { // If we found declarations but rejected all of them (i.e. didn't return), we need to keep searching. findInNext = true; declarations.clear(); } } if (!findBeyondUse && currentContext->owner() && currentContext->owner()->isFunctionDeclaration()) { // Names in the body may be defined after the function definition, before the function is called. // Note: only the parameter list has type DUContext::Function, so we have to do this instead. findBeyondUse = findInNext = true; } } while ((currentContext = currentContext->parentContext())); return nullptr; } Declaration* Helper::declarationForName(const Python::NameAst* name, CursorInRevision location, KDevelop::DUChainPointer context) { const Ast* checkNode = name; while ((checkNode = checkNode->parent)) { switch (checkNode->astType) { default: continue; case Ast::ListComprehensionAstType: case Ast::SetComprehensionAstType: case Ast::DictionaryComprehensionAstType: case Ast::GeneratorExpressionAstType: // Variables in comprehensions are used before their definition. `[foo for foo in bar]` auto cmpEnd = CursorInRevision(checkNode->endLine, checkNode->endCol); if (cmpEnd > location) { location = cmpEnd; } } } return declarationForName(name->identifier->value, location, context); } QVector Helper::internalContextsForClass(const StructureType::Ptr classType, const TopDUContext* context, ContextSearchFlags flags, int depth) { QVector searchContexts; if ( ! classType ) { return searchContexts; } if ( auto c = classType->internalContext(context) ) { searchContexts << c; } auto decl = Helper::resolveAliasDeclaration(classType->declaration(context)); if ( auto classDecl = dynamic_cast(decl) ) { FOREACH_FUNCTION ( const auto& base, classDecl->baseClasses ) { if ( flags == PublicOnly && base.access == KDevelop::Declaration::Private ) { continue; } auto baseClassType = base.baseClass.type(); // recursive call, because the base class will have more base classes eventually if ( depth < 10 ) { searchContexts.append(Helper::internalContextsForClass(baseClassType, context, flags, depth + 1)); } } } return searchContexts; } Declaration* Helper::resolveAliasDeclaration(Declaration* decl) { AliasDeclaration* alias = dynamic_cast(decl); if ( alias ) { DUChainReadLocker lock; return alias->aliasedDeclaration().data(); } else return decl; } QStringList Helper::getDataDirs() { if ( Helper::dataDirs.isEmpty() ) { Helper::dataDirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files", QStandardPaths::LocateDirectory); } return Helper::dataDirs; } -QString Helper::getDocumentationFile() { - if ( Helper::documentationFile.isNull() ) { - Helper::documentationFile = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); +KDevelop::IndexedString Helper::getDocumentationFile() +{ + if ( Helper::documentationFile.isEmpty() ) { + auto path = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); + Helper::documentationFile = IndexedString(path); } return Helper::documentationFile; } ReferencedTopDUContext Helper::getDocumentationFileContext() { if ( Helper::documentationFileContext ) { return ReferencedTopDUContext(Helper::documentationFileContext.data()); } else { DUChainReadLocker lock; - auto file = IndexedString(Helper::getDocumentationFile()); + auto file = Helper::getDocumentationFile(); ReferencedTopDUContext ctx = ReferencedTopDUContext(DUChain::self()->chainForDocument(file)); Helper::documentationFileContext = DUChainPointer(ctx.data()); return ctx; } } // stolen from KUrl. duh. static QString _relativePath(const QString &base_dir, const QString &path) { QString _base_dir(QDir::cleanPath(base_dir)); QString _path(QDir::cleanPath(path.isEmpty() || QDir::isRelativePath(path) ? _base_dir+QLatin1Char('/')+path : path)); if (_base_dir.isEmpty()) return _path; if (_base_dir[_base_dir.length()-1] != QLatin1Char('/')) _base_dir.append(QLatin1Char('/') ); const QStringList list1 = _base_dir.split(QLatin1Char('/'), QString::SkipEmptyParts); const QStringList list2 = _path.split(QLatin1Char('/'), QString::SkipEmptyParts); // Find where they meet int level = 0; int maxLevel = qMin(list1.count(), list2.count()); while((level < maxLevel) && (list1[level] == list2[level])) level++; QString result; // Need to go down out of the first path to the common branch. for(int i = level; i < list1.count(); i++) result.append(QLatin1String("../")); // Now up up from the common branch to the second path. for(int i = level; i < list2.count(); i++) result.append(list2[i]).append(QLatin1Char('/')); if ((level < list2.count()) && (path[path.length()-1] != QLatin1Char('/'))) result.truncate(result.length()-1); return result; } QUrl Helper::getCorrectionFile(const QUrl& document) { if ( Helper::correctionFileDirs.isEmpty() ) { Helper::correctionFileDirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files/", QStandardPaths::LocateDirectory); } foreach (QString correctionFileDir, correctionFileDirs) { foreach ( const QUrl& basePath, Helper::getSearchPaths(QUrl()) ) { if ( ! basePath.isParentOf(document) ) { continue; } auto base = basePath.path(); auto doc = document.path(); auto relative = _relativePath(base, doc); auto fullPath = correctionFileDir + "/" + relative; if ( QFile::exists(fullPath) ) { return QUrl::fromLocalFile(fullPath).adjusted(QUrl::NormalizePathSegments); } } } return {}; } QUrl Helper::getLocalCorrectionFile(const QUrl& document) { if ( Helper::localCorrectionFileDir.isNull() ) { Helper::localCorrectionFileDir = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + QLatin1Char('/') + "kdevpythonsupport/correction_files/"; } auto absolutePath = QUrl(); foreach ( const auto& basePath, Helper::getSearchPaths(QUrl()) ) { if ( ! basePath.isParentOf(document) ) { continue; } auto path = QDir(basePath.path()).relativeFilePath(document.path()); absolutePath = QUrl::fromLocalFile(Helper::localCorrectionFileDir + path); break; } return absolutePath; } QString Helper::getPythonExecutablePath(IProject* project) { if ( project ) { auto interpreter = project->projectConfiguration()->group("pythonsupport").readEntry("interpreter"); if ( !interpreter.isEmpty() ) { // we have a user-configured interpreter, try using it QFile f(interpreter); if ( f.exists() ) { return interpreter; } qCWarning(KDEV_PYTHON_DUCHAIN) << "Custom python interpreter" << interpreter << "configured for project" << project->name() << "is invalid, using default"; } } // Find python 3 (https://www.python.org/dev/peps/pep-0394/) auto result = QStandardPaths::findExecutable("python" PYTHON_VERSION_STR); if ( ! result.isEmpty() ) { return result; } result = QStandardPaths::findExecutable("python" PYTHON_VERSION_MAJOR_STR); if ( ! result.isEmpty() ) { return result; } result = QStandardPaths::findExecutable("python"); if ( ! result.isEmpty() ) { return result; } #ifdef Q_OS_WIN QStringList extraPaths; // Check for default CPython installation path, because the // installer does not add the path to $PATH. QStringList keys = { "HKEY_LOCAL_MACHINE\\Software\\Python\\PythonCore\\PYTHON_VERSION\\InstallPath", "HKEY_LOCAL_MACHINE\\Software\\Python\\PythonCore\\PYTHON_VERSION-32\\InstallPath", "HKEY_CURRENT_USER\\Software\\Python\\PythonCore\\PYTHON_VERSION\\InstallPath", "HKEY_CURRENT_USER\\Software\\Python\\PythonCore\\PYTHON_VERSION-32\\InstallPath" }; auto version = QString(PYTHON_VERSION_STR); foreach ( QString key, keys ) { key.replace("PYTHON_VERSION", version); QSettings base(key.left(key.indexOf("Python")), QSettings::NativeFormat); if ( ! base.childGroups().contains("Python") ) { continue; } QSettings keySettings(key, QSettings::NativeFormat); auto path = keySettings.value("Default").toString(); if ( ! path.isEmpty() ) { extraPaths << path; break; } } result = QStandardPaths::findExecutable("python", extraPaths); if ( ! result.isEmpty() ) { return result; } #endif // fallback return PYTHON_EXECUTABLE; } QVector Helper::getSearchPaths(const QUrl& workingOnDocument) { QMutexLocker lock(&Helper::cacheMutex); QVector searchPaths; // search in the projects, as they're packages and likely to be installed or added to PYTHONPATH later // and also add custom include paths that are defined in the projects auto project = ICore::self()->projectController()->findProjectForUrl(workingOnDocument); { QMutexLocker lock(&Helper::projectPathLock); searchPaths << Helper::projectSearchPaths; searchPaths << Helper::cachedCustomIncludes.value(project); } foreach ( const QString& path, getDataDirs() ) { searchPaths.append(QUrl::fromLocalFile(path)); } if ( !cachedSearchPaths.contains(project) ) { QVector cachedForProject; qCDebug(KDEV_PYTHON_DUCHAIN) << "*** Collecting search paths..."; QStringList getpath; getpath << "-c" << "import sys; sys.stdout.write('$|$'.join(sys.path))"; QProcess python; python.start(getPythonExecutablePath(project), getpath); python.waitForFinished(1000); QString pythonpath = QString::fromUtf8(python.readAllStandardOutput()); if ( ! pythonpath.isEmpty() ) { const auto paths = pythonpath.split("$|$", QString::SkipEmptyParts); foreach ( const QString& path, paths ) { cachedForProject.append(QUrl::fromLocalFile(path)); } } else { qCWarning(KDEV_PYTHON_DUCHAIN) << "Could not get search paths! Defaulting to stupid stuff."; searchPaths.append(QUrl::fromLocalFile("/usr/lib/python" PYTHON_VERSION_STR)); searchPaths.append(QUrl::fromLocalFile("/usr/lib/python" PYTHON_VERSION_STR "/site-packages")); QString path = qgetenv("PYTHONPATH"); QStringList paths = path.split(':'); foreach ( const QString& path, paths ) { cachedForProject.append(QUrl::fromLocalFile(path)); } } qCDebug(KDEV_PYTHON_DUCHAIN) << " *** Done. Got search paths: " << cachedSearchPaths; cachedSearchPaths.insert(project, cachedForProject); } searchPaths.append(cachedSearchPaths.value(project)); auto dir = workingOnDocument.adjusted(QUrl::RemoveFilename); if ( ! dir.isEmpty() ) { // search in the current packages searchPaths.append(dir); } return searchPaths; } bool Helper::isUsefulType(AbstractType::Ptr type) { return TypeUtils::isUsefulType(type); } AbstractType::Ptr Helper::contentOfIterable(const AbstractType::Ptr iterable, const TopDUContext* topContext) { auto types = filterType(iterable, [](AbstractType::Ptr t) { return t->whichType() == AbstractType::TypeStructure; } ); static const IndexedIdentifier iterId(KDevelop::Identifier("__iter__")); static const IndexedIdentifier nextId(KDevelop::Identifier("__next__")); AbstractType::Ptr content(new IntegralType(IntegralType::TypeMixed)); for ( const auto& type: types ) { if ( auto map = type.cast() ) { // Iterating over dicts gets keys, not values content = mergeTypes(content, map->keyType().abstractType()); continue; } else if ( auto list = type.cast() ) { content = mergeTypes(content, list->contentType().abstractType()); continue; } else if ( auto indexed = type.cast() ) { content = mergeTypes(content, indexed->asUnsureType()); continue; } DUChainReadLocker lock; // Content of an iterable object is iterable.__iter__().__next__(). if ( auto iterFunc = dynamic_cast(accessAttribute(type, iterId, topContext)) ) { if ( auto iterator = iterFunc->type()->returnType().cast() ) { if ( auto nextFunc = dynamic_cast(accessAttribute(iterator, nextId, topContext)) ) { content = mergeTypes(content, nextFunc->type()->returnType()); } } } } return content; } AbstractType::Ptr Helper::mergeTypes(AbstractType::Ptr type, const AbstractType::Ptr newType) { UnsureType::Ptr ret; return TypeUtils::mergeTypes(type, newType); } } diff --git a/duchain/helpers.h b/duchain/helpers.h index 018214a5..b77ad77d 100644 --- a/duchain/helpers.h +++ b/duchain/helpers.h @@ -1,227 +1,227 @@ /***************************************************************************** * This file is part of KDevelop * * Copyright 2011-2012 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #ifndef GLOBALHELPERS_H #define GLOBALHELPERS_H #include "pythonduchainexport.h" #include "types/unsuretype.h" #include "ast.h" #include #include #include #include #include #include #include #include #include #include #include "pythonduchainexport.h" #include "types/unsuretype.h" #include "declarations/functiondeclaration.h" #include "ast.h" using namespace KDevelop; namespace Python { class KDEVPYTHONDUCHAIN_EXPORT Helper { public: /** get search paths for python files **/ static QVector getSearchPaths(const QUrl& workingOnDocument); static QStringList dataDirs; - static QString documentationFile; + static IndexedString documentationFile; static QStringList correctionFileDirs; static QString localCorrectionFileDir; static DUChainPointer documentationFileContext; static QStringList getDataDirs(); - static QString getDocumentationFile(); + static IndexedString getDocumentationFile(); static ReferencedTopDUContext getDocumentationFileContext(); static QUrl getCorrectionFile(const QUrl& document); static QUrl getLocalCorrectionFile(const QUrl& document); static QMutex cacheMutex; static QMap> cachedCustomIncludes; static QMap> cachedSearchPaths; static QMutex projectPathLock; static QVector projectSearchPaths; static AbstractType::Ptr extractTypeHints(AbstractType::Ptr type); /** * @brief Get the declaration of 'accessed.attribute', or return null. * * @param accessed Type (Structure or Unsure) that should have this attribute. * @param attribute Which attribute to look for. * @param topContext Top context (for this file?) * @return Declaration* of the attribute, or null. * If UnsureType with >1 matching attributes, returns an arbitrary choice. **/ static KDevelop::Declaration* accessAttribute(const KDevelop::AbstractType::Ptr accessed, const KDevelop::IndexedIdentifier& attribute, const KDevelop::TopDUContext* topContext); static KDevelop::Declaration* accessAttribute(const KDevelop::AbstractType::Ptr accessed, const QString& attribute, const KDevelop::TopDUContext* topContext) { return accessAttribute(accessed, IndexedIdentifier(KDevelop::Identifier(attribute)), topContext); } static AbstractType::Ptr resolveAliasType(const AbstractType::Ptr eventualAlias); /** * @brief Get the content type(s) of something that is an iterable. * * @param iterable Type to get the contents of. Can be an unsure. * @return KDevelop::AbstractType::Ptr Content type. Might be an unsure. */ static AbstractType::Ptr contentOfIterable(const AbstractType::Ptr iterable, const TopDUContext* topContext); /** * @brief Get a list of types inside the passed type which match the specified filter. * The filter will be matched against the type only if it is not an unsure type, * or else against all types inside that unsure type. * @param type The type to search * @param accept Filter function, return true if you want the type. * @return QList< KDevelop::AbstractType::Ptr > list of types accepted by the filter. */ template static QList filterType(AbstractType::Ptr type, std::function accept, std::function map = std::function()) { QList types; if ( ! type ) { return types; } if ( type->whichType() == KDevelop::AbstractType::TypeUnsure ) { UnsureType::Ptr unsure(type.cast()); for ( unsigned int i = 0; i < unsure->typesSize(); i++ ) { AbstractType::Ptr t = unsure->types()[i].abstractType(); if ( accept(t) ) { types << ( map ? map(t) : t.cast() ); } } } else if ( accept(type) ) { types << ( map ? map(type) : type.cast() ); } return types; } static void scheduleDependency(const IndexedString& dependency, int betterThanPriority); static KDevelop::IndexedDeclaration declarationUnderCursor(bool allowUse = true); struct FuncInfo { FunctionDeclaration* declaration; bool isConstructor; }; /** * @brief Finds whether the specified called declaration is a function declaration, and if not, * checks for a class declaration; then returns the constructor * * @param called the declaration to check * @param isAlias whether the called declaration aliases a class or function definition. * @return the function pointer which was found, or an invalid pointer, and a bool * which is true when it is a constructor **/ static FuncInfo functionForCalled(Declaration* called, bool isAlias=true); static bool docstringContainsHint(const QString& comment, const QString& hintName, QStringList* args = nullptr) { // TODO cache types! this is horribly inefficient const QString search = "! " + hintName + " !"; int index = comment.indexOf(search); if ( index >= 0 ) { if ( args ) { int eol = comment.indexOf('\n', index); int start = index+search.size()+1; QString decl = comment.mid(start, eol-start); *args = decl.split(' '); } return true; } return false; } /** * @copydoc TypeUtils::mergeTypes */ static AbstractType::Ptr mergeTypes(AbstractType::Ptr type, const AbstractType::Ptr newType); /** * @brief Like mergeTypes(), but merges a list of types into a newly allocated type. * Returns mixed if the list is empty. * @return KDevelop::AbstractType::Ptr an unsure type consisting of all types in the list. */ template static AbstractType::Ptr foldTypes(QList types, std::function transform = std::function()) { AbstractType::Ptr result(new IntegralType(IntegralType::TypeMixed)); for ( T type : types ) { result = Helper::mergeTypes(result, transform ? transform(type) : AbstractType::Ptr::staticCast(type)); } return result; }; /** check whether the argument is a null, mixed, or none integral type **/ static bool isUsefulType(AbstractType::Ptr type); enum ContextSearchFlags { NoFlags, PublicOnly }; /** * @brief Find all internal contexts for this class and its base classes recursively * * @param classType Type object for the class to search contexts * @param context TopContext for finding the declarations for types * @return list of contexts which were found **/ static QVector internalContextsForClass(const KDevelop::StructureType::Ptr classType, const TopDUContext* context, ContextSearchFlags flags = NoFlags, int depth = 0); /** * @brief Resolve the given declaration if it is an alias declaration. * * @param decl the declaration to resolve * @return :Declaration* decl if not an alias declaration, decl->aliasedDeclaration().data otherwise * DUChain must be read locked **/ static Declaration* resolveAliasDeclaration(Declaration* decl); static Declaration* declarationForName(const QString& name, const CursorInRevision& location, DUChainPointer context); static Declaration* declarationForName(const Python::NameAst* name, CursorInRevision location, DUChainPointer context); static QString getPythonExecutablePath(IProject* project); }; } #endif diff --git a/duchain/tests/pyduchaintest.cpp b/duchain/tests/pyduchaintest.cpp index 28c6ca69..3bfe2b3d 100644 --- a/duchain/tests/pyduchaintest.cpp +++ b/duchain/tests/pyduchaintest.cpp @@ -1,1800 +1,1801 @@ /***************************************************************************** * Copyright 2010 (c) Miquel Canes Gonzalez * * Copyright 2012 (c) Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include "duchaindebug.h" #include "pyduchaintest.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "astdefaultvisitor.h" #include "expressionvisitor.h" #include "contextbuilder.h" #include "astbuilder.h" #include "duchain/helpers.h" #include "kdevpythonversion.h" QTEST_MAIN(PyDUChainTest) using namespace KDevelop; using namespace Python; PyDUChainTest::PyDUChainTest(QObject* parent): QObject(parent) { assetsDir = QDir(DUCHAIN_PY_DATA_DIR); if (!assetsDir.cd("data")) { qFatal("Failed find data directory for test files. Aborting"); } testDir = QDir(testDirOwner.path()); qputenv("PYTHONPATH", assetsDir.absolutePath().toUtf8()); initShell(); } QList PyDUChainTest::FindPyFiles(QDir& rootDir) { QList foundfiles; rootDir.setFilter(QDir::Files | QDir::Dirs | QDir::NoDot | QDir::NoDotDot); rootDir.setNameFilters(QStringList() << "*.py"); // We only want .py files QDirIterator it(rootDir, QDirIterator::Subdirectories); while(it.hasNext()) { foundfiles.append(it.next()); } return foundfiles; } void PyDUChainTest::init() { QString currentTest = QString(QTest::currentTestFunction()); if (lastTest == currentTest) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Already prepared assets for " << currentTest << ", skipping"; return; } else { lastTest = currentTest; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Preparing assets for test " << currentTest; QDir assetModuleDir = QDir(assetsDir.absolutePath()); if (!assetModuleDir.cd(currentTest)) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Asset directory " << currentTest << " does not exist under " << assetModuleDir.absolutePath() << ". Skipping it."; return; } qCDebug(KDEV_PYTHON_DUCHAIN) << "Searching for python files in " << assetModuleDir.absolutePath(); QList foundfiles = FindPyFiles(assetModuleDir); QString correctionFileDir = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/correction_files", QStandardPaths::LocateDirectory); auto correctionFileUrl = QUrl(QDir::cleanPath(correctionFileDir + "/testCorrectionFiles/example.py")); foundfiles.prepend(correctionFileUrl.path()); for ( int i = 0; i < 2; i++ ) { // Parse each file twice, to ensure no parsing-order related bugs appear. // Such bugs will need separate unit tests and should not influence these. foreach(const QString filename, foundfiles) { qCDebug(KDEV_PYTHON_DUCHAIN) << "Parsing asset: " << filename; DUChain::self()->updateContextForUrl(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); } foreach(const QString filename, foundfiles) { DUChain::self()->waitForUpdate(IndexedString(filename), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); } while ( ICore::self()->languageController()->backgroundParser()->queuedCount() > 0 ) { // make sure to wait for all parsejobs to finish QTest::qWait(10); } } } void PyDUChainTest::initShell() { AutoTestShell::init(); TestCore* core = new TestCore(); core->initialize(KDevelop::Core::NoUi); auto doc_url = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/documentation_files/builtindocumentation.py"); qCDebug(KDEV_PYTHON_DUCHAIN) << doc_url; DUChain::self()->updateContextForUrl(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); ICore::self()->languageController()->backgroundParser()->parseDocuments(); DUChain::self()->waitForUpdate(IndexedString(doc_url), KDevelop::TopDUContext::AllDeclarationsContextsAndUses); DUChain::self()->disablePersistentStorage(); KDevelop::CodeRepresentation::setDiskChangesForbidden(true); } ReferencedTopDUContext PyDUChainTest::parse(const QString& code) { TestFile* testfile = new TestFile(code + "\n", "py", nullptr, testDir.absolutePath().append("/")); createdFiles << testfile; testfile->parse((TopDUContext::Features) (TopDUContext::ForceUpdate | TopDUContext::AST) ); testfile->waitForParsed(2000); if ( testfile->isReady() ) { Q_ASSERT(testfile->topContext()); m_ast = static_cast(testfile->topContext()->ast().data())->ast; return testfile->topContext(); } else Q_ASSERT(false && "Timed out waiting for parser results, aborting all tests"); return nullptr; } PyDUChainTest::~PyDUChainTest() { foreach ( TestFile* f, createdFiles ) { delete f; } testDir.rmdir(testDir.absolutePath()); } void PyDUChainTest::testMultiFromImport() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList a = ctx->findDeclarations(QualifiedIdentifier("a")); QList b = ctx->findDeclarations(QualifiedIdentifier("b")); QVERIFY(! a.isEmpty()); QVERIFY(! b.isEmpty()); QVERIFY(a.first()->abstractType()->toString().endsWith("int")); QVERIFY(b.first()->abstractType()->toString().endsWith("int")); } void PyDUChainTest::testMultiFromImport_data() { QTest::addColumn("code"); QTest::newRow("multiimport") << "import testMultiFromImport.i.localvar1\n" "import testMultiFromImport.i.localvar2\n" "a = testMultiFromImport.i.localvar1\n" "b = testMultiFromImport.i.localvar2\n"; } void PyDUChainTest::testRelativeImport() { QFETCH(QString, code); QFETCH(QString, token); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QList t = ctx->findDeclarations(QualifiedIdentifier(token)); QVERIFY(! t.isEmpty()); QVERIFY(t.first()->abstractType()->toString().endsWith(type)); } void PyDUChainTest::testRelativeImport_data() { QTest::addColumn("code"); QTest::addColumn("token"); QTest::addColumn("type"); QTest::newRow(".local") << "from testRelativeImport.m.sm1.go import i1" << "i1" << "int"; QTest::newRow(".init") << "from testRelativeImport.m.sm1.go import i2" << "i2" << "int"; QTest::newRow("..local") << "from testRelativeImport.m.sm1.go import i3" << "i3" << "int"; QTest::newRow("..init") << "from testRelativeImport.m.sm1.go import i4" << "i4" << "int"; QTest::newRow("..sub.local") << "from testRelativeImport.m.sm1.go import i5" << "i5" << "int"; QTest::newRow("..sub.init") << "from testRelativeImport.m.sm1.go import i6" << "i6" << "int"; } void PyDUChainTest::testImportFiles() { QString code = "import testImportFiles\nk = testImportFiles.fromInit()\np = testImportFiles.other.fromOther()"; ReferencedTopDUContext ctx = parse(code.toUtf8()); DUChainReadLocker lock; QVERIFY(ctx); auto k = ctx->findDeclarations(QualifiedIdentifier("k")); auto p = ctx->findDeclarations(QualifiedIdentifier("p")); QCOMPARE(k.size(), 1); QCOMPARE(p.size(), 1); QVERIFY(k.first()->abstractType()); QCOMPARE(k.first()->abstractType()->toString(), QString("fromInit")); QCOMPARE(p.first()->abstractType()->toString(), QString("fromOther")); } void PyDUChainTest::testCrashes() { QFETCH(QString, code); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); QVERIFY(! m_ast->body.isEmpty()); } void PyDUChainTest::testCrashes_data() { QTest::addColumn("code"); QTest::newRow("unicode_char") << "a = \"í\""; QTest::newRow("unicode escape char") << "print(\"\\xe9\")"; QTest::newRow("augassign") << "a = 3\na += 5"; QTest::newRow("delete") << "a = 3\ndel a"; QTest::newRow("double_comprehension") << "q = [[x for x in a] + [x for x in a] for y in b]"; QTest::newRow("for_else") << "for i in range(3): pass\nelse: pass"; QTest::newRow("for_while") << "while i < 4: pass\nelse: pass"; QTest::newRow("ellipsis") << "a[...]"; QTest::newRow("tuple_assign_unknown") << "foo = (unknown, unknown, unknown)"; QTest::newRow("for_assign_unknown") << "for foo, bar, baz in unknown: pass"; QTest::newRow("negative slice index") << "t = (1, 2, 3)\nd = t[-1]"; QTest::newRow("decorator_with_args") << "@foo('bar', 'baz')\ndef myfunc(): pass"; QTest::newRow("non_name_decorator") << "@foo.crazy_decorators\ndef myfunc(): pass"; QTest::newRow("static_method") << "class c:\n @staticmethod\n def method(): pass"; QTest::newRow("vararg_in_middle") << "def func(a, *b, c): pass\nfunc(1, 2, 3, 4, 5)"; QTest::newRow("whatever") << "for attr in updated:\n " " getattr.update"; QTest::newRow("return_outside_function") << "return 3"; QTest::newRow("return_context_outside_function") << "return [x for x in range(3)]"; QTest::newRow("paren_attrib_access") << "a = (xxx or yyy).zzz"; QTest::newRow("func_call") << "a = xxx.func(yyy.zzz)"; QTest::newRow("comprehension_attrib") << "a = [foo for foo in bar].baz"; QTest::newRow("comprehension_attrib2") << "a = [foo.bar for foo in bar]"; + QTest::newRow("lambda_cmpr_defarg") << "a = lambda foo=[b for b in (1, 2, 3)]: foo"; QTest::newRow("attrib") << "(sep or ' ').join(xxxx.capitalize() for xxxx in ssss.split(sep))"; QTest::newRow("attrib2") << "(sep or ' ').join(x.capitalize() for x in s.split(sep))"; QTest::newRow("attrib3") << "known_threads = {line.strip()}"; QTest::newRow("attrib4") << "known_threads = {line.strip() for line in [\"foo\"] if line.strip()}"; QTest::newRow("stacked_lambdas") << "l4 = lambda x = lambda y = lambda z=1 : z : y() : x()"; QTest::newRow("newline_attrib2") << "raise TypeError(\"argument should be a bound method, not {}\"\n" ".format(type(meth))) from None"; QTest::newRow("newline_attrib") << "some_instance \\\n" ". attr1 \\\n" ".funcfunc(argarg, arg2arg) \\\n" ".foo"; QTest::newRow("fancy generator context range") << "c1_list = sorted(letter for (letter, meanings) \\\n" "in ambiguous_nucleotide_values.iteritems() \\\n" "if set([codon[0] for codon in codons]).issuperset(set(meanings)))"; QTest::newRow("fancy class range") << "class SchemeLexer(RegexLexer):\n" " valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+'\n" "\n" " tokens = {\n" " 'root' : [\n" " # the comments - always starting with semicolon\n" " # and going to the end of the line\n" " (r';.*$', Comment.Single),\n" "\n" " # whitespaces - usually not relevant\n" " (r'\\s+', Text),\n" "\n" " # numbers\n" " (r'-?\\d+\\.\\d+', Number.Float),\n" " (r'-?\\d+', Number.Integer)\n" " ],\n" " }\n"; QTest::newRow("another fancy range") << "setup_args['data_files'] = [\n" " (os.path.dirname(os.path.join(install_base_dir, pattern)),\n" " [ f for f in glob.glob(pattern) ])\n" " for pattern in patterns\n" "]\n"; QTest::newRow("kwarg_empty_crash") << "def myfun(): return\ncheckme = myfun(kw=something)"; QTest::newRow("stacked_tuple_hang") << "tree = (1,(2,(3,(4,(5,'Foo')))))"; QTest::newRow("stacked_tuple_hang2") << "tree = (257," "(264," "(285," "(259," "(272," "(275," "(1, 'return')))))))"; QTest::newRow("very_large_tuple_hang") << "tree = " "(257," "(264," "(285," "(259," "(1, 'def')," "(1, 'f')," "(260, (7, '('), (8, ')'))," "(11, ':')," "(291," "(4, '')," "(5, '')," "(264," "(265," "(266," "(272," "(275," "(1, 'return')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302, (303, (304, (305, (2, '1'))))))))))))))))))," "(264," "(265," "(266," "(272," "(276," "(1, 'yield')," "(313," "(292," "(293," "(294," "(295," "(297," "(298," "(299," "(300," "(301," "(302," "(303, (304, (305, (2, '1'))))))))))))))))))," "(4, '')))," "(6, '')))))," "(4, '')," "(0, ''))))"; QTest::newRow("attribute_hang") << "s = \"123\"\n" "s = s.replace(u'ł', 'l').\\\n" "replace(u'ó', 'o').\\\n" "replace(u'ą', 'a').\\\n" "replace(u'ę', 'e').\\\n" "replace(u'ś', 's').\\\n" "replace(u'ż', 'z').\\\n" "replace(u'ź', 'z').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo').\\\n" "replace(u'ć', 'c').\\\n" "replace(u'ń', 'n').\\\n" "replace(u'б', 'b').\\\n" "replace(u'в', 'v').\\\n" "replace(u'г', 'g').\\\n" "replace(u'д', 'd').\\\n" "replace(u'ё', 'yo')\n"; QTest::newRow("function context range crash") << "def myfunc(arg):\n foo = 3 + \\\n[x for x in range(20)]"; QTest::newRow("decorator comprehension crash") << "@implementer_only(interfaces.ISSLTransport,\n" " *[i for i in implementedBy(tcp.Client)\n" " if i != interfaces.ITLSTransport])\n" "class Client(tcp.Client):\n" " pass\n"; QTest::newRow("comprehension_as_default_crash") << "def foo(bar = [item for (_, item) in items()]):\n return"; QTest::newRow("try_except") << "try: pass\nexcept: pass"; QTest::newRow("try_except_type") << "try: pass\nexcept FooException: pass"; QTest::newRow("try_except_type_as") << "try: pass\nexcept FooException as bar: pass"; QTest::newRow("import_missing") << "from this_does_not_exist import nor_does_this"; QTest::newRow("list_append_missing") << "foo = []\nfoo.append(missing)"; QTest::newRow("list_append_missing_arg") << "foo = []\nfoo.append()"; QTest::newRow("list_extend_missing") << "foo = []\nfoo.extend(missing)"; QTest::newRow("list_extend_missing_arg") << "foo = []\nfoo.extend()"; QTest::newRow("method_of_call_with_list_arg") << "class MyClass:\n" " def bar(self): pass\n" "def foo(x):\n" " return MyClass()\n" "foo([0]).bar()"; QTest::newRow("unpacked_dict_kwarg") << "def foo(arg): pass\nfoo(**{'arg': 2})"; QTest::newRow("negative_container_hints") << "class Evil:\n" " def aa(self, arg):\n" " \"\"\"! addsTypeOfArgContent ! -1\"\"\"\n" " def bb(self, arg):\n" " \"\"\"! addsTypeOfArg ! -2\"\"\"\n" " def cc(self, arg):\n" " \"\"\"! returnContentEqualsContentOf ! -3\"\"\"\n" "e = Evil()\n" "z = [e.aa(1), e.bb(2), e.cc(3)]"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("comprehension_in_fstring") << "def crash(): return f'expr={ {x: y for x, y in [(1, 2), ]}}'"; #endif QTest::newRow("comprehension_in_lambda") << "lambda foo: [bar for bar in foo]"; } void PyDUChainTest::testClassVariables() { ReferencedTopDUContext ctx = parse("class c():\n myvar = 3;\n def meth(self):\n print(myvar)"); QVERIFY(ctx.data()); DUChainWriteLocker lock(DUChain::lock()); CursorInRevision relevantPosition(3, 10); DUContext* c = ctx->findContextAt(relevantPosition); QVERIFY(c); int useIndex = c->findUseAt(relevantPosition); if ( useIndex != -1 ) { QVERIFY(useIndex < c->usesCount()); const Use* u = &(c->uses()[useIndex]); QVERIFY(!u->usedDeclaration(c->topContext())); } } void PyDUChainTest::testWarnNewNotCls() { QFETCH(QString, code); QFETCH(int, probs); ReferencedTopDUContext ctx = parse(code); DUChainReadLocker lock; QCOMPARE(ctx->problems().count(), probs); } void PyDUChainTest::testWarnNewNotCls_data() { QTest::addColumn("code"); QTest::addColumn("probs"); QTest::newRow("check_for_new_first_arg_cls") << "class c():\n def __new__(clf, other):\n pass" << 1; QTest::newRow("check_for_new_first_arg_cls_0") << "class c():\n def __new__(cls, other):\n pass" << 0; QTest::newRow("check_first_arg_class_self") << "class c():\n def test(seff, masik):\n pass" << 1; QTest::newRow("check_first_arg_class_self_0") << "class c():\n def test(self, masik):\n pass" << 0; } // this is actually for both binary and boolean operators void PyDUChainTest::testBinaryOperatorsUnsure() { QFETCH(QString, code); QFETCH(QString, type); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), type); } void PyDUChainTest::testBinaryOperatorsUnsure_data() { QTest::addColumn("code"); QTest::addColumn("type"); QTest::newRow("check_unsure_type_0") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3.5;\ny = 3;\ncheckme = x * y;" << "unsure (float, int)"; QTest::newRow("check_unsure_type_1") << "class c():\n def __mul__(self, other):\n return int();\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_2") << "class c():\n pass;\nx = c();\nx = 3;\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_3") << "class c():\n pass;\nclass d():\n pass;\nx = c();\nx = d();\ny = 3;\ncheckme = x * y;" << "int"; QTest::newRow("check_unsure_type_4") << "checkme = True or False" << "bool"; QTest::newRow("check_unsure_type_5") << "a = 'foo'; checkme = a or 'bar';" << "str"; QTest::newRow("check_unsure_type_6") << "class A(): pass\nclass B(): pass;\ncheckme = A() or B() or 42;" << "unsure (A, B, int)"; } void PyDUChainTest::testFlickering() { QFETCH(QStringList, code); QFETCH(int, before); QFETCH(int, after); TestFile f(code[0], "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx = f.topContext(); QVERIFY(ctx); DUChainWriteLocker lock(DUChain::lock()); int count = ctx->localDeclarations().size(); qDebug() << "Declaration count before: " << count; QVERIFY(count == before); lock.unlock(); f.setFileContents(code[1]); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ctx = f.topContext(); QVERIFY(ctx); lock.lock(); count = ctx->localDeclarations().size(); qDebug() << "Declaration count afterwards: " << count; QVERIFY(count == after); foreach(Declaration* dec, ctx->localDeclarations()) { qDebug() << dec->toString() << dec->range(); qDebug() << dec->uses().size(); } } void PyDUChainTest::testFlickering_data() { QTest::addColumn("code"); QTest::addColumn("before"); QTest::addColumn("after"); QTest::newRow("declaration_flicker") << ( QStringList() << "a=2\n" << "b=3\na=2\n" ) << 1 << 2; } void PyDUChainTest::testCannotOverwriteBuiltins() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; QList ds = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(!ds.isEmpty()); Declaration* d = ds.first(); QVERIFY(d); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testCannotOverwriteBuiltins_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("list_assign") << "class list(): pass\ncheckme = []\ncheckme.append(3)" << "list of int"; QTest::newRow("str_assign") << "str = 5; checkme = 'Foo'" << "str"; QTest::newRow("str_assign2") << "class Foo: pass\nstr = Foo; checkme = 'Foo'" << "str"; QTest::newRow("str_assign3") << "from testCannotOverwriteBuiltins.i import Foo as str\ncheckme = 'Foo'" << "str"; QTest::newRow("for") << "for str in [1, 2, 3]: pass\ncheckme = 'Foo'" << "str"; QTest::newRow("assert") << "assert isinstance(str, int)\ncheckme = 'Foo'" << "str"; QTest::newRow("assert2") << "assert isinstance(str, int)\ncheckme = 3" << "int"; QTest::newRow("can_have_custom") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open()" << "int"; QTest::newRow("can_have_custom2") << "from testCannotOverwriteBuiltins import mod\ncheckme = open().read()" << "str"; QTest::newRow("can_have_custom3") << "from testCannotOverwriteBuiltins import mod\ncheckme = mod.open().read()" << "mixed"; } void PyDUChainTest::testVarKWArgs() { ReferencedTopDUContext ctx = parse("def myfun(arg, *vararg, **kwarg):\n pass\n pass"); DUChainWriteLocker lock; QVERIFY(ctx); DUContext* func = ctx->findContextAt(CursorInRevision(1, 0)); QVERIFY(func); QVERIFY(! func->findDeclarations(QualifiedIdentifier("arg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("vararg")).isEmpty()); QVERIFY(! func->findDeclarations(QualifiedIdentifier("kwarg")).isEmpty()); QVERIFY(func->findDeclarations(QualifiedIdentifier("vararg")).first()->abstractType()->toString().startsWith("tuple")); QCOMPARE(func->findDeclarations(QualifiedIdentifier("kwarg")).first()->abstractType()->toString(), QString("dict of str : unknown")); } void PyDUChainTest::testSimple() { QFETCH(QString, code); QFETCH(int, decls); QFETCH(int, uses); ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVector< Declaration* > declarations = ctx->localDeclarations(); QCOMPARE(declarations.size(), decls); int usesCount = 0; foreach(Declaration* d, declarations) { usesCount += d->uses().size(); QVERIFY(d->abstractType()); } QCOMPARE(usesCount, uses); } void PyDUChainTest::testSimple_data() { QTest::addColumn("code"); QTest::addColumn("decls"); QTest::addColumn("uses"); QTest::newRow("assign") << "b = 2;" << 1 << 0; QTest::newRow("assign_str") << "b = 'hola';" << 1 << 0; QTest::newRow("op") << "a = 3; b = a+2;" << 2 << 1; QTest::newRow("bool") << "a = True" << 1 << 0; QTest::newRow("op") << "a = True and True;" << 1 << 0; } class AttributeRangeTestVisitor : public AstDefaultVisitor { public: bool found; KTextEditor::Range searchingForRange; QString searchingForIdentifier; void visitAttribute(AttributeAst* node) override { auto r = KTextEditor::Range(0, node->startCol, 0, node->endCol); qDebug() << "Found attr: " << r << node->attribute->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->attribute->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitAttribute(node); } void visitFunctionDefinition(FunctionDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found func: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; qDebug() << node->arguments->vararg << node->arguments->kwarg; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } if ( node->arguments->vararg ) { auto r = KTextEditor::Range(0, node->arguments->vararg->startCol, 0, node->arguments->vararg->startCol+node->arguments->vararg->argumentName->value.length()); qDebug() << "Found vararg: " << node->arguments->vararg->argumentName->value << r; if ( r == searchingForRange && node->arguments->vararg->argumentName->value == searchingForIdentifier ) { found = true; return; } } if ( node->arguments->kwarg ) { auto r = KTextEditor::Range(0, node->arguments->kwarg->startCol, 0, node->arguments->kwarg->startCol+node->arguments->kwarg->argumentName->value.length()); qDebug() << "Found kwarg: " << node->arguments->kwarg->argumentName->value << r; if ( r == searchingForRange && node->arguments->kwarg->argumentName->value == searchingForIdentifier ) { found = true; return; } } AstDefaultVisitor::visitFunctionDefinition(node); } void visitClassDefinition(ClassDefinitionAst* node) override { auto r = KTextEditor::Range(0, node->name->startCol, 0, node->name->endCol); qDebug() << "Found cls: " << r << node->name->value << ", looking for: " << searchingForRange << searchingForIdentifier; if ( r == searchingForRange && node->name->value == searchingForIdentifier ) { found = true; return; } AstDefaultVisitor::visitClassDefinition(node); } void visitImport(ImportAst* node) override { foreach ( const AliasAst* name, node->names ) { if ( name->name ) { qDebug() << "found import" << name->name->value << name->name->range(); } if ( name->name && name->name->value == searchingForIdentifier && name->name->range() == searchingForRange ) { found = true; return; } if ( name->asName ) { qDebug() << "found import" << name->asName->value << name->asName->range(); } if ( name->asName && name->asName->value == searchingForIdentifier && name->asName->range() == searchingForRange ) { found = true; return; } } } }; void PyDUChainTest::testRanges() { QFETCH(QString, code); QFETCH(int, expected_amount_of_variables); Q_UNUSED(expected_amount_of_variables); QFETCH(QStringList, column_ranges); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); QVERIFY(m_ast); for ( int i = 0; i < column_ranges.length(); i++ ) { int scol = column_ranges.at(i).split(",")[0].toInt(); int ecol = column_ranges.at(i).split(",")[1].toInt(); QString identifier = column_ranges.at(i).split(",")[2]; auto r = KTextEditor::Range(0, scol, 0, ecol); AttributeRangeTestVisitor* visitor = new AttributeRangeTestVisitor(); visitor->searchingForRange = r; visitor->searchingForIdentifier = identifier; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("attr_dot_name_hash", "Insufficiently magic hack", Continue); QCOMPARE(visitor->found, true); delete visitor; } } void PyDUChainTest::testRanges_data() { QTest::addColumn("code"); QTest::addColumn("expected_amount_of_variables"); QTest::addColumn("column_ranges"); QTest::newRow("attr_two_attributes") << "base.attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_binary") << "base.attr + base.attr" << 4 << ( QStringList() << "5,8,attr" << "17,20,attr" ); QTest::newRow("attr_same") << "aaa.aaa.aaa + aaa.aaa.aaa" << 6 << ( QStringList() << "4,6,aaa" << "8,10,aaa" << "18,20,aaa" << "22,24,aaa" ); QTest::newRow("attr_three_attributes") << "base.attr.subattr" << 3 << ( QStringList() << "5,8,attr" << "10,16,subattr" ); QTest::newRow("attr_functionCall") << "base.attr().subattr" << 3 << ( QStringList() << "5,8,attr" << "12,18,subattr" ); QTest::newRow("attr_stringSubscript") << "base.attr[\"a.b.c..de\"].subattr" << 3 << ( QStringList() << "5,8,attr" << "23,29,subattr" ); QTest::newRow("attr_functionCallWithArguments") << "base.attr(arg1, arg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" ); QTest::newRow("attr_functionCallWithArgument_withInner") << "base.attr(arg1.parg2).subattr" << 5 << ( QStringList() << "5,8,attr" << "22,28,subattr" << "15,19,parg2" ); QTest::newRow("attr_complicated") << "base.attr(arg1.arg2(arg4.arg5, [func(a.b)]).arg3(arg6.arg7)).subattr" << 5 << ( QStringList() << "5,8,attr" << "15,18,arg2" << "25,28,arg5" << "39,39,b" << "44,47,arg3" << "54,57,arg7" << "61,67,subattr"); QTest::newRow("attr_two_in_call") << "func(inst.aaa, inst.bbbb)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,bbbb" ); QTest::newRow("attr_two_in_call_same") << "func(inst.aaa, inst.aaaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,23,aaaa" ); QTest::newRow("attr_two_in_call_same2") << "func(inst.aaa, inst.aaa)" << 2 << ( QStringList() << "10,12,aaa" << "20,22,aaa" ); QTest::newRow("attr_of_string_slash") << "'/'.join(a)" << 1 << ( QStringList() << "4,7,join" ); QTest::newRow("attr_of_string_in_list") << "[\"*{0}*\".format(foo)]" << 1 << ( QStringList() << "9,14,format" ); QTest::newRow("attr_of_call_in_list") << "[foo().format(foo)]" << 1 << ( QStringList() << "7,12,format" ); QTest::newRow("attr_parentheses") << "(\"foo\" + \"foo\").capitalize()" << 1 << ( QStringList() << "16,25,capitalize" ); QTest::newRow("attr_commented_name") << "base.attr # attr" << 2 << ( QStringList() << "5,8,attr" ); QTest::newRow("attr_name_in_strings") << "'attr' + base['attr'].attr # attr" << 4 << ( QStringList() << "22,25,attr" ); QTest::newRow("attr_dot_hash_in_strings") << "'.foo#' + base['.#'].attr # attr" << 4 << ( QStringList() << "21,24,attr" ); QTest::newRow("attr_dot_name_hash") << "base['.attr#'].attr" << 4 << ( QStringList() << "15,18,attr" ); QTest::newRow("string_parentheses") << "(\"asdf\".join())" << 1 << ( QStringList() << "8,11,join" ); QTest::newRow("string_parentheses2") << "(\"asdf\").join()" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("string_parentheses3") << "(\"asdf\".join()).join()" << 2 << ( QStringList() << "8,11,join" << "16,19,join" ); QTest::newRow("string_parentheses4") << "(\"asdf\".join()+2).join()" << 2 << ( QStringList() << "8,11,join" << "18,21,join" ); QTest::newRow("string_parentheses_call") << "f(\"asdf\".join())" << 1 << ( QStringList() << "9,12,join" ); QTest::newRow("funcrange_def") << "def func(): pass" << 1 << ( QStringList() << "4,7,func" ); QTest::newRow("funcrange_spaces_def") << "def func(): pass" << 1 << ( QStringList() << "7,10,func" ); QTest::newRow("classdef_range") << "class cls(): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance") << "class cls(parent1, parent2): pass" << 1 << ( QStringList() << "6,8,cls" ); QTest::newRow("classdef_range_inheritance_spaces") << "class cls( parent1, parent2 ):pass" << 1 << ( QStringList() << "12,14,cls" ); QTest::newRow("vararg_kwarg") << "def func(*vararg, **kwargs): pass" << 2 << ( QStringList() << "10,16,vararg" << "20,26,kwargs" ); QTest::newRow("import") << "import sys" << 1 << ( QStringList() << "7,10,sys" ); QTest::newRow("import2") << "import i.localvar1" << 1 << ( QStringList() << "7,18,i.localvar1" ); QTest::newRow("import3") << "import sys as a" << 1 << ( QStringList() << "13,14,a" ); } class TypeTestVisitor : public AstDefaultVisitor { public: QString searchingForType; TopDUContextPointer ctx; bool found; void visitName(NameAst* node) override { if ( node->identifier->value != "checkme" ) return; QList decls = ctx->findDeclarations(QualifiedIdentifier(node->identifier->value)); if ( ! decls.length() ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "No declaration found for " << node->identifier->value; return; } Declaration* d = decls.last(); QVERIFY(d->abstractType()); qCDebug(KDEV_PYTHON_DUCHAIN) << "found: " << node->identifier->value << "is" << d->abstractType()->toString() << "should be" << searchingForType; if ( d->abstractType()->toString().replace("__kdevpythondocumentation_builtin_", "").startsWith(searchingForType) ) { found = true; return; } }; }; void PyDUChainTest::testTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QEXPECT_FAIL("tuple_func", "no suitable docstring hint", Continue); QEXPECT_FAIL("tuple_add", "not implemented", Continue); QEXPECT_FAIL("tuple_mul", "not implemented", Continue); QEXPECT_FAIL("return_builtin_iterator", "fake builtin iter()", Continue); QEXPECT_FAIL("parent_constructor_arg_type", "Not enough passes?", Continue); QEXPECT_FAIL("init_class_no_decl", "aliasing info lost", Continue); QEXPECT_FAIL("property_wrong", "visitCall uses declaration if no type", Continue); QEXPECT_FAIL("property_setter", "very basic property support", Continue); QCOMPARE(visitor->found, true); } void PyDUChainTest::testTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) QTest::newRow("annotate_decl") << "checkme: int" << "int"; QTest::newRow("annotate_assign") << "checkme: int = 3.5" << "unsure (float, int)"; #endif QTest::newRow("listtype") << "checkme = []" << "list"; QTest::newRow("listtype_func") << "checkme = list()" << "list"; QTest::newRow("listtype_with_contents") << "checkme = [1, 2, 3, 4, 5]" << "list of int"; QTest::newRow("listtype_extended") << "some_misc_var = []; checkme = some_misc_var" << "list"; QTest::newRow("dicttype") << "checkme = {}" << "dict"; QTest::newRow("dicttype_get") << "d = {0.4:5}; checkme = d.get(0)" << "int"; QTest::newRow("dicttype_func") << "checkme = dict()" << "dict"; QTest::newRow("dicttype_extended") << "some_misc_var = {}; checkme = some_misc_var" << "dict"; QTest::newRow("tuple") << "checkme = ()" << "tuple of ()"; QTest::newRow("tuple_func") << "checkme = tuple((1, 2.3))" << "tuple of (int, float)"; QTest::newRow("tuple_with_contents") << "checkme = 1, 2.3" << "tuple of (int, float)"; QTest::newRow("tuple_extended") << "some_misc_var = (); checkme = some_misc_var" << "tuple of ()"; QTest::newRow("tuple_max_display") << "checkme = 1,2,3,4,5,6" << "tuple of (int, int, int, int, int, ...)"; QTest::newRow("bool") << "checkme = True" << "bool"; QTest::newRow("float") << "checkme = 3.7" << "float"; QTest::newRow("int") << "checkme = 3" << "int"; QTest::newRow("str") << "checkme = \"foo\"" << "str"; QTest::newRow("bytes") << "checkme = b\"foo\"" << "bytes"; QTest::newRow("function_arg_scope") << "class Foo:\n" " a = 3\n" " def func(self, x=a):\n" " return x\n" "f = Foo()\n" "checkme = f.func()" << "int"; QTest::newRow("with") << "with open('foo') as f: checkme = f.read()" << "str"; QTest::newRow("arg_after_vararg") << "def func(x, y, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("arg_after_vararg_with_default") << "def func(x=5, y=3, *, z:int): return z\ncheckme = func()" << "int"; QTest::newRow("class_scope_end_inside") << "a = str()\nclass M:\n" " a = 2\n foo = a\n" "checkme = M().foo" << "int"; QTest::newRow("class_scope_end_outside") << "a = str()\nclass M:\n a = 2\ncheckme = a" << "str"; QTest::newRow("list_access_right_open_slice") << "some_list = []; checkme = some_list[2:]" << "list"; QTest::newRow("list_access_left_open_slice") << "some_list = []; checkme = some_list[:2]" << "list"; QTest::newRow("list_access_closed_slice") << "some_list = []; checkme = some_list[2:17]" << "list"; QTest::newRow("list_access_step") << "some_list = []; checkme = some_list[::2]" << "list"; QTest::newRow("list_access_singleItem") << "some_list = []; checkme = some_list[42]" << "mixed"; QTest::newRow("funccall_number") << "def foo(): return 3; \ncheckme = foo();" << "int"; QTest::newRow("funccall_string") << "def foo(): return 'a'; \ncheckme = foo();" << "str"; QTest::newRow("funccall_list") << "def foo(): return []; \ncheckme = foo();" << "list"; QTest::newRow("funccall_dict") << "def foo(): return {}; \ncheckme = foo();" << "dict"; QTest::newRow("funccall_no_return") << "def foo(): pass\ncheckme = foo()" << "None"; QTest::newRow("funccall_def_return") << "def foo(): return\ncheckme = foo()" << "None"; QTest::newRow("funccall_maybe_def_return") << "def foo():\n if False: return\n return 7\ncheckme = foo()" << "unsure (None, int)"; QTest::newRow("tuple1") << "checkme, foo = 3, \"str\"" << "int"; QTest::newRow("tuple2") << "foo, checkme = 3, \"str\"" << "str"; QTest::newRow("tuple2_negative_index") << "foo = (1, 2, 'foo')\ncheckme = foo[-1]" << "str"; QTest::newRow("tuple_type") << "checkme = 1, 2" << "tuple"; QTest::newRow("tuple_rhs_unpack") << "foo = 1, 2.5\nbar = 1, *foo, 2\ncheckme = bar[2]" << "float"; QTest::newRow("dict_iteritems") << "d = {1:2, 3:4}\nfor checkme, k in d.iteritems(): pass" << "int"; QTest::newRow("enumerate_key") << "d = [str(), str()]\nfor checkme, value in enumerate(d): pass" << "int"; QTest::newRow("enumerate_value") << "d = [str(), str()]\nfor key, checkme in enumerate(d): pass" << "str"; QTest::newRow("dict_enumerate") << "d = {1:2, 3:4}\nfor key, checkme in enumerate(d.values()): pass" << "int"; QTest::newRow("dict_assign_twice") << "d = dict(); d[''] = 0; d = dict(); d[''] = 0; checkme = d" << "unsure (dict of str : int, dict)"; QTest::newRow("class_method_import") << "class c:\n attr = \"foo\"\n def m():\n return attr;\n return 3;\ni=c()\ncheckme=i.m()" << "int"; QTest::newRow("getsListDocstring") << "foo = [1, 2, 3]\ncheckme = foo.reverse()" << "list of int"; QTest::newRow("str_iter") << "checkme = [char for char in 'Hello, world!']" << "list of str"; QTest::newRow("str_subscript") << "checkme = 'Hello, world!'[0]" << "str"; QTest::newRow("bytes_iter") << "checkme = [byte for byte in b'Hello, world!']" << "list of int"; QTest::newRow("bytes_subscript") << "checkme = b'Hello, world!'[0]" << "int"; QTest::newRow("fromAssertIsinstance") << "class c(): pass\ncheckme = mixed()\nassert isinstance(checkme, c)\n" << "c"; QTest::newRow("fromAssertIsinstanceInvalid") << "class c(): pass\ncheckme = mixed()\nassert isinstance(c, checkme)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid2") << "class c(): pass\ncheckme = mixed()\nassert isinstance(D, c)\n" << "mixed"; QTest::newRow("fromAssertIsinstanceInvalid3") << "checkme = int()\nassert isinstance(checkme, X)\n" << "int"; QTest::newRow("fromAssertIsinstanceInvalid4") << "checkme = int()\nassert isinstance(checkme)\n" << "int"; QTest::newRow("fromAssertType") << "class c(): pass\ncheckme = mixed()\nassert type(checkme) == c\n" << "c"; QTest::newRow("fromIfType") << "class c(): pass\ncheckme = mixed()\nif type(checkme) == c: pass\n" << "c"; QTest::newRow("fromIfIsinstance") << "class c(): pass\ncheckme = mixed()\nif isinstance(checkme, c): pass\n" << "c"; QTest::newRow("diff_local_classattr") << "class c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; QTest::newRow("diff_local_classattr2") << "local=3\nclass c(): attr = 1\ninst=c()\ncheckme = c.local" << "mixed"; QTest::newRow("diff_local_classattr3") << "attr=3.5\nclass c(): attr = 1\ninst=c()\ncheckme = c.attr" << "int"; // QTest::newRow("class_method_self") << "class c:\n def func(checkme, arg, arg2):\n pass\n" << "c"; // QTest::newRow("funccall_dict") << "def foo(): return foo; checkme = foo();" << (uint) IntegralType::TypeFunction; // With only one subbed value we get a FormattedValue node QTest::newRow("fstring_formattedvalue") << "name = 'Jim'; checkme = f'{name}'" << "str"; // Otherwise a JoinedString, with FormattedValues as children. QTest::newRow("fstring_joinedstring") << "name = 'Jim'; checkme = f'Hello, {name}! Your name is {name}.'" << "str"; QTest::newRow("tuple_simple") << "mytuple = 3, 5.5\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_simple2") << "mytuple = 3, 5.5\nfoobar, checkme = mytuple" << "float"; QTest::newRow("tuple_simple3") << "mytuple = 3, 5.5, \"str\", 3, \"str\"\na, b, c, d, checkme = mytuple" << "str"; QTest::newRow("tuple_single") << "checkme = 4," << "tuple"; QTest::newRow("tuple_single2") << "checkme, = 4," << "int"; QTest::newRow("tuple_single3") << "mytuple = 4,\ncheckme, = mytuple" << "int"; QTest::newRow("tuple_ext_unpack") << "mytuple = 3, 5.5\nfoobar, *starred, checkme = mytuple" << "float"; QTest::newRow("tuple_ext_unpack2") << "mytuple = 3, 5.5\nfoobar, *checkme, another = mytuple" << "list"; QTest::newRow("tuple_ext_unpack3") << "mytuple = 3, 5.5\nfoobar, *checkme = mytuple" << "list of float"; QTest::newRow("tuple_ext_unpack4") << "mytuple = 3, 5.5\n*checkme, = mytuple" << "list of unsure (int, float)"; QTest::newRow("tuple_nested") << "mytuple = 3, ('foo', 5.5)\ncheckme, foobar = mytuple" << "int"; QTest::newRow("tuple_nested2") << "mytuple = 3, ('foo', 5.5)\nfoobar, (checkme, other) = mytuple" << "str"; QTest::newRow("tuple_nested3") << "mytuple = ((7, 'foo'), 5.5), 3\n((baz, checkme), other), foo = mytuple" << "str"; QTest::newRow("tuple_nested_ext") << "mytuple = (2, ('foo', 'bar', 6), 7)\na, (b, *checkme, c), *d = mytuple" << "list of str"; QTest::newRow("tuple_multi_assign") << "mytuple = 2, 'foo'\ncheckme = a = mytuple" << "tuple"; QTest::newRow("tuple_multi_assign2") << "mytuple = 2, 'foo'\ncheckme, a = b = mytuple" << "int"; QTest::newRow("list_unpack") << "mylist = [1, 2, 3]\ncheckme, b, c = mylist" << "int"; QTest::newRow("list_unpack2") << "mylist = [1, 'x', 3]\ncheckme, b, c = mylist" << "unsure (int, str)"; QTest::newRow("list_ext_unpack") << "mylist = [1, 2, 3]\n*checkme, foo = mylist" << "list of int"; QTest::newRow("list_ext_unpack2") << "mylist = [1, 'x', 3]\n*checkme, foo = mylist" << "list of unsure (int, str)"; QTest::newRow("if_expr_sure") << "checkme = 3 if 7 > 9 else 5" << "int"; QTest::newRow("unary_op") << "checkme = -42" << "int"; QTest::newRow("tuple_funcret") << "def myfun(): return 3, 5\ncheckme, a = myfun()" << "int"; QTest::newRow("tuple_funcret2") << "def myfun():\n t = 3, 5\n return t\ncheckme, a = myfun()" << "int"; QTest::newRow("yield") << "def myfun():\n yield 3\ncheckme = myfun()" << "list of int"; QTest::newRow("yield_twice") << "def myfun():\n yield 3\n yield 'foo'\ncheckme = myfun()" << "list of unsure (int, str)"; // this is mostly a check that it doesn't crash QTest::newRow("yield_return") << "def myfun():\n return 3\n yield 'foo'\ncheckme = myfun()" << "unsure (int, list of str)"; QTest::newRow("lambda") << "x = lambda t: 3\ncheckme = x()" << "int"; QTest::newRow("lambda_failure") << "x = lambda t: 3\ncheckme = t" << "mixed"; QTest::newRow("function_arg_tuple") << "def func(*arg):\n foo, bar = arg\n return bar\ncheckme = func(3, 5)" << "int"; QTest::newRow("function_arg_tuple2") << "def func(*arg):\n return arg[-1]\ncheckme = func(3, \"Foo\")" << "str"; QTest::newRow("tuple_indexaccess") << "t = 3, 5.5\ncheckme = t[0]" << "int"; QTest::newRow("tuple_indexaccess2") << "t = 3, 5.5\ncheckme = t[1]" << "float"; QTest::newRow("tuple_indexaccess3") << "t = 3, 4\ncheckme = t[1]" << "int"; QTest::newRow("tuple_indexaccess4") << "t = 3, 4.5\ncheckme = t[2]" << "unsure (int, float)"; QTest::newRow("tuple_indexaccess_neg") << "t = 3, 4.5; checkme = t[-1]" << "float"; QTest::newRow("tuple_indexaccess_neg2") << "t = 3, 4.5; checkme = t[-2]" << "int"; QTest::newRow("tuple_indexaccess_neg3") << "t = 3, 4.5; checkme = t[-3]" << "unsure (int, float)"; QTest::newRow("tuple_slice") << "t = 3, 'q', 4.5; checkme = t[-3: 2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_normal") << "t = 1, 2.3, 'a', {}; checkme = t[1:3]" << "tuple of (float, str)"; QTest::newRow("tuple_slice_defstart") << "t = 1, 2.3, 'a', {}; checkme = t[:3]" << "tuple of (int, float, str)"; QTest::newRow("tuple_slice_defstop") << "t = 1, 2.3, 'a', {}; checkme = t[1:]" << "tuple of (float, str, dict)"; QTest::newRow("tuple_slice_defboth") << "t = 1, 2.3, 'a', {}; checkme = t[:]" << "tuple of (int, float, str, dict)"; QTest::newRow("tuple_slice_step") << "t = 1, 2.3, 'a', {}; checkme = t[0:3:2]" << "tuple of (int, str)"; QTest::newRow("tuple_slice_reverse") << "t = 1, 2.3, 'a', {}; checkme = t[3:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstart") << "t = 1, 2.3, 'a', {}; checkme = t[:1:-1]" << "tuple of (dict, str)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[2::-1]" << "tuple of (str, float, int)"; QTest::newRow("tuple_slice_revstop") << "t = 1, 2.3, 'a', {}; checkme = t[::-1]" << "tuple of (dict, str, float, int)"; QTest::newRow("tuple_slice_no_elems") << "t = 1, 2.3, 'a', {}; checkme = t[1:1]" << "tuple of ()"; // TODO unsure-tuples. QTest::newRow("tuple_slice_not_literal") << "n = 2; t = 1, 2.3, 'a', {}; checkme = t[0:n]" << "tuple of ()"; // These are allowed, for whatever reason. QTest::newRow("tuple_slice_past_range") << "t = 1, 2.3; checkme = t[-999999999:8888888888]" << "tuple of (int, float)"; QTest::newRow("tuple_slice_wrong_direction") << "t = 1, 2.3, 'a'; checkme = t[0:3:-1]" << "tuple of ()"; // This isn't. QTest::newRow("tuple_slice_zero_step") << "t = 1, 2.3; checkme = t[::0]" << "tuple of ()"; QTest::newRow("tuple_add") << "t, u = (3,), ('q', 4.5); checkme = t + u" << "tuple of (int, str, float)"; QTest::newRow("tuple_mul") << "t = 3, 4.5; checkme = t * 2" << "tuple of (int, float, int, float)"; QTest::newRow("dict_unsure") << "t = dict(); t = {3: str()}\ncheckme = t[1].capitalize()" << "str"; QTest::newRow("unsure_attr_access") << "d = str(); d = 3; checkme = d.capitalize()" << "str"; QTest::newRow("class_create_var") << "class c: pass\nd = c()\nd.foo = 3\ncheckme = d.foo" << "int"; QTest::newRow("tuple_loop") << "t = [(1, \"str\")]\nfor checkme, a in t: pass" << "int"; QTest::newRow("no_hints_type") << "def myfun(arg): arg = 3; return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("hints_type") << "def myfun(arg): return arg\ncheckme = myfun(3)" << "int"; QTest::newRow("args_type") << "def myfun(*args): return args[0]\ncheckme = myfun(3)" << "int"; QTest::newRow("kwarg_type") << "def myfun(**kwargs): return kwargs['a']\ncheckme = myfun(a=3)" << "int"; QTest::newRow("dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\ncheckme = foo(**{'a': 12})" << "int"; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("dict_norm_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2)" << "unsure (int, float)"; QTest::newRow("multi_dict_kwarg_type") << "def foo(**kwargs): return kwargs['']\n" "checkme = foo(**{'a': 12}, b=1.2, **{'c': ''})" << "unsure (int, float, str)"; #endif QTest::newRow("named_arg_type") << "def myfun(arg): return arg\ncheckme = myfun(arg=3)" << "int"; QTest::newRow("arg_args_type") << "def myfun(arg, *args): return args[0]\n" "checkme = myfun(3, str())" << "str"; QTest::newRow("arg_kwargs_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(12, a=str())" << "str"; QTest::newRow("named_kwargs_type_1") << "def myfun(arg, **kwargs): return arg\n" "checkme = myfun(arg=12, a=str())" << "int"; QTest::newRow("named_kwargs_type_2") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(arg=12, a=str())" << "str"; QTest::newRow("kwargs_named_type") << "def myfun(arg, **kwargs): return kwargs['a']\n" "checkme = myfun(a=str(), arg=12)" << "str"; QTest::newRow("varied_args_type_1") << "def myfun(arg, *args, **kwargs): return arg\n" "checkme = myfun(1, 1.5, a=str())" << "int"; QTest::newRow("varied_args_type_2") << "def myfun(arg, *args, **kwargs): return args[0]\n" "checkme = myfun(1, 1.5, a=str())" << "float"; QTest::newRow("varied_args_type_3") << "def myfun(arg, *args, **kwargs): return kwargs['a']\n" "checkme = myfun(1, 1.5, a=str())" << "str"; QTest::newRow("nested_arg_name_type") << "def foo(xxx):\n" " def bar(xxx): return xxx\n" " return bar('test')\n" "checkme = foo(10)\n" << "str"; QTest::newRow("method_args_type_1") << "class MyClass:\n" " def method(self, arg): return self\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("method_args_type_2") << "class MyClass:\n" " def method(self, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("clsmethod_args_type_1") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return cls\n" "checkme = MyClass().method(12)" << "MyClass"; QTest::newRow("clsmethod_args_type_2") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_args_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg): return arg\n" "checkme = MyClass().method(12)" << "int"; QTest::newRow("staticmethod_vararg_type") << "class MyClass:\n" " @staticmethod\n" " def method(arg, *args): return args[0]\n" "checkme = MyClass().method(12, 2.5)" << "float"; QTest::newRow("method_explicit_self") << "class MyClass:\n" " def method(self, arg): return arg\n" "instance = MyClass()\n" "checkme = MyClass.method(instance, 12)" << "int"; QTest::newRow("method_vararg_explicit_self") << "class MyClass:\n" " def foo(self, arg, *args): return args[0]\n" "mc = MyClass()\n" "checkme = MyClass.foo(mc, 'str', 3, 4.5)" << "int"; QTest::newRow("clsmethod_explicit_self") << "class MyClass:\n" " @classmethod\n" " def method(cls, arg1, arg2): return arg2\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "int"; QTest::newRow("staticmethod_explicit_self") << "class MyClass:\n" " @staticmethod\n" " def method(arg1, arg2): return arg1\n" "instance = MyClass()\n" "checkme = MyClass.method('a', 12)" << "str"; QTest::newRow("parent_constructor_arg_type") << "class Base:\n" // https://bugs.kde.org/show_bug.cgi?id=369364 " def __init__(self, foo):\n" " self.foo = foo\n" "class Derived(Base):\n" " def __init__(self, foo):\n" " Base.__init__(self, foo)\n" "instance = Derived('string')\n" "checkme = instance.foo" << "str"; QTest::newRow("nested_class_self_inside") << "class Foo:\n" " def foo(self):\n" " class Bar:\n" " def bar(self): return self\n" " return Bar().bar()\n" "checkme = Foo().foo()\n" << "Foo::foo::Bar"; QTest::newRow("nested_class_self_after") << "class Foo:\n" " class Bar: pass\n" " def foo(self): return self\n" "checkme = Foo().foo()\n" << "Foo"; QTest::newRow("tuple_unsure") << "q = (3, str())\nq=(str(), 3)\ncheckme, _ = q" << "unsure (int, str)"; QTest::newRow("custom_iterable") << "class Gen2:\n" " def __iter__(self): return self\n" " def __next__(self): return 'blah'\n" "for checkme in Gen2(): pass" << "str"; QTest::newRow("separate_iterator") << "class Foo:\n" " def __iter__(self): return Bar()\n" " def __next__(self): return 'blah'\n" // Not used (or shouldn't be!) "class Bar:\n" " def __next__(self): return {1}\n" "checkme = [a for a in Foo()]" << "list of set of int"; QTest::newRow("return_builtin_iterator") << "class Gen2:\n" " contents = [1, 2, 3]\n" " def __iter__(self): return iter(Gen2.contents)\n" "for checkme in Gen2(): pass" << "int"; QTest::newRow("init_class") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "checkme = Foo()\n" << "Foo"; QTest::newRow("init_class_no_decl") << "class Foo:\n" " def __init__(self): pass\n" " def __call__(self): return 1.5\n" "a = [Foo]\n" "checkme = a[0]()\n" << "Foo"; QTest::newRow("call_class") << "class Foo:\n" " def __call__(self):\n" " return 0\n" "f = Foo()\n" "checkme = f()\n" << "int"; QTest::newRow("call_class_no_decl") << "class Foo:\n" " def __call__(self): return 1.5\n" "a = [Foo()]\n" "checkme = a[0]()" << "float"; QTest::newRow("classmethod") << "class Foo:\n" " @classmethod\n" " def foo(cls):\n" " k = cls()\n" " return k\n" "f = Foo.foo()\n" "checkme = f\n" << "Foo"; QTest::newRow("property_getter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" "checkme = Foo().bar" << "int"; QTest::newRow("property_wrong") << "class Foo:\n" " @property\n" " def bar(self): return True\n" "checkme = Foo().bar()" << "mixed"; QTest::newRow("property_setter") << "class Foo:\n" " @property\n" " def bar(self): return 35\n" " @bar.setter\n" " def bar(self, value): return 18.3\n" // Return should be ignored "checkme = Foo().bar" << "int"; QTest::newRow("tuple_listof") << "l = [(1, 2), (3, 4)]\ncheckme = l[1][0]" << "int"; QTest::newRow("getitem") << "class c:\n def __getitem__(self, slice): return 3.14\na = c()\ncheckme = a[2]" << "float"; QTest::newRow("constructor_type_deduction") << "class myclass:\n" "\tdef __init__(self, param): self.foo=param\n" "checkme = myclass(3).foo" << "int"; QTest::newRow("simpe_type_deduction") << "def myfunc(arg): return arg\n" "checkme = myfunc(3)" << "int"; QTest::newRow("functionCall_functionArg_part1") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "checkme = f1()" << "str"; QTest::newRow("functionCall_functionArg_part2") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = identity(getstr)\n" "checkme = f1()\n" << "str"; QTest::newRow("functionCall_functionArg_full") << "def getstr(): return \"foo\"\n" "def identity(f): return f\n" "f1 = getstr\n" "f2 = identity(getstr)\n" "a = getstr()\n" "b = f1()\n" "checkme = f2()\n" << "str"; QTest::newRow("vararg_before_other_args") << "def myfun(a, b, *z, x): return z[0]\n" "checkme = myfun(False, False, 1, x = False)" << "int"; QTest::newRow("vararg_before_other_args2") << "def myfun(a, b, *z, x): return z[3]\n" "checkme = myfun(False, False, 1, 2, 3, \"str\", x = False)" << "str"; QTest::newRow("vararg_constructor") << "class myclass():\n" " def __init__(self, *arg): self.prop = arg[0]\n" "obj = myclass(3, 5); checkme = obj.prop" << "int"; QTest::newRow("declaration_order_var") << "aaa = 2\n" "checkme = aaa" << "int"; QTest::newRow("declaration_order_var2") << "checkme = aaa\n" "aaa = 2" << "mixed"; QTest::newRow("declaration_order_func_defarg") << "aaa = 2\n" "def foo(x=aaa): return x\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_defarg2") << "def foo(x=aaa): return x\n" "aaa = 2\n" "checkme = foo()" << "mixed"; QTest::newRow("declaration_order_func_body") << "aaa = 2\n" "def foo(): return aaa\n" "checkme = foo()" << "int"; QTest::newRow("declaration_order_func_body2") << "def foo(): return aaa\n" "aaa = 2\n" "checkme = foo()" << "int"; QTest::newRow("global_variable") << "a = 3\n" "def f1():\n" " global a\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_variable2") << "a = 3\n" "def f1():\n" " global a\n" " a = \"str\"\n" " return a\n" "checkme = f1()\n" << "str"; QTest::newRow("global_scope_variable") << "a = 3\n" "def f1():\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("global_no_toplevel_dec") << "def f1():\n" " global a\n a = 3\n" " return a\n" "checkme = f1()\n" << "int"; QTest::newRow("top_level_vs_class_attr") << "var = 3\n" "class MyClass:\n" " var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("top_level_vs_instance_attr") << "var = 3\n" "class MyClass:\n" " def __init__(self): self.var = 'str'\n" " def f1(): return var\n" "checkme = MyClass.f1()" << "int"; QTest::newRow("intermediate_vs_class/instance_attrs") << "def func():\n" " aa, bb = 3, 4\n" " class Foo:\n" " aa = 'a'\n" " def __init__(self):\n" " self.bb = 'b'\n" " def foo(self):\n" " return aa, bb\n" " return Foo().foo()\n" "checkme = func()" << "tuple of (int, int)"; QTest::newRow("top_level_vs_nested_class_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " aaa = 1\n" " class Bar:\n" " bbb = 2\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // Bar.bbb is visible here, Foo.aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; QTest::newRow("top_level_vs_nested_instance_attrs") << "aaa = 'foo'\n" "bbb = 'bar'\n" "class Foo:\n" " def __init__(self): self.aaa = 1\n" " class Bar:\n" " def __init__(self): self.bbb = 1\n" " def foo(self, ccc=aaa, ddd=bbb):\n" // self.bbb is visible here, Foo().aaa isn't. " return ccc, ddd\n" "checkme = Foo().Bar().foo()\n" << "tuple of (str, int)"; } typedef QPair pair; void PyDUChainTest::testImportDeclarations() { QFETCH(QString, code); QFETCH(QStringList, expectedDecls); QFETCH(bool, shouldBeAliased); ReferencedTopDUContext ctx = parse(code.toUtf8()); QVERIFY(ctx); QVERIFY(m_ast); DUChainReadLocker lock(DUChain::lock()); foreach ( const QString& expected, expectedDecls ) { bool found = false; QString name = expected; QList decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); qCDebug(KDEV_PYTHON_DUCHAIN) << "FOUND DECLARATIONS:"; foreach ( const pair& current, decls ) { qCDebug(KDEV_PYTHON_DUCHAIN) << current.first->toString() << current.first->identifier().identifier().byteArray() << name; } foreach ( const pair& current, decls ) { if ( ! ( current.first->identifier().identifier().byteArray() == name ) ) continue; qCDebug(KDEV_PYTHON_DUCHAIN) << "Found: " << current.first->toString() << " for " << name; AliasDeclaration* isAliased = dynamic_cast(current.first); if ( isAliased && shouldBeAliased ) { found = true; // TODO fixme } else if ( ! isAliased && ! shouldBeAliased ) { found = true; } } QVERIFY(found); } } void PyDUChainTest::testProblemCount() { QFETCH(QString, code); QFETCH(int, problemsCount); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock; QEXPECT_FAIL("fstring_visit_inside", "Ranges are broken so we don't visit the expression", Continue); QCOMPARE(ctx->problems().size(), problemsCount); } void PyDUChainTest::testProblemCount_data() { QTest::addColumn("code"); QTest::addColumn("problemsCount"); QTest::newRow("list_comp") << "[foo for foo in range(3)]" << 0; QTest::newRow("list_comp_wrong") << "[bar for foo in range(3)]" << 1; QTest::newRow("list_comp_staticmethod") << "class A:\n @staticmethod\n def func(cls):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_decorator") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [a for a in [1, 2, 3]]" << 0; QTest::newRow("list_comp_other_wrong") << "def decorate(): pass\nclass A:\n @decorate\n def func(self):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("list_comp_staticmethod_wrong") << "class A:\n @staticmethod\n def func(cls):\n" " [x for a in [1, 2, 3]]" << 1; QTest::newRow("misplaced_return_plain") << "return" << 1; QTest::newRow("misplaced_return_value") << "return 15" << 1; QTest::newRow("misplaced_return_class") << "class A:\n return 25" << 1; QTest::newRow("correct_return") << "def foo():\n return" << 0; QTest::newRow("lambda_argument_outside") << "def bar():\n lambda foo: 3\n foo" << 1; QTest::newRow("use_found_at_decl") << "foo = 3" << 0; QTest::newRow("fstring_visit_inside") << "checkme = f'{name}'" << 1; } void PyDUChainTest::testImportDeclarations_data() { QTest::addColumn("code"); QTest::addColumn("expectedDecls"); QTest::addColumn("shouldBeAliased"); QTest::newRow("from_import") << "from testImportDeclarations.i import checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("import") << "import testImportDeclarations.i" << ( QStringList() << "testImportDeclarations" ) << false; QTest::newRow("import_as") << "import testImportDeclarations.i as checkme" << ( QStringList() << "checkme" ) << false; QTest::newRow("from_import_as") << "from testImportDeclarations.i import checkme as checkme" << ( QStringList() << "checkme" ) << true; QTest::newRow("from_import_missing") << "from testImportDeclarations.i import missing as checkme" << ( QStringList() ) << true; } typedef QPair p; void PyDUChainTest::testAutocompletionFlickering() { TestFile f("foo = 3\nfoo2 = 2\nfo", "py"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx1 = f.topContext(); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx1); QList

decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); QList declIds; foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); f.setFileContents("foo = 3\nfoo2 = 2\nfoo"); f.parse(TopDUContext::ForceUpdate); f.waitForParsed(500); ReferencedTopDUContext ctx2 = f.topContext(); QVERIFY(ctx2); lock.lock(); QList

decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@1: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); qDebug() << "========================="; TestFile g("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfo", "py"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx1 = g.topContext(); lock.lock(); QVERIFY(ctx1); decls1 = ctx1->allDeclarations(CursorInRevision::invalid(), ctx1->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx1->topContext()); declIds.clear(); foreach ( p d, decls1 ) { declIds << d.first->id(); } lock.unlock(); g.setFileContents("def func():\n\tfoo = 3\n\tfoo2 = 2\n\tfoo"); g.parse(TopDUContext::ForceUpdate); g.waitForParsed(500); ctx2 = g.topContext(); QVERIFY(ctx2); lock.lock(); decls2 = ctx2->allDeclarations(CursorInRevision::invalid(), ctx2->topContext(), false).first().first->internalContext() ->allDeclarations(CursorInRevision::invalid(), ctx2->topContext()); foreach ( p d2, decls2 ) { qCDebug(KDEV_PYTHON_DUCHAIN) << "@2: " << d2.first->toString() << "::" << d2.first->id().hash() << "<>" << declIds.first().hash(); QVERIFY(d2.first->id() == declIds.first()); declIds.removeFirst(); } lock.unlock(); } void PyDUChainTest::testFunctionHints() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testFunctionHints_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("func_return_type") << "def myfun(arg) -> int: pass\ncheckme = myfun(\"3\")" << "unsure (None, int)"; QTest::newRow("argument_type") << "def myfun(arg : int): return arg\ncheckme = myfun(foobar)" << "int"; QTest::newRow("argument_type_only_if_typeof") << "def myfun(arg : 3): return arg\ncheckme = myfun(foobar)" << "mixed"; } void PyDUChainTest::testHintedTypes() { QFETCH(QString, code); QFETCH(QString, expectedType); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainWriteLocker lock; QList< Declaration* > decls = ctx->findDeclarations(KDevelop::Identifier("checkme")); QVERIFY(! decls.isEmpty()); Declaration* d = decls.first(); QVERIFY(d->abstractType()); QCOMPARE(d->abstractType()->toString(), expectedType); } void PyDUChainTest::testHintedTypes_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("simple_hint") << "def myfunc(x): return x\ncheckme = myfunc(3)" << "int"; QTest::newRow("hint_unsure") << "def myfunc(x): return x\nmyfunc(3.5)\ncheckme = myfunc(3)" << "unsure (float, int)"; QTest::newRow("unsure_attribute") << "def myfunc(x): return x.capitalize()\nmyfunc(3.5)\ncheckme = myfunc(str())" << "str"; } void PyDUChainTest::testOperators() { QFETCH(QString, code); QFETCH(QString, expectedType); code.prepend("from testOperators.example import *\n\n"); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); TypeTestVisitor* visitor = new TypeTestVisitor(); visitor->ctx = TopDUContextPointer(ctx.data()); visitor->searchingForType = expectedType; visitor->visitCode(m_ast.data()); QVERIFY(visitor->found); } void PyDUChainTest::testOperators_data() { QTest::addColumn("code"); QTest::addColumn("expectedType"); QTest::newRow("add") << "checkme = Example() + Example()" << "Add"; QTest::newRow("sub") << "checkme = Example() - Example()" << "Sub"; QTest::newRow("mul") << "checkme = Example() * Example()" << "Mul"; QTest::newRow("floordiv") << "checkme = Example() // Example()" << "Floordiv"; QTest::newRow("mod") << "checkme = Example() % Example()" << "Mod"; QTest::newRow("pow") << "checkme = Example() ** Example()" << "Pow"; QTest::newRow("lshift") << "checkme = Example() << Example()" << "Lshift"; QTest::newRow("rshift") << "checkme = Example() >> Example()" << "Rshift"; QTest::newRow("and") << "checkme = Example() & Example()" << "And"; QTest::newRow("xor") << "checkme = Example() ^ Example()" << "Xor"; QTest::newRow("or") << "checkme = Example() | Example()" << "Or"; } void PyDUChainTest::testFunctionArgs() { ReferencedTopDUContext ctx = parse("def ASDF(arg1, arg2):\n" " arg1 = arg2"); DUChainWriteLocker lock(DUChain::lock()); QVERIFY(ctx); QVERIFY(m_ast); // dumpDUContext(ctx); QCOMPARE(ctx->childContexts().size(), 2); DUContext* funcArgCtx = ctx->childContexts().first(); QCOMPARE(funcArgCtx->type(), DUContext::Function); QCOMPARE(funcArgCtx->localDeclarations().size(), 2); QVERIFY(!funcArgCtx->owner()); Python::FunctionDeclaration* decl = dynamic_cast( ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext()).first().first); QVERIFY(decl); QCOMPARE(decl->type()->arguments().length(), 2); qDebug() << decl->type()->arguments().length() << 2; DUContext* funcBodyCtx = ctx->childContexts().last(); QCOMPARE(funcBodyCtx->type(), DUContext::Other); QVERIFY(funcBodyCtx->owner()); QVERIFY(funcBodyCtx->localDeclarations().isEmpty()); } void PyDUChainTest::testInheritance() { QFETCH(QString, code); QFETCH(int, expectedBaseClasses); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList

decls = ctx->allDeclarations(CursorInRevision::invalid(), ctx->topContext(), false); bool found = false; bool classDeclFound = false; foreach ( const p& item, decls ) { if ( item.first->identifier().toString() == "B" ) { auto klass = dynamic_cast(item.first); QVERIFY(klass); QCOMPARE(klass->baseClassesSize(), static_cast(expectedBaseClasses)); classDeclFound = true; } if ( item.first->identifier().toString() == "checkme" ) { QCOMPARE(item.first->abstractType()->toString(), QString("int")); found = true; } } QVERIFY(found); QVERIFY(classDeclFound); } void PyDUChainTest::testInheritance_data() { QTest::addColumn("code"); QTest::addColumn("expectedBaseClasses"); QTest::newRow("simple") << "class A():\n\tattr = 3\n\nclass B(A):\n\tpass\n\ninst=B()\ncheckme = inst.attr" << 1; QTest::newRow("context_import_prereq") << "import testInheritance.i\ninst=testInheritance.i.testclass()\n" "checkme = inst.attr\nclass B(): pass" << 1; // 1 because object QTest::newRow("context_import") << "import testInheritance.i\n\nclass B(testInheritance.i.testclass):\n" "\ti = 4\n\ninst=B()\ncheckme = inst.attr" << 1; } void PyDUChainTest::testClassContextRanges() { QString code = "class my_class():\n pass\n \n \n \n \n"; ReferencedTopDUContext ctx = parse(code); DUChainWriteLocker lock; DUContext* classContext = ctx->findContextAt(CursorInRevision(5, 0)); QVERIFY(classContext); QVERIFY(classContext->type() == DUContext::Class); } void PyDUChainTest::testContainerTypes() { QFETCH(QString, code); QFETCH(QString, contenttype); QFETCH(bool, use_type); ReferencedTopDUContext ctx = parse(code); QVERIFY(ctx); DUChainReadLocker lock(DUChain::lock()); QList decls = ctx->findDeclarations(QualifiedIdentifier("checkme")); QVERIFY(decls.length() > 0); QVERIFY(decls.first()->abstractType()); if ( ! use_type ) { auto type = ListType::Ptr::dynamicCast(decls.first()->abstractType()); QVERIFY(type); QVERIFY(type->contentType()); QCOMPARE(type->contentType().abstractType()->toString(), contenttype); } else { QVERIFY(decls.first()->abstractType()); QEXPECT_FAIL("dict_of_int_call", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("dict_from_tuples", "returnContentEqualsContentOf isn't suitable", Continue); QEXPECT_FAIL("comprehension_shadowing_ms", "Nothing is foolproof to a sufficiently capable fool", Continue); QEXPECT_FAIL("comprehension_shadowing_nest2", "See above", Continue); QCOMPARE(decls.first()->abstractType()->toString(), contenttype); } } void PyDUChainTest::testContainerTypes_data() { QTest::addColumn("code"); QTest::addColumn("contenttype"); QTest::addColumn("use_type"); QTest::newRow("list_of_int") << "checkme = [1, 2, 3]" << "int" << false; QTest::newRow("list_from_unpacked") << "foo = [1.3]\ncheckme = [1, *foo, 3]" << "unsure (int, float)" << false; QTest::newRow("list_of_int_call") << "checkme = list([1, 2, 3])" << "int" << false; QTest::newRow("list_from_tuple") << "checkme = list((1, 2, 3))" << "int" << false; QTest::newRow("list_from_dict") << "checkme = list({'a':1, 'b':2})" << "str" << false; // Gets key type! QTest::newRow("list_from_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = list(MyClass())" << "float" << false; QTest::newRow("generator") << "checkme = [i for i in [1, 2, 3]]" << "int" << false; QTest::newRow("list_access") << "list = [1, 2, 3]\ncheckme = list[0]" << "int" << true; QTest::newRow("set_of_int") << "checkme = {1, 2, 3}" << "int" << false; QTest::newRow("set_of_int_call") << "checkme = set({1, 2, 3})" << "int" << false; QTest::newRow("set_from_tuple") << "checkme = set((1, 2, 3))" << "int" << false; QTest::newRow("set_generator") << "checkme = {i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_of_str_int") << "checkme = {'a':1, 'b':2, 'c':3}" << "dict of str : int" << true; QTest::newRow("frozenset_of_int_call") << "checkme = frozenset({1, 2, 3})" << "int" << false; QTest::newRow("dict_of_int") << "checkme = {a:1, b:2, c:3}" << "int" << false; QTest::newRow("dict_of_int_call") << "checkme = dict({'a':1, 'b':2, 'c':3})" << "dict of str : int" << true; QTest::newRow("dict_from_tuples") << "checkme = dict([('a', 1), ('b', 2)])" << "dict of str : int" << true; QTest::newRow("dict_generator") << "checkme = {\"Foo\":i for i in [1, 2, 3]}" << "int" << false; QTest::newRow("dict_access") << "list = {'a':1, 'b':2, 'c':3}\ncheckme = list[0]" << "int" << true; #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) QTest::newRow("set_from_unpacked") << "foo = [1.3]\ncheckme = {1, *foo, 3}" << "unsure (int, float)" << false; QTest::newRow("dict_from_unpacked") << "checkme = {**{'a': 1}}" << "dict of str : int" << true; QTest::newRow("dict_from_varied") << "checkme = {**{'a': 1}, 1: 1.5}" << "dict of unsure (str, int) : unsure (int, float)" << true; #endif QTest::newRow("generator_attribute") << "checkme = [item.capitalize() for item in ['foobar']]" << "str" << false; QTest::newRow("cannot_change_type") << "checkme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("cannot_change_type2") << "[1, 2, 3].append(5)\ncheckme = [\"Foo\", \"Bar\"]" << "str" << false; QTest::newRow("list_append") << "d = []\nd.append(3)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend") << "d = []; q = [int()]\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_tuple") << "d = []; q = (1, 2)\nd.extend(q)\ncheckme = d[0]" << "int" << true; QTest::newRow("list_extend_with_custom_iter") << "class MyClass:\n" " def __iter__(self): return self\n" " def __next__(self): return 3.1417\n" "checkme = []\ncheckme.extend(MyClass())" << "float" << false; QTest::newRow("for_loop") << "d = [3]\nfor item in d:\n checkme = item" << "int" << true; QTest::newRow("for_loop_unsure") << "d = [3, \"foo\"]\nfor item in d:\n checkme = item" << "unsure (int, str)" << true; QTest::newRow("for_loop_tuple_1") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = a" << "int" << true; QTest::newRow("for_loop_tuple_2") << "d = [(3, 3.5)]\nfor a, b in d:\n checkme = b" << "float" << true; QTest::newRow("for_loop_tuple_unsure") << "d = [(3, 3.5), (3.5, 3)]\nfor a, b in d:\n checkme = b" << "unsure (float, int)" << true; // Proposed by Nicolás Alvarez; why not? https://bugs.kde.org/show_bug.cgi?id=359915 QTest::newRow("comprehension_messy") << "users = {'a':19, 'b':42, 'c':35}\n" "sorted_list = sorted(users.items(), key=lambda kv: (-kv[1], kv[0]))\n" "checkme = [k for r,(k,v) in enumerate(sorted_list, 1)]" << "list of str" << true; QTest::newRow("comprehension_multiline") << "checkme = [a for\n a in \n (1, 2)]" << "list of int" << true; QTest::newRow("comprehension_multistage") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for bar in nested for foo in bar]" << "list of int" << true; QTest::newRow("comprehension_shadowing_ms") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in nested for foo in foo]" << "list of int" << true; QTest::newRow("comprehension_shadowing_nest1") << "nested = (1, 2), (3, 4)\n" "checkme = [foo for foo in [foo for foo in nested]]" << "list of tuple of (int, int)" << true; QTest::newRow("comprehension_shadowing_nest2") << "nested = (1, 2), (3, 4)\n" "checkme = [[foo for foo in foo] for foo in nested]" << "list of list of int" << true; // From https://bugs.kde.org/show_bug.cgi?id=359912 QTest::newRow("subscript_multi") << "class Middle:\n def __getitem__(self, key):\n return str()\n" "class Outer:\n def __getitem__(self, key):\n return Middle()\n" "aaa = Outer()\ncheckme = aaa[0][0]" << "str" << true; QTest::newRow("subscript_func_call") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "def bar():\n return Foo()\n" "checkme = bar()[0]" << "str" << true; QTest::newRow("subscript_unknown_index") << "a = 1,str()\ncheckme = a[5-4]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure") << "a = 1,2\na=[str()]\ncheckme = a[0]" << "unsure (int, str)" << true; QTest::newRow("subscript_unsure_getitem") << "class Foo:\n def __getitem__(self, key):\n return str()\n" "class Bar:\n def __getitem__(self, key):\n return float()\n" "a = Foo()\na=Bar()\na=[1,2]\ncheckme = a[1]" << "unsure (str, float, int)" << true; } void PyDUChainTest::testVariableCreation() { QFETCH(QString, code); QFETCH(QStringList, expected_local_declarations); QFETCH(QStringList, expected_types); ReferencedTopDUContext top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto localDecls = top->localDeclarations(); QVector localDeclNames; for ( const Declaration* d: localDecls ) { localDeclNames.append(d->identifier().toString()); } Q_ASSERT(expected_types.size() == expected_local_declarations.size()); int offset = 0; for ( const QString& expected : expected_local_declarations ) { int index = localDeclNames.indexOf(expected); QVERIFY(index != -1); QVERIFY(localDecls[index]->abstractType()); QCOMPARE(localDecls[index]->abstractType()->toString(), expected_types[offset]); offset++; } } void PyDUChainTest::testVariableCreation_data() { QTest::addColumn("code"); QTest::addColumn("expected_local_declarations"); QTest::addColumn("expected_types"); QTest::newRow("simple") << "a = 3" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("tuple_wrong") << "a, b = 3" << QStringList{"a", "b"} << QStringList{"mixed", "mixed"}; QTest::newRow("tuple_unpack_inplace") << "a, b = 3, 5.5" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_indirect") << "c = 3, 3.5\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("tuple_unpack_stacked_inplace") << "a, (b, c) = 1, (2, 3.5)" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("tuple_unpack_stacked_indirect") << "d = 3.5, (3, 1)\na, (b, c) = d" << QStringList{"a", "b", "c"} << QStringList{"float", "int", "int"}; QTest::newRow("unpack_from_list_inplace") << "a, b = [1, 2, 3]" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_from_list_indirect") << "c = [1, 2, 3]\na, b = c" << QStringList{"a", "b"} << QStringList{"int", "int"}; QTest::newRow("unpack_custom_iterable") << "class Foo:\n" " def __iter__(self): return self\n" " def __next__(self): return 1.5\n" "a, *b = Foo()" << QStringList{"a", "b"} << QStringList {"float", "list of float"}; QTest::newRow("for_loop_simple") << "for i in range(3): pass" << QStringList{"i"} << QStringList{"int"}; QTest::newRow("for_loop_unpack") << "for a, b in [(3, 5.1)]: pass" << QStringList{"a", "b"} << QStringList{"int", "float"}; QTest::newRow("for_loop_stacked") << "for a, (b, c) in [(1, (2, 3.5))]: pass" << QStringList{"a", "b", "c"} << QStringList{"int", "int", "float"}; QTest::newRow("for_loop_tuple") << "for a in 1, 2: pass" << QStringList{"a"} << QStringList{"int"}; QTest::newRow("for_loop_dict") << "for a in {'foo': 1}: pass" << QStringList{"a"} << QStringList{"str"}; } void PyDUChainTest::testCleanupMultiplePasses() { for ( int j = 0; j < 20; j++ ) { ReferencedTopDUContext top = parse("from testCleanupMultiplePasses import foo\ndef fonc(): return 3+2j\nfoo.foo.func = fonc"); } } void PyDUChainTest::testManyDeclarations() { ReferencedTopDUContext top = parse("from testManyDeclarations import test\nk=test.Foo()"); } void PyDUChainTest::testComments() { QFETCH(QString, code); auto top = parse(code); QVERIFY(top); DUChainReadLocker lock; auto decls = top->findDeclarations(QualifiedIdentifier("a")); QCOMPARE(decls.size(), 1); auto a = decls.first(); QCOMPARE(a->comment(), QByteArray("comment")); decls = top->findDeclarations(QualifiedIdentifier("b")); if ( decls.isEmpty() ) { decls = top->childContexts().last()->findDeclarations(QualifiedIdentifier("b")); } auto b = decls.first(); QCOMPARE(b->comment(), QByteArray()); } void PyDUChainTest::testComments_data() { QTest::addColumn("code"); QTest::newRow("variable") << "b=5\n\"\"\"comment\"\"\"\na=5\nb=5"; QTest::newRow("function") << "def a():\n \"\"\"comment\"\"\"\n b=5"; QTest::newRow("class") << "class a:\n \"\"\"comment\"\"\"\n b=5"; } diff --git a/duchain/usebuilder.cpp b/duchain/usebuilder.cpp index 3b73b9dd..19ac4542 100644 --- a/duchain/usebuilder.cpp +++ b/duchain/usebuilder.cpp @@ -1,177 +1,181 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2010-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "usebuilder.h" #include #include "duchaindebug.h" #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "ast.h" #include "expressionvisitor.h" #include "helpers.h" using namespace KTextEditor; using namespace KDevelop; namespace Python { UseBuilder::UseBuilder(PythonEditorIntegrator* editor, QVector ignoreVariables) : UseBuilderBase() , m_errorReportingEnabled(true) , m_ignoreVariables(ignoreVariables) { setEditor(editor); } DUContext* UseBuilder::contextAtOrCurrent(const CursorInRevision& pos) { DUContext* context = nullptr; { DUChainReadLocker lock; context = topContext()->findContextAt(pos, true); } if ( ! context ) { context = currentContext(); } return context; } void UseBuilder::useHiddenMethod(ExpressionAst* value, Declaration* function) { + if ( !function || function->topContext() == Helper::getDocumentationFileContext() ) { + // Don't add a use for e.g. `list.__getitem__` in `foo[0]`, no-one cares. + return; + } RangeInRevision useRange; // TODO fixme! this does not necessarily use the opening bracket as it should useRange.start = CursorInRevision(value->endLine, value->endCol + 1); useRange.end = CursorInRevision(value->endLine, value->endCol + 2); if ( function && function->isFunctionDeclaration() ) { UseBuilderBase::newUse(value, useRange, DeclarationPointer(function)); } } void UseBuilder::visitName(NameAst* node) { DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node)); Declaration* declaration = Helper::declarationForName(node, editorFindPositionSafe(node), DUChainPointer(context)); Q_ASSERT(node->identifier); RangeInRevision useRange = rangeForNode(node->identifier, true); if ( declaration && declaration->range() == useRange ) return; if ( ! declaration && m_errorReportingEnabled ) { if ( ! m_ignoreVariables.contains(IndexedString(node->identifier->value)) ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Hint); p->setDescription(i18n("Undefined variable: %1", node->identifier->value)); { DUChainWriteLocker wlock(DUChain::lock()); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } UseBuilderBase::newUse(node, useRange, DeclarationPointer(declaration)); } void UseBuilder::visitCall(CallAst* node) { UseBuilderBase::visitCall(node); DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node)); ExpressionVisitor v(context); v.visitNode(node->function); if ( auto classType = v.lastType().cast() ) { DUChainReadLocker lock; // This is either __init__() or __call__(): `a = Foo()` or `b = a()`. auto function = Helper::functionForCalled(classType->declaration(topContext()), v.isAlias()); lock.unlock(); useHiddenMethod(node->function, function.declaration); } } void UseBuilder::visitAttribute(AttributeAst* node) { UseBuilderBase::visitAttribute(node); DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node)); ExpressionVisitor v(context); v.visitNode(node); RangeInRevision useRange(node->attribute->startLine, node->attribute->startCol, node->attribute->endLine, node->attribute->endCol + 1); DeclarationPointer declaration = v.lastDeclaration(); DUChainWriteLocker wlock; if ( declaration && declaration->range() == useRange ) { // this is the declaration, don't build a use for it return; } if ( ! declaration && v.isConfident() && ( ! v.lastType() || Helper::isUsefulType(v.lastType()) ) ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); p->setSource(KDevelop::IProblem::SemanticAnalysis); p->setSeverity(KDevelop::IProblem::Hint); p->setDescription(i18n("Attribute \"%1\" not found on accessed object", node->attribute->value)); ProblemPointer ptr(p); topContext()->addProblem(ptr); } UseBuilderBase::newUse(node, useRange, declaration); } void UseBuilder::visitSubscript(SubscriptAst* node) { UseBuilderBase::visitSubscript(node); DUContext* context = contextAtOrCurrent(editorFindPositionSafe(node->value)); ExpressionVisitor v(context); v.visitNode(node->value); static const IndexedIdentifier getitemIdentifier(KDevelop::Identifier("__getitem__")); static const IndexedIdentifier setitemIdentifier(KDevelop::Identifier("__setitem__")); bool isAugTarget = (node->parent->astType == Ast::AugmentedAssignmentAstType && static_cast(node->parent)->target == node); // e.g `a[0] += 2` uses both __getitem__ and __setitem__. if (isAugTarget || node->context == ExpressionAst::Context::Load) { DUChainReadLocker lock; auto getItemFunc = Helper::accessAttribute(v.lastType(), getitemIdentifier, context->topContext()); lock.unlock(); useHiddenMethod(node->value, getItemFunc); } if ( node->context == ExpressionAst::Context::Store ) { DUChainReadLocker lock; auto setItemFunc = Helper::accessAttribute(v.lastType(), setitemIdentifier, context->topContext()); lock.unlock(); useHiddenMethod(node->value, setItemFunc); } } ParseSession *UseBuilder::parseSession() const { return m_session; } } // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on diff --git a/parser/CMakeLists.txt b/parser/CMakeLists.txt index 15e7ea0d..df12ddde 100644 --- a/parser/CMakeLists.txt +++ b/parser/CMakeLists.txt @@ -1,34 +1,34 @@ set(parser_STAT_SRCS codehelpers.cpp parsesession.cpp ast.cpp astdefaultvisitor.cpp astvisitor.cpp astbuilder.cpp cythonsyntaxremover.cpp + rangefixvisitor.cpp ) ecm_qt_declare_logging_category(parser_STAT_SRCS HEADER parserdebug.h IDENTIFIER KDEV_PYTHON_PARSER CATEGORY_NAME "kdevelop.languages.python.parser" ) include_directories(${PYTHON_INCLUDE_DIRS}) add_library( kdevpythonparser SHARED ${parser_STAT_SRCS} ) generate_export_header(kdevpythonparser EXPORT_MACRO_NAME KDEVPYTHONPARSER_EXPORT EXPORT_FILE_NAME parserexport.h) target_link_libraries(kdevpythonparser LINK_PRIVATE KDev::Language - KDev::Util Qt5::Core ${PYTHON_LIBRARIES} ) install(TARGETS kdevpythonparser DESTINATION ${INSTALL_TARGETS_DEFAULT_ARGS}) if (BUILD_TESTING) add_subdirectory(tests) endif() diff --git a/parser/astbuilder.cpp b/parser/astbuilder.cpp index 1d02d659..a1ccd68d 100644 --- a/parser/astbuilder.cpp +++ b/parser/astbuilder.cpp @@ -1,766 +1,282 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astbuilder.h" #include "ast.h" -#include -#include -#include -#include -#include #include #include #include #include #include "python_header.h" #include "astdefaultvisitor.h" #include "cythonsyntaxremover.h" - -#include -#include -#include -#include +#include "rangefixvisitor.h" #include #include "parserdebug.h" using namespace KDevelop; extern grammar _PyParser_Grammar; namespace Python { - -class NextAstFindVisitor : public AstDefaultVisitor { -public: - KTextEditor::Cursor findNext(Python::Ast* node) { - m_root = node; - auto parent = node; - while ( parent->parent && parent->parent->isExpression() ) { - parent = parent->parent; - } - visitNode(parent); - - while ( ! m_next.isValid() && parent->parent ) { - // no next expression found in that statement, advance to the next statement - parent = parent->parent; - visitNode(parent); - } - - return m_next; - }; - void visitNode(Python::Ast* node) override { - if ( ! node ) { - return; - } - AstDefaultVisitor::visitNode(node); - if ( node->start() > m_root->start() && ! node->isChildOf(m_root) ) { - m_next = (m_next < node->start() && m_next.isValid()) ? m_next : node->start(); - } - } - -private: - KTextEditor::Cursor m_next{-1, -1}; - Ast* m_root; -}; - -// This class is used to fix some of the remaining issues -// with the ranges of objects obtained from the python parser. -// Issues addressed are: -// 1) decorators and "def" / "class" statements for classes / functions -// 2) ranges of aliases -// Both issues are easy to correct since the possible syntax is very restricted -// (no bracket matching, no strings, no nesting, ...) -// For the aliases, fortunately only imports and excepthandlers are affected; -// the "with" statement, which has more complicated syntax, provides -// the necessary information already. -// 3) attribute rangees -// Not so easy, but when starting from the end of the expression it works okay -class RangeFixVisitor : public AstDefaultVisitor { -public: - RangeFixVisitor(const QString& contents) - : lines(contents.split('\n')) { }; - - void visitNode(Ast* node) override { - AstDefaultVisitor::visitNode(node); - if ( node && node->parent && node->parent->astType != Ast::AttributeAstType ) { - if ( ( node->parent->endLine <= node->endLine && node->parent->endCol <= node->endCol ) - || node->parent->endLine < node->endLine ) - { - node->parent->endLine = node->endLine; - node->parent->endCol = node->endCol; - } - } - }; - - void visitFunctionDefinition(FunctionDefinitionAst* node) override { - cutDefinitionPreamble(node->name, node->async ? "asyncdef" : "def"); - AstDefaultVisitor::visitFunctionDefinition(node); - }; - - void visitClassDefinition(ClassDefinitionAst* node) override { - cutDefinitionPreamble(node->name, "class"); - AstDefaultVisitor::visitClassDefinition(node); - }; - - void visitAttribute(AttributeAst* node) override { - // Work around the weird way to count columns in Python's AST module. - - // Find where the next expression (of any kind) behind this one starts - NextAstFindVisitor v; - auto next_start = v.findNext(node); - if ( ! next_start.isValid() ) { - // use end of document as reference - next_start = {lines.size() - 1, lines.last().size() - 1}; - } - - // take only the portion of the line up to that next expression - auto endLine = next_start.line(); - auto endCol = next_start.column(); - if ( ! (next_start > node->start()) ) { - endLine = node->startLine; - endCol = -1; - } - - const QString& name(node->attribute->value); - - QString line; - for ( int n = node->startLine, - pos = node->value->endCol + 1, - dotFound = false, - nameFound = false; - n <= endLine; ++n, pos = 0 ) { - line = lines.at(n); - if ( n == endLine && endCol != -1 ) { - // Never look at the next expression. - line = line.left(endCol); - } - if ( !dotFound ) { - // The real attr name can never be before a dot. - // Nor can the start of a comment. - // (Don't be misled by `foo["bar"].bar` or `foo["#"].bar`) - pos = line.indexOf('.', pos); - if ( pos == -1 ) continue; - dotFound = true; - } - if ( !nameFound ) { - // Track if the attr name has appeared at least once. - // This helps avoid interpreting '#'s in strings as comments - - // there can never be a comment before the real attr name. - pos = line.indexOf(name, pos + 1); - if ( pos == -1 ) continue; - nameFound = true; - } - if ( dotFound && nameFound && - (pos = line.indexOf('#', pos + name.length())) != -1) { - // Remove the comment after a '#' iff we're certain it can't - // be inside a string literal (e.g. `foo["#"].bar`). - line = line.left(pos); - } - // Take the last occurrence, any others are in string literals. - pos = line.lastIndexOf(name); - if ( pos != -1 ) { - node->startLine = n; - node->startCol = pos; - } - // N.B. we do this for all lines, the last non-comment occurrence - // is the real one. - } - // This fails (only, AFAIK) in a very limited case: - // If the value expression (`foo` in `foo.bar`) contains a dot, the - // attr name, _and_ a hash in that order (may not be consecutive), - // and the hash is on the same line as the real attr name, - // we wrongly interpret the hash as the start of a comment. - // e.g `foo["...barrier#"].bar` will highlight part of the string. - - node->endLine = node->startLine; - node->endCol = node->startCol + name.length() - 1; - node->attribute->copyRange(node); - - AstDefaultVisitor::visitAttribute(node); - }; - - // alias for imports (import foo as bar, baz as bang) - // no strings, brackets, or whatever are allowed here, so the "parser" - // can be very straightforward. - void visitImport(ImportAst* node) override { - AstDefaultVisitor::visitImport(node); - int aliasIndex = 0; - foreach ( AliasAst* alias, node->names ) { - fixAlias(alias->name, alias->asName, node->startLine, aliasIndex); - aliasIndex += 1; - } - }; - - // alias for exceptions (except FooBarException as somethingterriblehappened: ...) - void visitExceptionHandler(ExceptionHandlerAst* node) override { - AstDefaultVisitor::visitExceptionHandler(node); - if ( ! node->name ) { - return; - } - const QString& line = lines.at(node->startLine); - const int end = line.count() - 1; - int back = backtrackDottedName(line, end); - node->name->startCol = end - back; - node->name->endCol = end; - } - - void visitString(Python::StringAst* node) override { - AstDefaultVisitor::visitString(node); - auto match = findString.match(lines.at(node->startLine), node->startCol); - if ( match.capturedLength() > 0 ) { - node->endCol += match.capturedLength() - 1; // Ranges are inclusive. - } - } - void visitBytes(Python::BytesAst* node) override { - AstDefaultVisitor::visitBytes(node); - auto match = findString.match(lines.at(node->startLine), node->startCol + 1); - if ( match.capturedLength() > 0 ) { - node->endCol += match.capturedLength(); // -1 then +1, because of the 'b'. - } - } - void visitFormattedValue(Python::FormattedValueAst * node) override { - AstDefaultVisitor::visitFormattedValue(node); - auto match = findString.match(lines.at(node->startLine), node->startCol + 1); - if ( match.capturedLength() > 0 ) { - node->endCol += match.capturedLength(); - } - } - - void visitNumber(Python::NumberAst* node) override { - AstDefaultVisitor::visitNumber(node); - auto match = findNumber.match(lines.at(node->startLine), node->startCol); - if ( match.capturedLength() > 0 ) { - node->endCol += match.capturedLength() - 1; // Ranges are inclusive. - } - } - - // Add one column after the last child to cover the closing bracket: `[1,2,3]` - // TODO This is still wrong if the last child is followed by parens or whitespace. - // endCol matters most in single-line expressions, so this isn't a huge problem. - void visitSubscript(Python::SubscriptAst* node) override { - AstDefaultVisitor::visitSubscript(node); - node->endCol++; - } - void visitComprehension(Python::ComprehensionAst* node) override { - AstDefaultVisitor::visitComprehension(node); - node->endCol++; - } - void visitList(Python::ListAst* node) override { - AstDefaultVisitor::visitList(node); - node->endCol++; - } - void visitTuple(Python::TupleAst* node) override { - AstDefaultVisitor::visitTuple(node); - node->endCol++; - } - -private: - const QStringList lines; - QVector dots; - KTextEditor::Cursor attributeStart; - static const QRegularExpression findString; - static const QRegularExpression findNumber; - - // skip the decorators and the "def" at the beginning - // of a class or function declaration and modify @arg node - // example: - // @decorate(foo) - // @decorate(bar) - // class myclass(parent): pass - // before: start of class->name is [0, 0] - // after: start of class->name is [2, 5] - // line continuation characters are not supported, - // because code needing those in this case is not worth being supported. - void cutDefinitionPreamble(Ast* fixNode, const QString& defKeyword) { - if ( ! fixNode ) { - return; - } - int currentLine = fixNode->startLine; - - // cut away decorators - while ( currentLine < lines.size() ) { - if ( lines.at(currentLine).trimmed().remove(' ').remove('\t').startsWith(defKeyword) ) { - // it's not a decorator, so stop skipping lines. - break; - } - currentLine += 1; - } -// qDebug() << "FIX:" << fixNode->range(); - fixNode->startLine = currentLine; - fixNode->endLine = currentLine; -// qDebug() << "FIXED:" << fixNode->range() << fixNode->astType; - - // cut away the "def" / "class" - int currentColumn = -1; - if ( currentLine > lines.size() ) { - // whops? - return; - } - const QString& lineData = lines.at(currentLine); - bool keywordFound = false; - while ( currentColumn < lineData.size() - 1 ) { - currentColumn += 1; - if ( lineData.at(currentColumn).isSpace() ) { - // skip space at the beginning of the line - continue; - } - else if ( keywordFound ) { - // if the "def" / "class" was already found, and the current char is - // non space, then this is indeed the start of the identifier we're looking for. - break; - } - else { - keywordFound = true; - currentColumn += defKeyword.size(); - } - } - const int previousLength = fixNode->endCol - fixNode->startCol; - fixNode->startCol = currentColumn; - fixNode->endCol = currentColumn + previousLength; - }; - - int backtrackDottedName(const QString& data, const int start) { - bool haveDot = true; - bool previousWasSpace = true; - for ( int i = start - 1; i >= 0; i-- ) { - if ( data.at(i).isSpace() ) { - previousWasSpace = true; - continue; - } - if ( data.at(i) == ':' ) { - // excepthandler - continue; - } - if ( data.at(i) == '.' ) { - haveDot = true; - } - else if ( haveDot ) { - haveDot = false; - previousWasSpace = false; - continue; - } - if ( previousWasSpace && ! haveDot ) { - return start-i-2; - } - previousWasSpace = false; - } - return 0; - } - - void fixAlias(Ast* dotted, Ast* asname, const int startLine, int aliasIndex) { - if ( ! asname && ! dotted ) { - return; - } - QString line = lines.at(startLine); - int lineno = startLine; - for ( int i = 0; i < line.size(); i++ ) { - const QChar& current = line.at(i); - if ( current == '\\' ) { - // line continuation character - // splitting like "import foo as \ \n bar" is not supported. - lineno += 1; - line = lines.at(lineno); - i = 0; - continue; - } - if ( current == ',' ) { - if ( aliasIndex == 0 ) { - // nothing found, continue below - line = line.left(i); - break; - } - // next alias expression - aliasIndex -= 1; - } - if ( i > line.length() - 3 ) { - continue; - } - if ( current.isSpace() && line.mid(i+1).startsWith("as") && ( line.at(i+3).isSpace() || line.at(i+3) == '\\' ) ) { - // there's an "as" - if ( aliasIndex == 0 ) { - // it's the one we're looking for - // find the expression - if ( dotted ) { - int dottedNameLength = backtrackDottedName(line, i); - dotted->startLine = lineno; - dotted->endLine = lineno; - dotted->startCol = i-dottedNameLength; - dotted->endCol = i; - } - // find the asname - if ( asname ) { - bool atStart = true; - int textStart = i+3; - for ( int j = i+3; j < line.size(); j++ ) { - if ( atStart && ! line.at(j).isSpace() ) { - atStart = false; - textStart = j; - } - if ( ! atStart && ( line.at(j).isSpace() || j == line.size() - 1 ) ) { - // found it - asname->startLine = lineno; - asname->endLine = lineno; - asname->startCol = textStart - 1; - asname->endCol = j; - } - } - } - return; - } - } - } - // no "as" found, use last dotted name in line - const int end = line.count() - whitespaceAtEnd(line); - int back = backtrackDottedName(line, end); - dotted->startLine = lineno; - dotted->endLine = lineno; - dotted->startCol = end - back; - dotted->endCol = end; - }; - - int whitespaceAtEnd(const QString& line) { - for ( int i = 0; i < line.size(); i++ ) { - if ( ! line.at(line.size() - i - 1).isSpace() ) { - return i; - } - } - return 0; - }; -}; -// FIXME This doesn't work for triple-quoted strings -// (it gives length 2, which is no worse than before). -const QRegularExpression RangeFixVisitor::findString = QRegularExpression("\\G(['\"]).*?(?(PyObject_Str(obj), pyObjectCleanup); const auto str = strOwner.get(); if (PyUnicode_READY(str) < 0) { qWarning("PyUnicode_READY(%p) returned false!", (void*)str); return QString(); } const auto length = PyUnicode_GET_LENGTH(str); switch(PyUnicode_KIND(str)) { case PyUnicode_1BYTE_KIND: return QString::fromLatin1((const char*)PyUnicode_1BYTE_DATA(str), length); case PyUnicode_2BYTE_KIND: return QString::fromUtf16(PyUnicode_2BYTE_DATA(str), length); case PyUnicode_4BYTE_KIND: return QString::fromUcs4(PyUnicode_4BYTE_DATA(str), length); case PyUnicode_WCHAR_KIND: qWarning("PyUnicode_KIND(%p) returned PyUnicode_WCHAR_KIND, this should not happen!", (void*)str); return QString::fromWCharArray(PyUnicode_AS_UNICODE(str), length); } Q_UNREACHABLE(); } -QPair fileHeaderHack(QString& contents, const QUrl& filename) -{ - IProject* proj = ICore::self()->projectController()->findProjectForUrl(filename); - // the file is not in a project, don't apply hack - if ( ! proj ) { - return QPair(contents, 0); - } - const QUrl headerFileUrl = QUrl::fromLocalFile(proj->path().path() + "/.kdev_python_header"); - QFile headerFile(headerFileUrl.path()); - QString headerFileContents; - if ( headerFile.exists() ) { - headerFile.open(QIODevice::ReadOnly); - headerFileContents = headerFile.readAll(); - headerFile.close(); - qCDebug(KDEV_PYTHON_PARSER) << "Found header file, applying hack"; - int insertAt = 0; - bool endOfCommentsReached = false; - bool commentSignEncountered = false; -// bool atLineBeginning = true; - int lastLineBeginning = 0; - int newlineCount = 0; - int l = contents.length(); - do { - if ( insertAt >= l ) { - qCDebug(KDEV_PYTHON_PARSER) << "File consist only of comments, not applying hack"; - return QPair(contents, 0); - } - if ( contents.at(insertAt) == '#' ) { - commentSignEncountered = true; - } - if ( !contents.at(insertAt).isSpace() ) { -// atLineBeginning = false; - if ( !commentSignEncountered ) { - endOfCommentsReached = true; - } - } - if ( contents.at(insertAt) == '\n' ) { -// atLineBeginning = true; - commentSignEncountered = false; - lastLineBeginning = insertAt; - newlineCount += 1; - } - if ( newlineCount == 2 ) { - endOfCommentsReached = true; - } - insertAt += 1; - } while ( !endOfCommentsReached ); - qCDebug(KDEV_PYTHON_PARSER) << "Inserting contents at char" << lastLineBeginning << "of file"; - contents = contents.left(lastLineBeginning) - + "\n" + headerFileContents + "\n#\n" - + contents.right(contents.length() - lastLineBeginning); - qCDebug(KDEV_PYTHON_PARSER) << contents; - return QPair(contents, - ( headerFileContents.count('\n') + 3 )); - } - else { - return QPair(contents, 0); - } -} - namespace { struct PythonInitializer : private QMutexLocker { PythonInitializer(QMutex& pyInitLock): QMutexLocker(&pyInitLock), arena(nullptr) { Py_InitializeEx(0); Q_ASSERT(Py_IsInitialized()); arena = PyArena_New(); Q_ASSERT(arena); // out of memory } ~PythonInitializer() { if (arena) PyArena_Free(arena); if (Py_IsInitialized()) Py_Finalize(); } PyArena* arena; }; } CodeAst::Ptr AstBuilder::parse(const QUrl& filename, QString &contents) { qCDebug(KDEV_PYTHON_PARSER) << " ====> AST ====> building abstract syntax tree for " << filename.path(); Py_NoSiteFlag = 1; contents.append('\n'); - QPair hacked = fileHeaderHack(contents, filename); - contents = hacked.first; - int lineOffset = hacked.second; - PythonInitializer pyIniter(pyInitLock); PyArena* arena = pyIniter.arena; PyCompilerFlags flags = {PyCF_SOURCE_IS_UTF8 | PyCF_IGNORE_COOKIE}; - PyObject *exception, *value, *backtrace; - PyErr_Fetch(&exception, &value, &backtrace); - CythonSyntaxRemover cythonSyntaxRemover; if (filename.fileName().endsWith(".pyx", Qt::CaseInsensitive)) { qCDebug(KDEV_PYTHON_PARSER) << filename.fileName() << "is probably Cython file."; contents = cythonSyntaxRemover.stripCythonSyntax(contents); } mod_ty syntaxtree = PyParser_ASTFromString(contents.toUtf8().data(), "", file_input, &flags, arena); if ( ! syntaxtree ) { qCDebug(KDEV_PYTHON_PARSER) << " ====< parse error, trying to fix"; - + + PyObject *exception, *value, *backtrace; PyErr_Fetch(&exception, &value, &backtrace); qCDebug(KDEV_PYTHON_PARSER) << "Error objects: " << exception << value << backtrace; if ( ! value ) { qCWarning(KDEV_PYTHON_PARSER) << "Internal parser error: exception value is null, aborting"; return CodeAst::Ptr(); } + PyErr_NormalizeException(&exception, &value, &backtrace); - PyObject_Print(value, stderr, Py_PRINT_RAW); - - PyObject* errorMessage_str = PyTuple_GetItem(value, 0); - PyObject* errorDetails_tuple = PyTuple_GetItem(value, 1); - - if ( ! errorDetails_tuple ) { - qCWarning(KDEV_PYTHON_PARSER) << "Error retrieving error message, not displaying, and not doing anything"; + if ( ! PyObject_IsInstance(value, PyExc_SyntaxError) ) { + qCWarning(KDEV_PYTHON_PARSER) << "Exception was not a SyntaxError, aborting"; return CodeAst::Ptr(); } - PyObject* linenoobj = PyTuple_GetItem(errorDetails_tuple, 1); - errorMessage_str = PyTuple_GetItem(value, 0); - errorDetails_tuple = PyTuple_GetItem(value, 1); - PyObject_Print(errorMessage_str, stderr, Py_PRINT_RAW); - - PyObject* colnoobj = PyTuple_GetItem(errorDetails_tuple, 2); + PyObject* errorMessage_str = PyObject_GetAttrString(value, "msg"); + PyObject* linenoobj = PyObject_GetAttrString(value, "lineno"); + PyObject* colnoobj = PyObject_GetAttrString(value, "offset"); + int lineno = PyLong_AsLong(linenoobj) - 1; int colno = PyLong_AsLong(colnoobj); - + ProblemPointer p(new Problem()); - KTextEditor::Cursor start(lineno + lineOffset, (colno-4 > 0 ? colno-4 : 0)); - KTextEditor::Cursor end(lineno + lineOffset, (colno+4 > 4 ? colno+4 : 4)); + KTextEditor::Cursor start(lineno, (colno-4 > 0 ? colno-4 : 0)); + KTextEditor::Cursor end(lineno, (colno+4 > 4 ? colno+4 : 4)); KTextEditor::Range range(start, end); qCDebug(KDEV_PYTHON_PARSER) << "Problem range: " << range; DocumentRange location(IndexedString(filename.path()), range); p->setFinalLocation(location); p->setDescription(PyUnicodeObjectToQString(errorMessage_str)); p->setSource(IProblem::Parser); m_problems.append(p); // try to recover. // Currently the following is tired: // * If the last non-space char before the error reported was ":", it's most likely an indent error. // The common easy-to-fix and annoying indent error is "for item in foo: ". In that case, just add "pass" after the ":" token. // * If it's not, we will just comment the line with the error, fixing problems like "foo = ". // * If both fails, everything including the first non-empty line before the one with the error will be deleted. int len = contents.length(); int currentLine = 0; QString currentLineContents; QChar c; QChar newline('\n'); int emptySince = 0; int emptySinceLine = 0; int emptyLinesSince = 0; int emptyLinesSinceLine = 0; unsigned short currentLineIndent = 0; bool atLineBeginning = true; QList indents; int errline = qMax(0, lineno); int currentLineBeginning = 0; for ( int i = 0; i < len; i++ ) { c = contents.at(i); if ( ! c.isSpace() ) { emptySince = i; emptySinceLine = currentLine; atLineBeginning = false; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); } else if ( c == newline ) { if ( currentLine == errline ) { atLineBeginning = false; } else { currentLine += 1; currentLineBeginning = i+1; // this line has had content, so reset the "empty lines since" counter if ( ! atLineBeginning ) { // lastNonemptyLineBeginning = emptyLinesSince; emptyLinesSince = i; emptyLinesSinceLine = currentLine; } atLineBeginning = true; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); currentLineIndent = 0; } } else if ( atLineBeginning ) { currentLineIndent += 1; } if ( currentLine == errline && ! atLineBeginning ) { // if the last non-empty char before the error opens a new block, it's likely an "empty block" problem // we can easily fix that by adding in a "pass" statement. However, we want to add that in the next line, if possible // so context ranges for autocompletion stay intact. if ( contents[emptySince] == QChar(':') ) { qCDebug(KDEV_PYTHON_PARSER) << indents.length() << emptySinceLine + 1 << indents; if ( indents.length() > emptySinceLine + 1 && indents.at(emptySinceLine) < indents.at(emptySinceLine + 1) ) { qCDebug(KDEV_PYTHON_PARSER) << indents.at(emptySinceLine) << indents.at(emptySinceLine + 1); contents.insert(emptyLinesSince + 1 + indents.at(emptyLinesSinceLine), "\tpass#"); } else { contents.insert(emptySince + 1, "\tpass#"); } } else if ( indents.length() >= currentLine && currentLine > 0 ) { qCDebug(KDEV_PYTHON_PARSER) << indents << currentLine; contents[i+1+indents.at(currentLine - 1)] = QChar('#'); contents.insert(i+1+indents.at(currentLine - 1), "pass"); } break; } } syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); // 3rd try: discard everything after the last non-empty line, but only until the next block start currentLineBeginning = qMin(contents.length() - 1, currentLineBeginning); errline = qMax(0, qMin(indents.length()-1, errline)); if ( ! syntaxtree ) { qCWarning(KDEV_PYTHON_PARSER) << "Discarding parts of the code to be parsed because of previous errors"; qCDebug(KDEV_PYTHON_PARSER) << indents; int indentAtError = indents.at(errline); QChar c; bool atLineBeginning = true; int currentIndent = -1; int currentLineBeginning_end = currentLineBeginning; int currentLineContentBeginning = currentLineBeginning; for ( int i = currentLineBeginning; i < len; i++ ) { c = contents.at(i); qCDebug(KDEV_PYTHON_PARSER) << c; if ( c == '\n' ) { if ( currentIndent <= indentAtError && currentIndent != -1 ) { qCDebug(KDEV_PYTHON_PARSER) << "Start of error code: " << currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "End of error block (current position): " << currentLineBeginning_end; qCDebug(KDEV_PYTHON_PARSER) << "Length: " << currentLineBeginning_end - currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "indent at error <> current indent:" << indentAtError << "<>" << currentIndent; // contents.remove(currentLineBeginning, currentLineBeginning_end-currentLineBeginning); break; } contents.insert(currentLineContentBeginning - 1, "pass#"); i += 5; i = qMin(i, contents.length()); len = contents.length(); atLineBeginning = true; currentIndent = 0; currentLineBeginning_end = i + 1; currentLineContentBeginning = i + 1; continue; } if ( ! c.isSpace() && atLineBeginning ) { currentLineContentBeginning = i; atLineBeginning = false; } if ( c.isSpace() && atLineBeginning ) currentIndent += 1; } qCDebug(KDEV_PYTHON_PARSER) << "This is what is left: " << contents; syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); } if ( ! syntaxtree ) { return CodeAst::Ptr(); // everything fails, so we abort. } } qCDebug(KDEV_PYTHON_PARSER) << "Got syntax tree from python parser:" << syntaxtree->kind << Module_kind; - PythonAstTransformer t(lineOffset); + PythonAstTransformer t; t.run(syntaxtree, filename.fileName().replace(".py", "")); RangeFixVisitor fixVisitor(contents); fixVisitor.visitNode(t.ast); cythonSyntaxRemover.fixAstRanges(t.ast); return CodeAst::Ptr(t.ast); } } diff --git a/parser/astbuilder.h b/parser/astbuilder.h index 2430112e..93a3ac77 100644 --- a/parser/astbuilder.h +++ b/parser/astbuilder.h @@ -1,64 +1,59 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef ASTBUILDER_H #define ASTBUILDER_H #include "ast.h" #include "parserexport.h" #include #include "astdefaultvisitor.h" #include typedef struct _object PyObject; namespace PythonParser { class Parser; class AstNode; } namespace Python { class Ast; class CodeAst; - -typedef QMap stringDictionary; - -QPair fileHeaderHack(QString& contents, const QUrl& filename); - QString PyUnicodeObjectToQString(PyObject* obj); class KDEVPYTHONPARSER_EXPORT AstBuilder { public: CodeAst::Ptr parse(const QUrl& filename, QString &contents); QList m_problems; private: static QMutex pyInitLock; }; } #endif diff --git a/parser/generated.h b/parser/generated.h index 1e6e3a51..1087ac2e 100644 --- a/parser/generated.h +++ b/parser/generated.h @@ -1,794 +1,792 @@ /* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ #include #include "kdevpythonversion.h" class PythonAstTransformer { public: CodeAst* ast; - PythonAstTransformer(int lineOffset) : m_lineOffset(lineOffset) {}; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } // Shift lines by some fixed amount inline int tline(int line) { if ( line == -99999 ) { // don't touch the marker return -99999; } - return line + m_lineOffset; + return line; }; private: QStack nodeStack; - int m_lineOffset; - + Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } Ast* visitNode(_alias* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; AliasAst* v = new AliasAst(parent()); v->name = node->name ? new Python::Identifier(PyUnicodeObjectToQString(node->name)) : nullptr; v->asName = node->asname ? new Python::Identifier(PyUnicodeObjectToQString(node->asname)) : nullptr; return v; } Ast* visitNode(_arg* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgAst* v = new ArgAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; if ( v->argumentName ) { v->argumentName->startCol = node->col_offset; v->startCol = v->argumentName->startCol; v->argumentName->startLine = tline(node->lineno - 1); v->startLine = v->argumentName->startLine; v->argumentName->endCol = node->col_offset + v->argumentName->value.length() - 1; v->endCol = v->argumentName->endCol; v->argumentName->endLine = tline(node->lineno - 1); v->endLine = v->argumentName->endLine; ranges_copied = true; } nodeStack.push(v); v->annotation = static_cast(visitNode(node->annotation)); nodeStack.pop(); return v; } Ast* visitNode(_arguments* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgumentsAst* v = new ArgumentsAst(parent()); nodeStack.push(v); v->vararg = static_cast(visitNode(node->vararg)); nodeStack.pop(); nodeStack.push(v); v->kwarg = static_cast(visitNode(node->kwarg)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_arg, ArgAst>(node->args); nodeStack.pop(); nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); nodeStack.push(v); v->kwonlyargs = visitNodeList<_arg, ArgAst>(node->kwonlyargs); nodeStack.pop(); return v; } Ast* visitNode(_comprehension* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ComprehensionAst* v = new ComprehensionAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->iter)); nodeStack.pop(); nodeStack.push(v); v->conditions = visitNodeList<_expr, ExpressionAst>(node->ifs); nodeStack.pop(); return v; } Ast* visitNode(_excepthandler* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case ExceptHandler_kind: { ExceptionHandlerAst* v = new ExceptionHandlerAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.ExceptHandler.type)); nodeStack.pop(); v->name = node->v.ExceptHandler.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ExceptHandler.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ExceptHandler.body); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _excepthandler AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_expr* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Await_kind: { AwaitAst* v = new AwaitAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Await.value)); nodeStack.pop(); result = v; break; } #endif case BoolOp_kind: { BooleanOperationAst* v = new BooleanOperationAst(parent()); v->type = (ExpressionAst::BooleanOperationTypes) node->v.BoolOp.op; nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.BoolOp.values); nodeStack.pop(); result = v; break; } case BinOp_kind: { BinaryOperationAst* v = new BinaryOperationAst(parent()); v->type = (ExpressionAst::OperatorTypes) node->v.BinOp.op; nodeStack.push(v); v->lhs = static_cast(visitNode(node->v.BinOp.left)); nodeStack.pop(); nodeStack.push(v); v->rhs = static_cast(visitNode(node->v.BinOp.right)); nodeStack.pop(); result = v; break; } case UnaryOp_kind: { UnaryOperationAst* v = new UnaryOperationAst(parent()); v->type = (ExpressionAst::UnaryOperatorTypes) node->v.UnaryOp.op; nodeStack.push(v); v->operand = static_cast(visitNode(node->v.UnaryOp.operand)); nodeStack.pop(); result = v; break; } case Lambda_kind: { LambdaAst* v = new LambdaAst(parent()); nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.Lambda.args)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Lambda.body)); nodeStack.pop(); result = v; break; } case IfExp_kind: { IfExpressionAst* v = new IfExpressionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.IfExp.test)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.IfExp.body)); nodeStack.pop(); nodeStack.push(v); v->orelse = static_cast(visitNode(node->v.IfExp.orelse)); nodeStack.pop(); result = v; break; } case Dict_kind: { DictAst* v = new DictAst(parent()); nodeStack.push(v); v->keys = visitNodeList<_expr, ExpressionAst>(node->v.Dict.keys); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Dict.values); nodeStack.pop(); result = v; break; } case Set_kind: { SetAst* v = new SetAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Set.elts); nodeStack.pop(); result = v; break; } case ListComp_kind: { ListComprehensionAst* v = new ListComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.ListComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.ListComp.generators); nodeStack.pop(); result = v; break; } case SetComp_kind: { SetComprehensionAst* v = new SetComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.SetComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.SetComp.generators); nodeStack.pop(); result = v; break; } case DictComp_kind: { DictionaryComprehensionAst* v = new DictionaryComprehensionAst(parent()); nodeStack.push(v); v->key = static_cast(visitNode(node->v.DictComp.key)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.DictComp.value)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.DictComp.generators); nodeStack.pop(); result = v; break; } case GeneratorExp_kind: { GeneratorExpressionAst* v = new GeneratorExpressionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.GeneratorExp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.GeneratorExp.generators); nodeStack.pop(); result = v; break; } case Yield_kind: { YieldAst* v = new YieldAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Yield.value)); nodeStack.pop(); result = v; break; } case Compare_kind: { CompareAst* v = new CompareAst(parent()); nodeStack.push(v); v->leftmostElement = static_cast(visitNode(node->v.Compare.left)); nodeStack.pop(); for ( int _i = 0; _i < node->v.Compare.ops->size; _i++ ) { v->operators.append((ExpressionAst::ComparisonOperatorTypes) node->v.Compare.ops->elements[_i]); } nodeStack.push(v); v->comparands = visitNodeList<_expr, ExpressionAst>(node->v.Compare.comparators); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION < QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop();};if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop();}; result = v; break; } #endif case Num_kind: { NumberAst* v = new NumberAst(parent()); v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n); result = v; break; } case Str_kind: { StringAst* v = new StringAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Str.s); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case JoinedStr_kind: { JoinedStringAst* v = new JoinedStringAst(parent()); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.JoinedStr.values); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case FormattedValue_kind: { FormattedValueAst* v = new FormattedValueAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.FormattedValue.value)); nodeStack.pop(); v->conversion = node->v.FormattedValue.conversion; nodeStack.push(v); v->formatSpec = static_cast(visitNode(node->v.FormattedValue.format_spec)); nodeStack.pop(); result = v; break; } #endif case Bytes_kind: { BytesAst* v = new BytesAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Bytes.s); result = v; break; } case Attribute_kind: { AttributeAst* v = new AttributeAst(parent()); v->attribute = node->v.Attribute.attr ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Attribute.attr)) : nullptr; if ( v->attribute ) { v->attribute->startCol = node->col_offset; v->startCol = v->attribute->startCol; v->attribute->startLine = tline(node->lineno - 1); v->startLine = v->attribute->startLine; v->attribute->endCol = node->col_offset + v->attribute->value.length() - 1; v->endCol = v->attribute->endCol; v->attribute->endLine = tline(node->lineno - 1); v->endLine = v->attribute->endLine; ranges_copied = true; } nodeStack.push(v); v->value = static_cast(visitNode(node->v.Attribute.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Attribute.ctx; result = v; break; } case Subscript_kind: { SubscriptAst* v = new SubscriptAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Subscript.value)); nodeStack.pop(); nodeStack.push(v); v->slice = static_cast(visitNode(node->v.Subscript.slice)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Subscript.ctx; result = v; break; } case Starred_kind: { StarredAst* v = new StarredAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Starred.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Starred.ctx; result = v; break; } case Name_kind: { NameAst* v = new NameAst(parent()); v->identifier = node->v.Name.id ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Name.id)) : nullptr; if ( v->identifier ) { v->identifier->startCol = node->col_offset; v->startCol = v->identifier->startCol; v->identifier->startLine = tline(node->lineno - 1); v->startLine = v->identifier->startLine; v->identifier->endCol = node->col_offset + v->identifier->value.length() - 1; v->endCol = v->identifier->endCol; v->identifier->endLine = tline(node->lineno - 1); v->endLine = v->identifier->endLine; ranges_copied = true; } v->context = (ExpressionAst::Context) node->v.Name.ctx; result = v; break; } case List_kind: { ListAst* v = new ListAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.List.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.List.ctx; result = v; break; } case Tuple_kind: { TupleAst* v = new TupleAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Tuple.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Tuple.ctx; result = v; break; } case Ellipsis_kind: { EllipsisAst* v = new EllipsisAst(parent()); result = v; break; } case NameConstant_kind: { NameConstantAst* v = new NameConstantAst(parent()); v->value = node->v.NameConstant.value == Py_None ? NameConstantAst::None : node->v.NameConstant.value == Py_False ? NameConstantAst::False : NameConstantAst::True; result = v; break; } case YieldFrom_kind: { YieldFromAst* v = new YieldFromAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.YieldFrom.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _expr AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_keyword* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; KeywordAst* v = new KeywordAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; nodeStack.push(v); v->value = static_cast(visitNode(node->value)); nodeStack.pop(); return v; } Ast* visitNode(_slice* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Slice_kind: { SliceAst* v = new SliceAst(parent()); nodeStack.push(v); v->lower = static_cast(visitNode(node->v.Slice.lower)); nodeStack.pop(); nodeStack.push(v); v->upper = static_cast(visitNode(node->v.Slice.upper)); nodeStack.pop(); nodeStack.push(v); v->step = static_cast(visitNode(node->v.Slice.step)); nodeStack.pop(); result = v; break; } case ExtSlice_kind: { ExtendedSliceAst* v = new ExtendedSliceAst(parent()); nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop(); result = v; break; } case Index_kind: { IndexAst* v = new IndexAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Index.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _slice AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_stmt* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Expr_kind: { ExpressionAst* v = new ExpressionAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Expr.value)); nodeStack.pop(); result = v; break; } case FunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.FunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.FunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.FunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.FunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.FunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.FunctionDef.returns)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.AsyncFunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.AsyncFunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.AsyncFunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.AsyncFunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.AsyncFunctionDef.returns)); nodeStack.pop(); v->async = true; result = v; break; } #endif case ClassDef_kind: { ClassDefinitionAst* v = new ClassDefinitionAst(parent()); v->name = node->v.ClassDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ClassDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->baseClasses = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.bases); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ClassDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.decorator_list); nodeStack.pop(); result = v; break; } case Return_kind: { ReturnAst* v = new ReturnAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Return.value)); nodeStack.pop(); result = v; break; } case Delete_kind: { DeleteAst* v = new DeleteAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Delete.targets); nodeStack.pop(); result = v; break; } case Assign_kind: { AssignmentAst* v = new AssignmentAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Assign.targets); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Assign.value)); nodeStack.pop(); result = v; break; } case AugAssign_kind: { AugmentedAssignmentAst* v = new AugmentedAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AugAssign.target)); nodeStack.pop(); v->op = (ExpressionAst::OperatorTypes) node->v.AugAssign.op; nodeStack.push(v); v->value = static_cast(visitNode(node->v.AugAssign.value)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case AnnAssign_kind: { AnnotationAssignmentAst* v = new AnnotationAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AnnAssign.target)); nodeStack.pop(); nodeStack.push(v); v->annotation = static_cast(visitNode(node->v.AnnAssign.annotation)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.AnnAssign.value)); nodeStack.pop(); result = v; break; } #endif case For_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.For.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.For.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.For.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.For.orelse); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFor_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AsyncFor.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.AsyncFor.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFor.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.AsyncFor.orelse); nodeStack.pop(); result = v; break; } #endif case While_kind: { WhileAst* v = new WhileAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.While.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.While.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.While.orelse); nodeStack.pop(); result = v; break; } case If_kind: { IfAst* v = new IfAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.If.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.If.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.If.orelse); nodeStack.pop(); result = v; break; } case With_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.With.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.With.items); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncWith_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncWith.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.AsyncWith.items); nodeStack.pop(); result = v; break; } #endif case Raise_kind: { RaiseAst* v = new RaiseAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.Raise.exc)); nodeStack.pop(); result = v; break; } case Try_kind: { TryAst* v = new TryAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.Try.body); nodeStack.pop(); nodeStack.push(v); v->handlers = visitNodeList<_excepthandler, ExceptionHandlerAst>(node->v.Try.handlers); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.Try.orelse); nodeStack.pop(); nodeStack.push(v); v->finally = visitNodeList<_stmt, Ast>(node->v.Try.finalbody); nodeStack.pop(); result = v; break; } case Assert_kind: { AssertionAst* v = new AssertionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.Assert.test)); nodeStack.pop(); nodeStack.push(v); v->message = static_cast(visitNode(node->v.Assert.msg)); nodeStack.pop(); result = v; break; } case Import_kind: { ImportAst* v = new ImportAst(parent()); nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.Import.names); nodeStack.pop(); result = v; break; } case ImportFrom_kind: { ImportFromAst* v = new ImportFromAst(parent()); v->module = node->v.ImportFrom.module ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ImportFrom.module)) : nullptr; if ( v->module ) { v->module->startCol = node->col_offset; v->startCol = v->module->startCol; v->module->startLine = tline(node->lineno - 1); v->startLine = v->module->startLine; v->module->endCol = node->col_offset + v->module->value.length() - 1; v->endCol = v->module->endCol; v->module->endLine = tline(node->lineno - 1); v->endLine = v->module->endLine; ranges_copied = true; } nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.ImportFrom.names); nodeStack.pop(); v->level = node->v.ImportFrom.level; result = v; break; } case Global_kind: { GlobalAst* v = new GlobalAst(parent()); for ( int _i = 0; _i < node->v.Global.names->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyUnicodeObjectToQString( static_cast(node->v.Global.names->elements[_i]) )); v->names.append(id); } result = v; break; } case Break_kind: { BreakAst* v = new BreakAst(parent()); result = v; break; } case Continue_kind: { ContinueAst* v = new ContinueAst(parent()); result = v; break; } case Pass_kind: { PassAst* v = new PassAst(parent()); result = v; break; } case Nonlocal_kind: { NonlocalAst* v = new NonlocalAst(parent()); result = v; break; } default: qWarning() << "Unsupported _stmt AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_withitem* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; WithItemAst* v = new WithItemAst(parent()); nodeStack.push(v); v->contextExpression = static_cast(visitNode(node->context_expr)); nodeStack.pop(); nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->optional_vars)); nodeStack.pop(); return v; } }; /* * End generated code */ diff --git a/parser/rangefixvisitor.cpp b/parser/rangefixvisitor.cpp new file mode 100644 index 00000000..43c4c627 --- /dev/null +++ b/parser/rangefixvisitor.cpp @@ -0,0 +1,407 @@ +/*************************************************************************** + * This file is part of KDevelop * + * Copyright 2013-2015 Sven Brauch * + * Copyright 2016-2017 Francis Herne * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU Library General Public License as * + * published by the Free Software Foundation; either version 2 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU Library General Public * + * License along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * + ***************************************************************************/ + +#include "rangefixvisitor.h" + +namespace Python { + +class NextAstFindVisitor : public AstDefaultVisitor { +public: + KTextEditor::Cursor findNext(Python::Ast* node) { + m_root = node; + auto parent = node; + while ( parent->parent && parent->parent->isExpression() ) { + parent = parent->parent; + } + visitNode(parent); + + while ( ! m_next.isValid() && parent->parent ) { + // no next expression found in that statement, advance to the next statement + parent = parent->parent; + visitNode(parent); + } + + return m_next; + }; + void visitNode(Python::Ast* node) { + if ( ! node ) { + return; + } + AstDefaultVisitor::visitNode(node); + if ( node->start() > m_root->start() && ! node->isChildOf(m_root) ) { + m_next = (m_next < node->start() && m_next.isValid()) ? m_next : node->start(); + } + } + +private: + KTextEditor::Cursor m_next{-1, -1}; + Ast* m_root; +}; + + +//BEGIN RangeFixVisitor +void RangeFixVisitor::visitNode(Ast* node) { + AstDefaultVisitor::visitNode(node); + if ( node && node->parent && node->parent->astType != Ast::AttributeAstType ) { + if ( ( node->parent->endLine <= node->endLine && node->parent->endCol <= node->endCol ) + || node->parent->endLine < node->endLine ) + { + node->parent->endLine = node->endLine; + node->parent->endCol = node->endCol; + } + } +}; + +void RangeFixVisitor::visitFunctionDefinition(FunctionDefinitionAst* node) { + cutDefinitionPreamble(node->name, node->async ? "asyncdef" : "def"); + AstDefaultVisitor::visitFunctionDefinition(node); +}; + +void RangeFixVisitor::visitClassDefinition(ClassDefinitionAst* node) { + cutDefinitionPreamble(node->name, "class"); + AstDefaultVisitor::visitClassDefinition(node); +}; + +void RangeFixVisitor::visitAttribute(AttributeAst* node) { + // Work around the weird way to count columns in Python's AST module. + + // Find where the next expression (of any kind) behind this one starts + NextAstFindVisitor v; + auto next_start = v.findNext(node); + if ( ! next_start.isValid() ) { + // use end of document as reference + next_start = {lines.size() - 1, lines.last().size() - 1}; + } + + // take only the portion of the line up to that next expression + auto endLine = next_start.line(); + auto endCol = next_start.column(); + if ( ! (next_start > node->start()) ) { + endLine = node->startLine; + endCol = -1; + } + + const QString& name(node->attribute->value); + + QString line; + for ( int n = node->startLine, + pos = node->value->endCol + 1, + dotFound = false, + nameFound = false; + n <= endLine; ++n, pos = 0 ) { + line = lines.at(n); + if ( n == endLine && endCol != -1 ) { + // Never look at the next expression. + line = line.left(endCol); + } + if ( !dotFound ) { + // The real attr name can never be before a dot. + // Nor can the start of a comment. + // (Don't be misled by `foo["bar"].bar` or `foo["#"].bar`) + pos = line.indexOf('.', pos); + if ( pos == -1 ) continue; + dotFound = true; + } + if ( !nameFound ) { + // Track if the attr name has appeared at least once. + // This helps avoid RangeFixVisitor::interpreting '#'s in strings as comments - + // there can never be a comment before the real attr name. + pos = line.indexOf(name, pos + 1); + if ( pos == -1 ) continue; + nameFound = true; + } + if ( dotFound && nameFound && + (pos = line.indexOf('#', pos + name.length())) != -1) { + // Remove the comment after a '#' iff we're certain it can't + // be inside a string literal (e.g. `foo["#"].bar`). + line = line.left(pos); + } + // Take the last occurrence, any others are in string literals. + pos = line.lastIndexOf(name); + if ( pos != -1 ) { + node->startLine = n; + node->startCol = pos; + } + // N.B. we do this for all lines, the last non-comment occurrence + // is the real one. + } + // This fails (only, AFAIK) in a very limited case: + // If the value expression (`foo` in `foo.bar`) contains a dot, the + // attr name, _and_ a hash in that order (may not be consecutive), + // and the hash is on the same line as the real attr name, + // we wrongly interpret the hash as the start of a comment. + // e.g `foo["...barrier#"].bar` will highlight part of the string. + + node->endLine = node->startLine; + node->endCol = node->startCol + name.length() - 1; + node->attribute->copyRange(node); + + AstDefaultVisitor::visitAttribute(node); +}; + +// alias for imports (import foo as bar, baz as bang) +// no strings, brackets, or whatever are allowed here, so the "parser" +// can be very straightforward. +void RangeFixVisitor::visitImport(ImportAst* node) { + AstDefaultVisitor::visitImport(node); + int aliasIndex = 0; + foreach ( AliasAst* alias, node->names ) { + fixAlias(alias->name, alias->asName, node->startLine, aliasIndex); + aliasIndex += 1; + } +}; + +// alias for exceptions (except FooBarException as somethingterriblehappened: ...) +void RangeFixVisitor::visitExceptionHandler(ExceptionHandlerAst* node) { + AstDefaultVisitor::visitExceptionHandler(node); + if ( ! node->name ) { + return; + } + const QString& line = lines.at(node->startLine); + const int end = line.count() - 1; + int back = backtrackDottedName(line, end); + node->name->startCol = end - back; + node->name->endCol = end; +} + +void RangeFixVisitor::visitString(Python::StringAst* node) { + AstDefaultVisitor::visitString(node); + auto match = findString.match(lines.at(node->startLine), node->startCol); + if ( match.capturedLength() > 0 ) { + node->endCol += match.capturedLength() - 1; // Ranges are inclusive. + } +} +void RangeFixVisitor::visitBytes(Python::BytesAst* node) { + AstDefaultVisitor::visitBytes(node); + auto match = findString.match(lines.at(node->startLine), node->startCol + 1); + if ( match.capturedLength() > 0 ) { + node->endCol += match.capturedLength(); // -1 then +1, because of the 'b'. + } +} +void RangeFixVisitor::visitFormattedValue(Python::FormattedValueAst * node) { + AstDefaultVisitor::visitFormattedValue(node); + auto match = findString.match(lines.at(node->startLine), node->startCol + 1); + if ( match.capturedLength() > 0 ) { + node->endCol += match.capturedLength(); + } +} + +void RangeFixVisitor::visitNumber(Python::NumberAst* node) { + AstDefaultVisitor::visitNumber(node); + auto match = findNumber.match(lines.at(node->startLine), node->startCol); + if ( match.capturedLength() > 0 ) { + node->endCol += match.capturedLength() - 1; // Ranges are inclusive. + } +} + +// Add one column after the last child to cover the closing bracket: `[1,2,3]` +// TODO This is still wrong if the last child is followed by parens or whitespace. +// endCol matters most in single-line expressions, so this isn't a huge problem. +void RangeFixVisitor::visitSubscript(Python::SubscriptAst* node) { + AstDefaultVisitor::visitSubscript(node); + node->endCol++; +} +void RangeFixVisitor::visitComprehension(Python::ComprehensionAst* node) { + AstDefaultVisitor::visitComprehension(node); + node->endCol++; +} +void RangeFixVisitor::visitList(Python::ListAst* node) { + AstDefaultVisitor::visitList(node); + node->endCol++; +} +void RangeFixVisitor::visitTuple(Python::TupleAst* node) { + AstDefaultVisitor::visitTuple(node); + node->endCol++; +} + +// skip the decorators and the "def" at the beginning +// of a class or function declaration and modify @arg node +// example: +// @decorate(foo) +// @decorate(bar) +// class myclass(parent): pass +// before: start of class->name is [0, 0] +// after: start of class->name is [2, 5] +// line continuation characters are not supported, +// because code needing those in this case is not worth being supported. +void RangeFixVisitor::cutDefinitionPreamble(Ast* fixNode, const QString& defKeyword) { + if ( ! fixNode ) { + return; + } + int currentLine = fixNode->startLine; + + // cut away decorators + while ( currentLine < lines.size() ) { + if ( lines.at(currentLine).trimmed().remove(' ').remove('\t').startsWith(defKeyword) ) { + // it's not a decorator, so stop skipping lines. + break; + } + currentLine += 1; + } +// qDebug() << "FIX:" << fixNode->range(); + fixNode->startLine = currentLine; + fixNode->endLine = currentLine; +// qDebug() << "FIXED:" << fixNode->range() << fixNode->astType; + + // cut away the "def" / "class" + int currentColumn = -1; + if ( currentLine > lines.size() ) { + // whops? + return; + } + const QString& lineData = lines.at(currentLine); + bool keywordFound = false; + while ( currentColumn < lineData.size() - 1 ) { + currentColumn += 1; + if ( lineData.at(currentColumn).isSpace() ) { + // skip space at the beginning of the line + continue; + } + else if ( keywordFound ) { + // if the "def" / "class" was already found, and the current char is + // non space, then this is indeed the start of the identifier we're looking for. + break; + } + else { + keywordFound = true; + currentColumn += defKeyword.size(); + } + } + const int previousLength = fixNode->endCol - fixNode->startCol; + fixNode->startCol = currentColumn; + fixNode->endCol = currentColumn + previousLength; +}; + +int RangeFixVisitor::backtrackDottedName(const QString& data, const int start) { + bool haveDot = true; + bool previousWasSpace = true; + for ( int i = start - 1; i >= 0; i-- ) { + if ( data.at(i).isSpace() ) { + previousWasSpace = true; + continue; + } + if ( data.at(i) == ':' ) { + // excepthandler + continue; + } + if ( data.at(i) == '.' ) { + haveDot = true; + } + else if ( haveDot ) { + haveDot = false; + previousWasSpace = false; + continue; + } + if ( previousWasSpace && ! haveDot ) { + return start-i-2; + } + previousWasSpace = false; + } + return 0; +} + +void RangeFixVisitor::fixAlias(Ast* dotted, Ast* asname, const int startLine, int aliasIndex) { + if ( ! asname && ! dotted ) { + return; + } + QString line = lines.at(startLine); + int lineno = startLine; + for ( int i = 0; i < line.size(); i++ ) { + const QChar& current = line.at(i); + if ( current == '\\' ) { + // line continuation character + // splitting like "import foo as \ \n bar" is not supported. + lineno += 1; + line = lines.at(lineno); + i = 0; + continue; + } + if ( current == ',' ) { + if ( aliasIndex == 0 ) { + // nothing found, continue below + line = line.left(i); + break; + } + // next alias expression + aliasIndex -= 1; + } + if ( i > line.length() - 3 ) { + continue; + } + if ( current.isSpace() && line.mid(i+1).startsWith("as") && ( line.at(i+3).isSpace() || line.at(i+3) == '\\' ) ) { + // there's an "as" + if ( aliasIndex == 0 ) { + // it's the one we're looking for + // find the expression + if ( dotted ) { + int dottedNameLength = backtrackDottedName(line, i); + dotted->startLine = lineno; + dotted->endLine = lineno; + dotted->startCol = i-dottedNameLength; + dotted->endCol = i; + } + // find the asname + if ( asname ) { + bool atStart = true; + int textStart = i+3; + for ( int j = i+3; j < line.size(); j++ ) { + if ( atStart && ! line.at(j).isSpace() ) { + atStart = false; + textStart = j; + } + if ( ! atStart && ( line.at(j).isSpace() || j == line.size() - 1 ) ) { + // found it + asname->startLine = lineno; + asname->endLine = lineno; + asname->startCol = textStart - 1; + asname->endCol = j; + } + } + } + return; + } + } + } + // no "as" found, use last dotted name in line + const int end = line.count() - whitespaceAtEnd(line); + int back = backtrackDottedName(line, end); + dotted->startLine = lineno; + dotted->endLine = lineno; + dotted->startCol = end - back; + dotted->endCol = end; +}; + +int RangeFixVisitor::whitespaceAtEnd(const QString& line) { + for ( int i = 0; i < line.size(); i++ ) { + if ( ! line.at(line.size() - i - 1).isSpace() ) { + return i; + } + } + return 0; +}; + +// FIXME This doesn't work for triple-quoted strings +// (it gives length 2, which is no worse than before). +const QRegularExpression RangeFixVisitor::findString("\\G(['\"]).*?(? * + * Copyright 2016-2017 Francis Herne * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU Library General Public License as * + * published by the Free Software Foundation; either version 2 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU Library General Public * + * License along with this program; if not, write to the * + * Free Software Foundation, Inc., * + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * + ***************************************************************************/ + +#ifndef RANGEFIXVISITOR_H +#define RANGEFIXVISITOR_H + +#include + +#include "astdefaultvisitor.h" + +namespace Python { + +/** + * This visitor attempts to give each AST node the correct range, + * with workarounds for unset or incorrect (for our purposes) ranges + * that are supplied by the CPython parser. + */ +class KDEVPYTHONPARSER_NO_EXPORT RangeFixVisitor : public AstDefaultVisitor { +public: + RangeFixVisitor(const QString& contents) : lines(contents.split('\n')) {}; + void visitNode(Ast* node) override; + void visitFunctionDefinition(FunctionDefinitionAst* node) override; + void visitClassDefinition(ClassDefinitionAst* node) override; + void visitAttribute(AttributeAst* node) override; + void visitImport(ImportAst* node) override; + void visitExceptionHandler(ExceptionHandlerAst* node) override; + void visitString(Python::StringAst* node) override; + void visitBytes(Python::BytesAst* node) override; + void visitFormattedValue(Python::FormattedValueAst * node) override; + void visitNumber(Python::NumberAst* node) override; + void visitSubscript(Python::SubscriptAst* node) override; + void visitComprehension(Python::ComprehensionAst* node) override; + void visitList(Python::ListAst* node) override; + void visitTuple(Python::TupleAst* node) override; + +private: + void cutDefinitionPreamble(Ast* fixNode, const QString& defKeyword); + int backtrackDottedName(const QString& data, const int start); + void fixAlias(Ast* dotted, Ast* asname, const int startLine, int aliasIndex); + int whitespaceAtEnd(const QString& line); + + const QStringList lines; + QVector dots; + KTextEditor::Cursor attributeStart; + static const QRegularExpression findString; + static const QRegularExpression findNumber; +}; + +} + +#endif // RANGEFIXVISITOR_H