diff --git a/debugger/variablecontroller.cpp b/debugger/variablecontroller.cpp index feb67e14..0c762800 100644 --- a/debugger/variablecontroller.cpp +++ b/debugger/variablecontroller.cpp @@ -1,176 +1,176 @@ /* This file is part of kdev-python, the python language plugin for KDevelop Copyright (C) 2012 Sven Brauch This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ #include "variablecontroller.h" #include "variable.h" #include "debugsession.h" #include "pdbframestackmodel.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "debuggerdebug.h" using namespace KDevelop; namespace Python { VariableController::VariableController(IDebugSession* parent) : IVariableController(parent) { m_updateTimer.setSingleShot(true); m_updateTimer.setInterval(100); QObject::connect(&m_updateTimer, &QTimer::timeout, this, &VariableController::_update); } void VariableController::addWatch(KDevelop::Variable* variable) { variableCollection()->watches()->add(variable->expression()); } void VariableController::addWatchpoint(KDevelop::Variable* /*variable*/) { qCWarning(KDEV_PYTHON_DEBUGGER) << "addWatchpoint requested (not implemented)"; } void VariableController::handleEvent(IDebugSession::event_t event) { if ( event == IDebugSession::thread_or_frame_changed ) { DebugSession* s = static_cast(session()); PdbFrameStackModel* model = static_cast(s->frameStackModel()); int delta = model->currentFrame() - model->debuggerAtFrame(); model->setDebuggerAtFrame(model->currentFrame()); bool positive = delta > 0; qCDebug(KDEV_PYTHON_DEBUGGER) << "changing frame by" << delta; for ( int i = delta; i != 0; i += ( positive ? -1 : 1 ) ) { qCDebug(KDEV_PYTHON_DEBUGGER) << ( positive ? "up" : "down" ) << model->currentFrame() << model->debuggerAtFrame(); s->addSimpleInternalCommand(positive ? "up" : "down"); } } KDevelop::IVariableController::handleEvent(event); } KDevelop::Variable* VariableController::createVariable(KDevelop::TreeModel* model, KDevelop::TreeItem* parent, const QString& expression, const QString& display) { return new Variable(model, parent, expression, display); } KTextEditor::Range VariableController::expressionRangeUnderCursor(KTextEditor::Document* doc, const KTextEditor::Cursor& cursor) { QString prefix; DUChainReadLocker lock; if ( ! doc->isModified() ) { if ( TopDUContext* context = DUChain::self()->chainForDocument(doc->url()) ) { DUContext* contextAtCursor = context->findContextAt(CursorInRevision(cursor.line(), cursor.column())); if ( contextAtCursor && contextAtCursor->type() == DUContext::Class ) { if ( contextAtCursor->owner() && ! contextAtCursor->owner()->identifier().isEmpty() ) { prefix = contextAtCursor->owner()->identifier().toString() + "."; } } } } else { qCDebug(KDEV_PYTHON_DEBUGGER) << "duchain unavailable for document" << doc->url() << "or document out of date"; } TextDocumentLazyLineFetcher linefetcher(doc); KTextEditor::Cursor startCursor; auto text = prefix + CodeHelpers::expressionUnderCursor(linefetcher, cursor, startCursor); return {startCursor, startCursor + KTextEditor::Cursor{0, text.length()}}; } void VariableController::localsUpdateReady(QByteArray rawData) { QRegExp formatExtract("([a-zA-Z0-9_]+) \\=\\> (.*)"); QList data = rawData.split('\n'); data.removeAll({}); qCDebug(KDEV_PYTHON_DEBUGGER) << "locals update:" << data; int i = 0; QStringList vars; QMap values; while ( i < data.length() ) { QByteArray d = data.at(i); if ( formatExtract.exactMatch(d) ) { QString key = formatExtract.capturedTexts().at(1); vars << key; values[key] = formatExtract.capturedTexts().at(2); } else qCWarning(KDEV_PYTHON_DEBUGGER) << "mismatch:" << d; i++; } QList variableObjects = KDevelop::ICore::self()->debugController()->variableCollection() ->locals()->updateLocals(vars); for ( int i = 0; i < variableObjects.length(); i++ ) { KDevelop::Variable* v = variableObjects[i]; auto model = v->model(); auto parent = model->indexForItem(v, 0); auto childCount = v->model()->rowCount(parent); - qDebug() << "updating:" << v->expression() << "active children:" << childCount; + qCDebug(KDEV_PYTHON_DEBUGGER) << "updating:" << v->expression() << "active children:" << childCount; for ( int j = 0; j < childCount; j++ ) { auto index = model->index(j, 0, parent); auto child = static_cast(index.internalPointer()); if ( auto childVariable = qobject_cast(child) ) { - qDebug() << " got child var:" << childVariable->expression(); + qCDebug(KDEV_PYTHON_DEBUGGER) << " got child var:" << childVariable->expression(); v->fetchMoreChildren(); break; } } v->setValue(values[v->expression()]); v->setHasMoreInitial(true); } } void VariableController::update() { m_updateTimer.start(); } void VariableController::_update() { - qDebug() << " ************************* update requested"; + qCDebug(KDEV_PYTHON_DEBUGGER) << " ************************* update requested"; DebugSession* d = static_cast(parent()); if (autoUpdate() & UpdateWatches) { variableCollection()->watches()->reinstall(); } if (autoUpdate() & UpdateLocals) { // TODO find a more elegant solution for this import! InternalPdbCommand* import = new InternalPdbCommand(0, 0, "import __kdevpython_debugger_utils\n"); InternalPdbCommand* cmd = new InternalPdbCommand(this, "localsUpdateReady", "__kdevpython_debugger_utils.format_locals(__kdevpython_debugger_utils.__kdevpython_builtin_locals())\n"); d->addCommand(import); d->addCommand(cmd); } } } diff --git a/duchain/contextbuilder.cpp b/duchain/contextbuilder.cpp index 781c6c46..430cadcd 100644 --- a/duchain/contextbuilder.cpp +++ b/duchain/contextbuilder.cpp @@ -1,485 +1,485 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "pythonlanguagesupport.h" #include "pythoneditorintegrator.h" #include "dumpchain.h" #include "usebuilder.h" #include "contextbuilder.h" #include "pythonducontext.h" #include "pythonparsejob.h" #include "declarationbuilder.h" #include "helpers.h" #include "duchaindebug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace KDevelop; using namespace KTextEditor; namespace Python { ReferencedTopDUContext ContextBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { if (!updateContext) { DUChainReadLocker lock(DUChain::lock()); updateContext = DUChain::self()->chainForDocument(url); if ( updateContext ) { Q_ASSERT(updateContext->type() == DUContext::Global); } } if (updateContext) { - qDebug() << " ====> DUCHAIN ====> rebuilding duchain for" << url.str() << "(was built before)"; + qCDebug(KDEV_PYTHON_DUCHAIN) << " ====> DUCHAIN ====> rebuilding duchain for" << url.str() << "(was built before)"; DUChainWriteLocker lock(DUChain::lock()); Q_ASSERT(updateContext->type() == DUContext::Global); updateContext->clearImportedParentContexts(); updateContext->parsingEnvironmentFile()->clearModificationRevisions(); updateContext->clearProblems(); } else { - qDebug() << " ====> DUCHAIN ====> building duchain for" << url.str(); + qCDebug(KDEV_PYTHON_DUCHAIN) << " ====> DUCHAIN ====> building duchain for" << url.str(); } return ContextBuilderBase::build(url, node, updateContext); } PythonEditorIntegrator* ContextBuilder::editor() const { return m_editor; } IndexedString ContextBuilder::currentlyParsedDocument() const { return m_currentlyParsedDocument; } void ContextBuilder::setCurrentlyParsedDocument(const IndexedString &document) { m_currentlyParsedDocument = document; } void ContextBuilder::setFutureModificationRevision(const ModificationRevision &rev) { m_futureModificationRevision = rev; } RangeInRevision ContextBuilder::rangeForNode(Ast* node, bool moveRight) { return RangeInRevision(node->startLine, node->startCol, node->endLine, node->endCol + (int) moveRight); } RangeInRevision ContextBuilder::rangeForNode(Identifier* node, bool moveRight) { return rangeForNode(static_cast(node), moveRight); } TopDUContext* ContextBuilder::newTopContext(const RangeInRevision& range, ParsingEnvironmentFile* file) { IndexedString currentDocumentUrl = currentlyParsedDocument(); if ( !file ) { file = new ParsingEnvironmentFile(currentDocumentUrl); file->setLanguage(IndexedString("python")); } TopDUContext* top = new PythonTopDUContext(currentDocumentUrl, range, file); ReferencedTopDUContext ref(top); m_topContext = ref; return top; } DUContext* ContextBuilder::newContext(const RangeInRevision& range) { return new PythonNormalDUContext(range, currentContext()); } void ContextBuilder::addUnresolvedImport(const IndexedString &module) { m_unresolvedImports.append(module); } QList ContextBuilder::unresolvedImports() const { return m_unresolvedImports; } void ContextBuilder::setEditor(PythonEditorIntegrator* editor) { m_editor = editor; } void ContextBuilder::startVisiting(Ast* node) { visitNode(node); } void ContextBuilder::setContextOnNode(Ast* node, DUContext* context) { node->context = context; } DUContext* ContextBuilder::contextFromNode(Ast* node) { return node->context; } RangeInRevision ContextBuilder::editorFindRange(Ast* fromNode, Ast* toNode) { return editor()->findRange(fromNode, toNode); } CursorInRevision ContextBuilder::editorFindPositionSafe(Ast* node) { if ( !node ) { return CursorInRevision::invalid(); } return editor()->findPosition(node); } CursorInRevision ContextBuilder::startPos( Ast* node ) { return m_editor->findPosition(node, PythonEditorIntegrator::FrontEdge); } QualifiedIdentifier ContextBuilder::identifierForNode( Python::Identifier* node ) { return QualifiedIdentifier(node->value); } void ContextBuilder::addImportedContexts() { if ( compilingContexts() && !m_importedParentContexts.isEmpty() ) { DUChainWriteLocker lock( DUChain::lock() ); foreach( DUContext* imported, m_importedParentContexts ) currentContext()->addImportedParentContext( imported ); m_importedParentContexts.clear(); } } void ContextBuilder::closeAlreadyOpenedContext(DUContextPointer context) { Q_ASSERT(currentContext() == context.data()); while ( ! m_temporarilyClosedContexts.isEmpty() ) { openContext(m_temporarilyClosedContexts.last().data()); m_temporarilyClosedContexts.removeLast(); } } void ContextBuilder::activateAlreadyOpenedContext(DUContextPointer context) { Q_ASSERT(m_temporarilyClosedContexts.isEmpty()); Q_ASSERT(contextAlreayOpen(context)); DUContext* current = currentContext(); bool reallyCompilingContexts = compilingContexts(); setCompilingContexts(false); // TODO this is very hackish. while ( current ) { if ( current == context.data() ) { setCompilingContexts(reallyCompilingContexts); return; } m_temporarilyClosedContexts.append(DUContextPointer(current)); closeContext(); current = currentContext(); } setCompilingContexts(reallyCompilingContexts); } bool ContextBuilder::contextAlreayOpen(DUContextPointer context) { DUContext* current = currentContext(); while ( current ) { if ( context.data() == current ) return true; current = current->parentContext(); } return false; } void ContextBuilder::visitListComprehension(ListComprehensionAst* node) { visitComprehensionCommon(node); } void ContextBuilder::visitDictionaryComprehension(DictionaryComprehensionAst* node) { visitComprehensionCommon(node); } void ContextBuilder::visitGeneratorExpression(GeneratorExpressionAst* node) { visitComprehensionCommon(node); } RangeInRevision ContextBuilder::comprehensionRange(Ast* node) { RangeInRevision range = editorFindRange(node, node); return range; } void ContextBuilder::visitComprehensionCommon(Ast* node) { RangeInRevision range = comprehensionRange(node); Q_ASSERT(range.isValid()); if ( range.isValid() ) { DUChainWriteLocker lock; openContext(node, range, KDevelop::DUContext::Other); qCDebug(KDEV_PYTHON_DUCHAIN) << "creating comprehension context" << node << range; Q_ASSERT(currentContext()); // currentContext()->setLocalScopeIdentifier(QualifiedIdentifier("")); lock.unlock(); if ( node->astType == Ast::DictionaryComprehensionAstType ) Python::AstDefaultVisitor::visitDictionaryComprehension(static_cast(node)); if ( node->astType == Ast::ListComprehensionAstType ) Python::AstDefaultVisitor::visitListComprehension(static_cast(node)); if ( node->astType == Ast::GeneratorExpressionAstType ) Python::AstDefaultVisitor::visitGeneratorExpression(static_cast(node)); if ( node->astType == Ast::SetComprehensionAstType ) Python::AstDefaultVisitor::visitSetComprehension(static_cast(node)); lock.lock(); closeContext(); } } void ContextBuilder::openContextForClassDefinition(ClassDefinitionAst* node) { // make sure the contexts ends at the next DEDENT token, not at the last statement. // also, make the context begin *after* the parent list and class name. int endLine = editor()->indent()->nextChange(node->endLine, FileIndentInformation::Dedent); CursorInRevision start = CursorInRevision(node->body.first()->startLine, node->body.first()->startCol); if ( start.line > node->startLine ) { start = CursorInRevision(node->startLine + 1, 0); } RangeInRevision range(start, CursorInRevision(endLine, 0)); DUChainWriteLocker lock; openContext(node, range, DUContext::Class, node->name); currentContext()->setLocalScopeIdentifier(identifierForNode(node->name)); lock.unlock(); addImportedContexts(); } void ContextBuilder::visitClassDefinition( ClassDefinitionAst* node ) { openContextForClassDefinition(node); Python::AstDefaultVisitor::visitClassDefinition(node); closeContext(); } void ContextBuilder::visitCode(CodeAst* node) { auto doc_url = Helper::getDocumentationFile(); IndexedString doc = IndexedString(doc_url); Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); if ( currentlyParsedDocument() != doc ) { // Search for the python built-in functions file, and dump its contents into the current file. auto internal = Helper::getDocumentationFileContext(); if ( ! internal ) { // If the built-in functions file is not yet parsed, schedule it with a high priority. m_unresolvedImports.append(doc); KDevelop::ICore::self()->languageController()->backgroundParser() ->addDocument(doc, KDevelop::TopDUContext::ForceUpdate, BackgroundParser::BestPriority*2, 0, ParseJob::FullSequentialProcessing); // This must NOT be called from parse threads! It's only meant to be used from the foreground thread, and will // cause thread starvation if called from here. // KDevelop::ICore::self()->languageController()->backgroundParser()->parseDocuments(); } else { DUChainWriteLocker wlock; currentContext()->addImportedParentContext(internal); } } AstDefaultVisitor::visitCode(node); } QPair ContextBuilder::findModulePath(const QString& name, const QUrl& currentDocument) { QStringList nameComponents = name.split("."); QVector searchPaths; if ( name.startsWith('.') ) { /* To take care for imports like "from ....xxxx.yyy import zzz" * we need to take current doc path and run "cd .." enough times */ nameComponents.removeFirst(); QString tname = name.mid(1); // remove first dot QDir curPathDir = QDir(currentDocument.adjusted(QUrl::RemoveFilename).toLocalFile()); foreach(QString c, tname) { if (c != ".") break; curPathDir.cdUp(); nameComponents.removeFirst(); } searchPaths << QUrl::fromLocalFile(curPathDir.path()); } else { // If this is not a relative import, use the project directory, // the current directory, and all system include paths. // FIXME: If absolute imports enabled, don't add curently parsed doc path searchPaths = Helper::getSearchPaths(currentDocument); } // Loop over all the name components, and find matching folders or files. QDir tmp; QStringList leftNameComponents; foreach ( const QUrl& currentPath, searchPaths ) { tmp.setPath(currentPath.toLocalFile()); leftNameComponents = nameComponents; foreach ( QString component, nameComponents ) { if ( component == "*" ) { // For "from ... import *", if "..." is a directory, use the "__init__.py" file component = QStringLiteral("__init__"); } else { // only empty the list if not importing *, this is convenient later on leftNameComponents.removeFirst(); } QString testFilename = tmp.path() + "/" + component; bool can_continue = tmp.cd(component); QFileInfo sourcedir(testFilename); const bool dir_exists = sourcedir.exists() && sourcedir.isDir(); // we can only parse those, so we don't care about anything else for now. // Any C modules (.so, .dll) will be ignored, and highlighted as "not found". TODO fix this static QStringList valid_extensions{".py", ".pyx"}; foreach ( const auto& extension, valid_extensions ) { QFile sourcefile(testFilename + extension); if ( ! dir_exists || leftNameComponents.isEmpty() ) { // If the search cannot continue further down into a hierarchy of directories, // the file matching the next name component will be returned, // toegether with a list of names which must be resolved inside that file. if ( sourcefile.exists() ) { auto sourceUrl = QUrl::fromLocalFile(testFilename + extension); // TODO QUrl: cleanPath? return qMakePair(sourceUrl, leftNameComponents); } else if ( dir_exists ) { auto path = QUrl::fromLocalFile(testFilename + "/__init__.py"); // TODO QUrl: cleanPath? return qMakePair(path, leftNameComponents); } } } if ( ! can_continue ) { // if not returned yet and the cd into the next component failed, // abort and try the next search path. break; } } } return {}; } RangeInRevision ContextBuilder::rangeForArgumentsContext(FunctionDefinitionAst* node) { auto start = node->name->range().end(); auto end = start; auto args = node->arguments; if ( args->kwarg ) { end = args->kwarg->range().end(); } else if ( args->vararg && ( args->arguments.isEmpty() || ! args->vararg->appearsBefore(args->arguments.last())) ) { end = args->vararg->range().end(); } else if ( ! args->arguments.isEmpty() ) { end = args->arguments.last()->range().end(); } if ( ! args->defaultValues.isEmpty() ) { end = qMax(args->defaultValues.last()->range().end(), end); } RangeInRevision range(start.line(), start.column(), end.line(), end.column()); range.start.column += 1; // Don't include end of name. range.end.column += 1; // Include end parenthesis (unless spaces...) return range; } void ContextBuilder::visitFunctionArguments(FunctionDefinitionAst* node) { RangeInRevision range = rangeForArgumentsContext(node); // The DUChain expects the context containing a function's arguments to be of type Function. // The function body will have DUContext::Other as type, as it contains only code. DUContext* funcctx = openContext(node->arguments, range, DUContext::Function, node->name); AstDefaultVisitor::visitArguments(node->arguments); visitArguments(node->arguments); closeContext(); // the parameters should be visible in the function body, so import that context there m_importedParentContexts.append(funcctx); m_mostRecentArgumentsContext = DUContextPointer(funcctx); } void ContextBuilder::visitFunctionDefinition(FunctionDefinitionAst* node) { visitNodeList(node->decorators); visitNode(node->returns); visitFunctionArguments(node); visitFunctionBody(node); } void ContextBuilder::visitFunctionBody(FunctionDefinitionAst* node) { // The function should end at the next DEDENT token, not at the body's last statement int endLine = node->endLine; if ( ! node->body.isEmpty() ) { endLine = node->body.last()->startLine; } if ( node->endLine != node->startLine ) { endLine = editor()->indent()->nextChange(endLine, FileIndentInformation::Dedent); if ( ! node->body.isEmpty() ) { endLine = qMax(endLine, node->body.last()->endLine + 1); } } CursorInRevision end = CursorInRevision(endLine, node->startLine == node->endLine ? INT_MAX : 0); CursorInRevision start = rangeForArgumentsContext(node).end; if ( start.line < node->body.first()->startLine ) { start = CursorInRevision(node->startLine + 1, 0); } RangeInRevision range(start, end); // Open the context for the function body (the list of statements) // It's of type Other, as it contains only code openContext(node, range, DUContext::Other, identifierForNode(node->name)); { DUChainWriteLocker lock; currentContext()->setLocalScopeIdentifier(identifierForNode(node->name)); } // import the parameters into the function body addImportedContexts(); visitNodeList(node->body); closeContext(); m_mostRecentArgumentsContext = DUContextPointer(0); } } diff --git a/parser/astbuilder.cpp b/parser/astbuilder.cpp index 475c93e1..630e8aa3 100644 --- a/parser/astbuilder.cpp +++ b/parser/astbuilder.cpp @@ -1,765 +1,765 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astbuilder.h" #include "ast.h" #include #include #include #include #include #include #include #include #include "python_header.h" #include "astdefaultvisitor.h" #include "cythonsyntaxremover.h" #include #include #include #include #include #include "parserdebug.h" using namespace KDevelop; extern grammar _PyParser_Grammar; namespace Python { class NextAstFindVisitor : public AstDefaultVisitor { public: KTextEditor::Cursor findNext(Python::Ast* node) { m_root = node; auto parent = node; while ( parent->parent && parent->parent->isExpression() ) { parent = parent->parent; } visitNode(parent); while ( ! m_next.isValid() && parent->parent ) { // no next expression found in that statement, advance to the next statement parent = parent->parent; visitNode(parent); } return m_next; }; void visitNode(Python::Ast* node) override { if ( ! node ) { return; } AstDefaultVisitor::visitNode(node); if ( node->start() > m_root->start() && ! node->isChildOf(m_root) ) { m_next = (m_next < node->start() && m_next.isValid()) ? m_next : node->start(); } } private: KTextEditor::Cursor m_next{-1, -1}; Ast* m_root; }; // This class is used to fix some of the remaining issues // with the ranges of objects obtained from the python parser. // Issues addressed are: // 1) decorators and "def" / "class" statements for classes / functions // 2) ranges of aliases // Both issues are easy to correct since the possible syntax is very restricted // (no bracket matching, no strings, no nesting, ...) // For the aliases, fortunately only imports and excepthandlers are affected; // the "with" statement, which has more complicated syntax, provides // the necessary information already. // 3) attribute rangees // Not so easy, but when starting from the end of the expression it works okay class RangeFixVisitor : public AstDefaultVisitor { public: RangeFixVisitor(const QString& contents) : lines(contents.split('\n')) { }; void visitNode(Ast* node) override { AstDefaultVisitor::visitNode(node); if ( node && node->parent && node->parent->astType != Ast::AttributeAstType ) { if ( ( node->parent->endLine <= node->endLine && node->parent->endCol <= node->endCol ) || node->parent->endLine < node->endLine ) { node->parent->endLine = node->endLine; node->parent->endCol = node->endCol; } } }; void visitFunctionDefinition(FunctionDefinitionAst* node) override { cutDefinitionPreamble(node->name, node->async ? "asyncdef" : "def"); AstDefaultVisitor::visitFunctionDefinition(node); }; void visitClassDefinition(ClassDefinitionAst* node) override { cutDefinitionPreamble(node->name, "class"); AstDefaultVisitor::visitClassDefinition(node); }; void visitAttribute(AttributeAst* node) override { // Work around the weird way to count columns in Python's AST module. // Find where the next expression (of any kind) behind this one starts NextAstFindVisitor v; auto next_start = v.findNext(node); if ( ! next_start.isValid() ) { // use end of document as reference next_start = {lines.size() - 1, lines.last().size() - 1}; } // take only the portion of the line up to that next expression auto endLine = next_start.line(); auto endCol = next_start.column(); if ( ! (next_start > node->start()) ) { endLine = node->startLine; endCol = -1; } const QString& name(node->attribute->value); QString line; for ( int n = node->startLine, pos = node->value->endCol + 1, dotFound = false, nameFound = false; n <= endLine; ++n, pos = 0 ) { line = lines.at(n); if ( n == endLine && endCol != -1 ) { // Never look at the next expression. line = line.left(endCol); } if ( !dotFound ) { // The real attr name can never be before a dot. // Nor can the start of a comment. // (Don't be misled by `foo["bar"].bar` or `foo["#"].bar`) pos = line.indexOf('.', pos); if ( pos == -1 ) continue; dotFound = true; } if ( !nameFound ) { // Track if the attr name has appeared at least once. // This helps avoid interpreting '#'s in strings as comments - // there can never be a comment before the real attr name. pos = line.indexOf(name, pos + 1); if ( pos == -1 ) continue; nameFound = true; } if ( dotFound && nameFound && (pos = line.indexOf('#', pos + name.length())) != -1) { // Remove the comment after a '#' iff we're certain it can't // be inside a string literal (e.g. `foo["#"].bar`). line = line.left(pos); } // Take the last occurrence, any others are in string literals. pos = line.lastIndexOf(name); if ( pos != -1 ) { node->startLine = n; node->startCol = pos; } // N.B. we do this for all lines, the last non-comment occurrence // is the real one. } // This fails (only, AFAIK) in a very limited case: // If the value expression (`foo` in `foo.bar`) contains a dot, the // attr name, _and_ a hash in that order (may not be consecutive), // and the hash is on the same line as the real attr name, // we wrongly interpret the hash as the start of a comment. // e.g `foo["...barrier#"].bar` will highlight part of the string. node->endLine = node->startLine; node->endCol = node->startCol + name.length() - 1; node->attribute->copyRange(node); AstDefaultVisitor::visitAttribute(node); }; // alias for imports (import foo as bar, baz as bang) // no strings, brackets, or whatever are allowed here, so the "parser" // can be very straightforward. void visitImport(ImportAst* node) override { AstDefaultVisitor::visitImport(node); int aliasIndex = 0; foreach ( AliasAst* alias, node->names ) { fixAlias(alias->name, alias->asName, node->startLine, aliasIndex); aliasIndex += 1; } }; // alias for exceptions (except FooBarException as somethingterriblehappened: ...) void visitExceptionHandler(ExceptionHandlerAst* node) override { AstDefaultVisitor::visitExceptionHandler(node); if ( ! node->name ) { return; } const QString& line = lines.at(node->startLine); const int end = line.count() - 1; int back = backtrackDottedName(line, end); node->name->startCol = end - back; node->name->endCol = end; } void visitString(Python::StringAst* node) override { AstDefaultVisitor::visitString(node); auto match = findString.match(lines.at(node->startLine), node->startCol); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength() - 1; // Ranges are inclusive. } } void visitBytes(Python::BytesAst* node) override { AstDefaultVisitor::visitBytes(node); auto match = findString.match(lines.at(node->startLine), node->startCol + 1); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength(); // -1 then +1, because of the 'b'. } } void visitFormattedValue(Python::FormattedValueAst * node) override { AstDefaultVisitor::visitFormattedValue(node); auto match = findString.match(lines.at(node->startLine), node->startCol + 1); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength(); } } void visitNumber(Python::NumberAst* node) override { AstDefaultVisitor::visitNumber(node); auto match = findNumber.match(lines.at(node->startLine), node->startCol); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength() - 1; // Ranges are inclusive. } } // Add one column after the last child to cover the closing bracket: `[1,2,3]` // TODO This is still wrong if the last child is followed by parens or whitespace. // endCol matters most in single-line expressions, so this isn't a huge problem. void visitSubscript(Python::SubscriptAst* node) override { AstDefaultVisitor::visitSubscript(node); node->endCol++; } void visitComprehension(Python::ComprehensionAst* node) override { AstDefaultVisitor::visitComprehension(node); node->endCol++; } void visitList(Python::ListAst* node) override { AstDefaultVisitor::visitList(node); node->endCol++; } void visitTuple(Python::TupleAst* node) override { AstDefaultVisitor::visitTuple(node); node->endCol++; } private: const QStringList lines; QVector dots; KTextEditor::Cursor attributeStart; static const QRegularExpression findString; static const QRegularExpression findNumber; // skip the decorators and the "def" at the beginning // of a class or function declaration and modify @arg node // example: // @decorate(foo) // @decorate(bar) // class myclass(parent): pass // before: start of class->name is [0, 0] // after: start of class->name is [2, 5] // line continuation characters are not supported, // because code needing those in this case is not worth being supported. void cutDefinitionPreamble(Ast* fixNode, const QString& defKeyword) { if ( ! fixNode ) { return; } int currentLine = fixNode->startLine; // cut away decorators while ( currentLine < lines.size() ) { if ( lines.at(currentLine).trimmed().remove(' ').remove('\t').startsWith(defKeyword) ) { // it's not a decorator, so stop skipping lines. break; } currentLine += 1; } // qDebug() << "FIX:" << fixNode->range(); fixNode->startLine = currentLine; fixNode->endLine = currentLine; // qDebug() << "FIXED:" << fixNode->range() << fixNode->astType; // cut away the "def" / "class" int currentColumn = -1; if ( currentLine > lines.size() ) { // whops? return; } const QString& lineData = lines.at(currentLine); bool keywordFound = false; while ( currentColumn < lineData.size() - 1 ) { currentColumn += 1; if ( lineData.at(currentColumn).isSpace() ) { // skip space at the beginning of the line continue; } else if ( keywordFound ) { // if the "def" / "class" was already found, and the current char is // non space, then this is indeed the start of the identifier we're looking for. break; } else { keywordFound = true; currentColumn += defKeyword.size(); } } const int previousLength = fixNode->endCol - fixNode->startCol; fixNode->startCol = currentColumn; fixNode->endCol = currentColumn + previousLength; }; int backtrackDottedName(const QString& data, const int start) { bool haveDot = true; bool previousWasSpace = true; for ( int i = start - 1; i >= 0; i-- ) { if ( data.at(i).isSpace() ) { previousWasSpace = true; continue; } if ( data.at(i) == ':' ) { // excepthandler continue; } if ( data.at(i) == '.' ) { haveDot = true; } else if ( haveDot ) { haveDot = false; previousWasSpace = false; continue; } if ( previousWasSpace && ! haveDot ) { return start-i-2; } previousWasSpace = false; } return 0; } void fixAlias(Ast* dotted, Ast* asname, const int startLine, int aliasIndex) { if ( ! asname && ! dotted ) { return; } QString line = lines.at(startLine); int lineno = startLine; for ( int i = 0; i < line.size(); i++ ) { const QChar& current = line.at(i); if ( current == '\\' ) { // line continuation character // splitting like "import foo as \ \n bar" is not supported. lineno += 1; line = lines.at(lineno); i = 0; continue; } if ( current == ',' ) { if ( aliasIndex == 0 ) { // nothing found, continue below line = line.left(i); break; } // next alias expression aliasIndex -= 1; } if ( i > line.length() - 3 ) { continue; } if ( current.isSpace() && line.mid(i+1).startsWith("as") && ( line.at(i+3).isSpace() || line.at(i+3) == '\\' ) ) { // there's an "as" if ( aliasIndex == 0 ) { // it's the one we're looking for // find the expression if ( dotted ) { int dottedNameLength = backtrackDottedName(line, i); dotted->startLine = lineno; dotted->endLine = lineno; dotted->startCol = i-dottedNameLength; dotted->endCol = i; } // find the asname if ( asname ) { bool atStart = true; int textStart = i+3; for ( int j = i+3; j < line.size(); j++ ) { if ( atStart && ! line.at(j).isSpace() ) { atStart = false; textStart = j; } if ( ! atStart && ( line.at(j).isSpace() || j == line.size() - 1 ) ) { // found it asname->startLine = lineno; asname->endLine = lineno; asname->startCol = textStart - 1; asname->endCol = j; } } } return; } } } // no "as" found, use last dotted name in line const int end = line.count() - whitespaceAtEnd(line); int back = backtrackDottedName(line, end); dotted->startLine = lineno; dotted->endLine = lineno; dotted->startCol = end - back; dotted->endCol = end; }; int whitespaceAtEnd(const QString& line) { for ( int i = 0; i <= line.size(); i++ ) { if ( ! line.at(line.size() - i - 1).isSpace() ) { return i; } } return 0; }; }; // FIXME This doesn't work for triple-quoted strings // (it gives length 2, which is no worse than before). const QRegularExpression RangeFixVisitor::findString = QRegularExpression("\\G(['\"]).*?(?(PyObject_Str(obj), pyObjectCleanup); const auto str = strOwner.get(); if (PyUnicode_READY(str) < 0) { qWarning("PyUnicode_READY(%p) returned false!", (void*)str); return QString(); } const auto length = PyUnicode_GET_LENGTH(str); switch(PyUnicode_KIND(str)) { case PyUnicode_1BYTE_KIND: return QString::fromLatin1((const char*)PyUnicode_1BYTE_DATA(str), length); case PyUnicode_2BYTE_KIND: return QString::fromUtf16(PyUnicode_2BYTE_DATA(str), length); case PyUnicode_4BYTE_KIND: return QString::fromUcs4(PyUnicode_4BYTE_DATA(str), length); case PyUnicode_WCHAR_KIND: qWarning("PyUnicode_KIND(%p) returned PyUnicode_WCHAR_KIND, this should not happen!", (void*)str); return QString::fromWCharArray(PyUnicode_AS_UNICODE(str), length); } Q_UNREACHABLE(); } QPair fileHeaderHack(QString& contents, const QUrl& filename) { IProject* proj = ICore::self()->projectController()->findProjectForUrl(filename); // the file is not in a project, don't apply hack if ( ! proj ) { return QPair(contents, 0); } const QUrl headerFileUrl = QUrl::fromLocalFile(proj->path().path() + "/.kdev_python_header"); QFile headerFile(headerFileUrl.path()); QString headerFileContents; if ( headerFile.exists() ) { headerFile.open(QIODevice::ReadOnly); headerFileContents = headerFile.readAll(); headerFile.close(); qCDebug(KDEV_PYTHON_PARSER) << "Found header file, applying hack"; int insertAt = 0; bool endOfCommentsReached = false; bool commentSignEncountered = false; // bool atLineBeginning = true; int lastLineBeginning = 0; int newlineCount = 0; int l = contents.length(); do { if ( insertAt >= l ) { qCDebug(KDEV_PYTHON_PARSER) << "File consist only of comments, not applying hack"; return QPair(contents, 0); } if ( contents.at(insertAt) == '#' ) { commentSignEncountered = true; } if ( !contents.at(insertAt).isSpace() ) { // atLineBeginning = false; if ( !commentSignEncountered ) { endOfCommentsReached = true; } } if ( contents.at(insertAt) == '\n' ) { // atLineBeginning = true; commentSignEncountered = false; lastLineBeginning = insertAt; newlineCount += 1; } if ( newlineCount == 2 ) { endOfCommentsReached = true; } insertAt += 1; } while ( !endOfCommentsReached ); qCDebug(KDEV_PYTHON_PARSER) << "Inserting contents at char" << lastLineBeginning << "of file"; contents = contents.left(lastLineBeginning) + "\n" + headerFileContents + "\n#\n" + contents.right(contents.length() - lastLineBeginning); qCDebug(KDEV_PYTHON_PARSER) << contents; return QPair(contents, - ( headerFileContents.count('\n') + 3 )); } else { return QPair(contents, 0); } } namespace { struct PythonInitializer : private QMutexLocker { PythonInitializer(QMutex& pyInitLock): QMutexLocker(&pyInitLock), arena(0) { Py_InitializeEx(0); Q_ASSERT(Py_IsInitialized()); arena = PyArena_New(); Q_ASSERT(arena); // out of memory } ~PythonInitializer() { if (arena) PyArena_Free(arena); if (Py_IsInitialized()) Py_Finalize(); } PyArena* arena; }; } CodeAst::Ptr AstBuilder::parse(const QUrl& filename, QString &contents) { - qDebug() << " ====> AST ====> building abstract syntax tree for " << filename.path(); + qCDebug(KDEV_PYTHON_PARSER) << " ====> AST ====> building abstract syntax tree for " << filename.path(); Py_NoSiteFlag = 1; contents.append('\n'); QPair hacked = fileHeaderHack(contents, filename); contents = hacked.first; int lineOffset = hacked.second; PythonInitializer pyIniter(pyInitLock); PyArena* arena = pyIniter.arena; PyCompilerFlags flags = {PyCF_SOURCE_IS_UTF8 | PyCF_IGNORE_COOKIE}; PyObject *exception, *value, *backtrace; PyErr_Fetch(&exception, &value, &backtrace); CythonSyntaxRemover cythonSyntaxRemover; if (filename.fileName().endsWith(".pyx", Qt::CaseInsensitive)) { qCDebug(KDEV_PYTHON_PARSER) << filename.fileName() << "is probably Cython file."; contents = cythonSyntaxRemover.stripCythonSyntax(contents); } mod_ty syntaxtree = PyParser_ASTFromString(contents.toUtf8().data(), "", file_input, &flags, arena); if ( ! syntaxtree ) { - qDebug() << " ====< parse error, trying to fix"; + qCDebug(KDEV_PYTHON_PARSER) << " ====< parse error, trying to fix"; PyErr_Fetch(&exception, &value, &backtrace); qCDebug(KDEV_PYTHON_PARSER) << "Error objects: " << exception << value << backtrace; if ( ! value ) { qCWarning(KDEV_PYTHON_PARSER) << "Internal parser error: exception value is null, aborting"; return CodeAst::Ptr(); } PyObject_Print(value, stderr, Py_PRINT_RAW); PyObject* errorMessage_str = PyTuple_GetItem(value, 0); PyObject* errorDetails_tuple = PyTuple_GetItem(value, 1); if ( ! errorDetails_tuple ) { qCWarning(KDEV_PYTHON_PARSER) << "Error retrieving error message, not displaying, and not doing anything"; return CodeAst::Ptr(); } PyObject* linenoobj = PyTuple_GetItem(errorDetails_tuple, 1); errorMessage_str = PyTuple_GetItem(value, 0); errorDetails_tuple = PyTuple_GetItem(value, 1); PyObject_Print(errorMessage_str, stderr, Py_PRINT_RAW); PyObject* colnoobj = PyTuple_GetItem(errorDetails_tuple, 2); int lineno = PyLong_AsLong(linenoobj) - 1; int colno = PyLong_AsLong(colnoobj); ProblemPointer p(new Problem()); KTextEditor::Cursor start(lineno + lineOffset, (colno-4 > 0 ? colno-4 : 0)); KTextEditor::Cursor end(lineno + lineOffset, (colno+4 > 4 ? colno+4 : 4)); KTextEditor::Range range(start, end); qCDebug(KDEV_PYTHON_PARSER) << "Problem range: " << range; DocumentRange location(IndexedString(filename.path()), range); p->setFinalLocation(location); p->setDescription(PyUnicodeObjectToQString(errorMessage_str)); p->setSource(IProblem::Parser); m_problems.append(p); // try to recover. // Currently the following is tired: // * If the last non-space char before the error reported was ":", it's most likely an indent error. // The common easy-to-fix and annoying indent error is "for item in foo: ". In that case, just add "pass" after the ":" token. // * If it's not, we will just comment the line with the error, fixing problems like "foo = ". // * If both fails, everything including the first non-empty line before the one with the error will be deleted. int len = contents.length(); int currentLine = 0; QString currentLineContents; QChar c; QChar newline('\n'); int emptySince = 0; int emptySinceLine = 0; int emptyLinesSince = 0; int emptyLinesSinceLine = 0; unsigned short currentLineIndent = 0; bool atLineBeginning = true; QList indents; int errline = qMax(0, lineno); int currentLineBeginning = 0; for ( int i = 0; i < len; i++ ) { c = contents.at(i); if ( ! c.isSpace() ) { emptySince = i; emptySinceLine = currentLine; atLineBeginning = false; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); } else if ( c == newline ) { if ( currentLine == errline ) { atLineBeginning = false; } else { currentLine += 1; currentLineBeginning = i+1; // this line has had content, so reset the "empty lines since" counter if ( ! atLineBeginning ) { // lastNonemptyLineBeginning = emptyLinesSince; emptyLinesSince = i; emptyLinesSinceLine = currentLine; } atLineBeginning = true; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); currentLineIndent = 0; } } else if ( atLineBeginning ) { currentLineIndent += 1; } if ( currentLine == errline && ! atLineBeginning ) { // if the last non-empty char before the error opens a new block, it's likely an "empty block" problem // we can easily fix that by adding in a "pass" statement. However, we want to add that in the next line, if possible // so context ranges for autocompletion stay intact. if ( contents[emptySince] == QChar(':') ) { qCDebug(KDEV_PYTHON_PARSER) << indents.length() << emptySinceLine + 1 << indents; if ( indents.length() > emptySinceLine + 1 && indents.at(emptySinceLine) < indents.at(emptySinceLine + 1) ) { qCDebug(KDEV_PYTHON_PARSER) << indents.at(emptySinceLine) << indents.at(emptySinceLine + 1); contents.insert(emptyLinesSince + 1 + indents.at(emptyLinesSinceLine), "\tpass#"); } else { contents.insert(emptySince + 1, "\tpass#"); } } else if ( indents.length() >= currentLine && currentLine > 0 ) { qCDebug(KDEV_PYTHON_PARSER) << indents << currentLine; contents[i+1+indents.at(currentLine - 1)] = QChar('#'); contents.insert(i+1+indents.at(currentLine - 1), "pass"); } break; } } syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); // 3rd try: discard everything after the last non-empty line, but only until the next block start currentLineBeginning = qMin(contents.length() - 1, currentLineBeginning); errline = qMax(0, qMin(indents.length()-1, errline)); if ( ! syntaxtree ) { qCWarning(KDEV_PYTHON_PARSER) << "Discarding parts of the code to be parsed because of previous errors"; qCDebug(KDEV_PYTHON_PARSER) << indents; int indentAtError = indents.at(errline); QChar c; bool atLineBeginning = true; int currentIndent = -1; int currentLineBeginning_end = currentLineBeginning; int currentLineContentBeginning = currentLineBeginning; for ( int i = currentLineBeginning; i < len; i++ ) { c = contents.at(i); qCDebug(KDEV_PYTHON_PARSER) << c; if ( c == '\n' ) { if ( currentIndent <= indentAtError && currentIndent != -1 ) { qCDebug(KDEV_PYTHON_PARSER) << "Start of error code: " << currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "End of error block (current position): " << currentLineBeginning_end; qCDebug(KDEV_PYTHON_PARSER) << "Length: " << currentLineBeginning_end - currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "indent at error <> current indent:" << indentAtError << "<>" << currentIndent; // contents.remove(currentLineBeginning, currentLineBeginning_end-currentLineBeginning); break; } contents.insert(currentLineContentBeginning - 1, "pass#"); i += 5; i = qMin(i, contents.length()); len = contents.length(); atLineBeginning = true; currentIndent = 0; currentLineBeginning_end = i + 1; currentLineContentBeginning = i + 1; continue; } if ( ! c.isSpace() && atLineBeginning ) { currentLineContentBeginning = i; atLineBeginning = false; } if ( c.isSpace() && atLineBeginning ) currentIndent += 1; } qCDebug(KDEV_PYTHON_PARSER) << "This is what is left: " << contents; syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); } if ( ! syntaxtree ) { return CodeAst::Ptr(); // everything fails, so we abort. } } qCDebug(KDEV_PYTHON_PARSER) << "Got syntax tree from python parser:" << syntaxtree->kind << Module_kind; PythonAstTransformer t(lineOffset); t.run(syntaxtree, filename.fileName().replace(".py", "")); RangeFixVisitor fixVisitor(contents); fixVisitor.visitNode(t.ast); cythonSyntaxRemover.fixAstRanges(t.ast); return CodeAst::Ptr(t.ast); } } diff --git a/pythondebug.cpp b/pythondebug.cpp index 4c845f30..5e3d8f3e 100644 --- a/pythondebug.cpp +++ b/pythondebug.cpp @@ -1,23 +1,23 @@ /* This file is part of the KDE project Copyright (C) 2014 Laurent Navet This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "pythondebug.h" -Q_LOGGING_CATEGORY(KDEV_PYTHON, "kdev.python") +Q_LOGGING_CATEGORY(KDEV_PYTHON, "kdevelop.languages.python") diff --git a/pythonlanguagesupport.cpp b/pythonlanguagesupport.cpp index e59136c2..3b0f5ce7 100644 --- a/pythonlanguagesupport.cpp +++ b/pythonlanguagesupport.cpp @@ -1,230 +1,230 @@ /***************************************************************************** * Copyright (c) 2007 Andreas Pakulat * * Copyright (c) 2007 Piyush verma * * Copyright (c) 2012-2016 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "pythonlanguagesupport.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "pythonparsejob.h" #include "pythonhighlighting.h" #include "duchain/pythoneditorintegrator.h" #include "codecompletion/model.h" #include "codegen/refactoring.h" #include "codegen/correctionfilegenerator.h" #include "kdevpythonversion.h" #include "pep8kcm/kcm_pep8.h" #include "projectconfig/projectconfigpage.h" #include "docfilekcm/kcm_docfiles.h" #include "pythonstylechecking.h" #include "helpers.h" #include #include #include "pythondebug.h" using namespace KDevelop; K_PLUGIN_FACTORY_WITH_JSON( KDevPythonSupportFactory, "kdevpythonsupport.json", registerPlugin(); ) namespace Python { LanguageSupport* LanguageSupport::m_self = 0; KDevelop::ContextMenuExtension LanguageSupport::contextMenuExtension(KDevelop::Context* context) { ContextMenuExtension cm; EditorContext *ec = dynamic_cast(context); if (ec && ICore::self()->languageController()->languagesForUrl(ec->url()).contains(this)) { // It's a Python file, let's add our context menu. m_refactoring->fillContextMenu(cm, context); TypeCorrection::self().doContextMenu(cm, context); } return cm; } LanguageSupport::LanguageSupport( QObject* parent, const QVariantList& /*args*/ ) : KDevelop::IPlugin("pythonlanguagesupport", parent ) , KDevelop::ILanguageSupport() , m_highlighting( new Highlighting( this ) ) , m_refactoring( new Refactoring( this ) ) , m_styleChecking( new StyleChecking( this ) ) { m_self = this; PythonCodeCompletionModel* codeCompletion = new PythonCodeCompletionModel(this); new KDevelop::CodeCompletion(this, codeCompletion, "Python"); auto assistantsManager = core()->languageController()->staticAssistantsManager(); assistantsManager->registerAssistant(StaticAssistant::Ptr(new RenameAssistant(this))); QObject::connect(ICore::self()->documentController(), &IDocumentController::documentOpened, this, &LanguageSupport::documentOpened); } void LanguageSupport::documentOpened(IDocument* doc) { if ( ! ICore::self()->languageController()->languagesForUrl(doc->url()).contains(this) ) { // not a python file return; } DUChainReadLocker lock; ReferencedTopDUContext top = DUChain::self()->chainForDocument(doc->url()); lock.unlock(); updateStyleChecking(top); } void LanguageSupport::updateStyleChecking(KDevelop::ReferencedTopDUContext top) { m_styleChecking->updateStyleChecking(top); } LanguageSupport::~LanguageSupport() { parseLock()->lockForWrite(); // By locking the parse-mutexes, we make sure that parse jobs get a chance to finish in a good state parseLock()->unlock(); delete m_highlighting; m_highlighting = 0; } KDevelop::ParseJob *LanguageSupport::createParseJob( const IndexedString& url ) { return new ParseJob(url, this); } QString LanguageSupport::name() const { return "Python"; } LanguageSupport* LanguageSupport::self() { return m_self; } SourceFormatterItemList LanguageSupport::sourceFormatterItems() const { SourceFormatterStyle autopep8("autopep8"); autopep8.setCaption("autopep8"); autopep8.setDescription(i18n("Format source with the autopep8 formatter.")); autopep8.setOverrideSample("class klass:\n def method(arg1,arg2):\n a=3+5\n" "def function(arg,*vararg,**kwargs): return arg+kwarg[0]\nfunction(3, 5, 7)"); using P = SourceFormatterStyle::MimeHighlightPair; autopep8.setMimeTypes(SourceFormatterStyle::MimeList{ P{"text/x-python", "Python"} }); QString autopep8path = QStandardPaths::findExecutable("autopep8"); if (autopep8path.isEmpty()) { // TODO: proper error handling/user notification - qDebug() << "Could not find the autopep8 executable"; + qCDebug(KDEV_PYTHON) << "Could not find the autopep8 executable"; autopep8path = "/usr/bin/autopep8"; } autopep8.setContent(autopep8path + " -i $TMPFILE"); return SourceFormatterItemList{SourceFormatterStyleItem{"customscript", autopep8}}; } KDevelop::ICodeHighlighting* LanguageSupport::codeHighlighting() const { return m_highlighting; } BasicRefactoring* LanguageSupport::refactoring() const { return m_refactoring; } int LanguageSupport::suggestedReparseDelayForChange(KTextEditor::Document* doc, const KTextEditor::Range& changedRange, const QString& changedText, bool /*removal*/) const { if ( changedRange.start().line() != changedRange.end().line() ) { // instant update return 0; } if ( std::all_of(changedText.begin(), changedText.end(), [](const QChar& c) { return c.isSpace(); }) ) { - qDebug() << changedText << changedRange.end().column() << doc->lineLength(changedRange.end().line()); + qCDebug(KDEV_PYTHON) << changedText << changedRange.end().column() << doc->lineLength(changedRange.end().line()); if ( changedRange.end().column()-1 == doc->lineLength(changedRange.end().line()) ) { return ILanguageSupport::NoUpdateRequired; } } return ILanguageSupport::DefaultDelay; } QList LanguageSupport::providedChecks() { return {}; } int LanguageSupport::configPages() const { return 2; } KDevelop::ConfigPage* LanguageSupport::configPage(int number, QWidget* parent) { if (number == 0) { return new PEP8KCModule(this, parent); } else if (number == 1) { return new DocfilesKCModule(this, parent); } return nullptr; } int LanguageSupport::perProjectConfigPages() const { return 1; } KDevelop::ConfigPage* LanguageSupport::perProjectConfigPage(int number, const KDevelop::ProjectConfigOptions& options, QWidget* parent) { if ( number == 0 ) { return new Python::ProjectConfigPage(this, options, parent); } return nullptr; } } #include "pythonlanguagesupport.moc" diff --git a/pythonparsejob.cpp b/pythonparsejob.cpp index 0dd3126e..66229bab 100644 --- a/pythonparsejob.cpp +++ b/pythonparsejob.cpp @@ -1,297 +1,298 @@ /***************************************************************************** * Copyright (c) 2007 Andreas Pakulat * * Copyright (c) 2007 Piyush verma * * Copyright (c) 2010-2013 Sven Brauch * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License as * * published by the Free Software Foundation; either version 2 of * * the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see . * ***************************************************************************** */ #include "pythonparsejob.h" +#include "pythondebug.h" #include "pythonhighlighting.h" #include "pythoneditorintegrator.h" #include "dumpchain.h" #include "parsesession.h" #include "pythonlanguagesupport.h" #include "declarationbuilder.h" #include "usebuilder.h" #include "kshell.h" #include "duchain/helpers.h" #include "pep8kcm/kcm_pep8.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace KDevelop; namespace Python { ParseJob::ParseJob(const IndexedString &url, ILanguageSupport* languageSupport) : KDevelop::ParseJob(url, languageSupport) , m_ast(0) , m_duContext(0) { IDefinesAndIncludesManager* iface = IDefinesAndIncludesManager::manager(); auto project = ICore::self()->projectController()->findProjectForUrl(url.toUrl()); if ( project ) { foreach (Path path, iface->includes(project->projectItem(), IDefinesAndIncludesManager::UserDefined)) { m_cachedCustomIncludes.append(path.toUrl()); } QMutexLocker lock(&Helper::cacheMutex); Helper::cachedCustomIncludes[project] = m_cachedCustomIncludes; } } ParseJob::~ParseJob() { } CodeAst *ParseJob::ast() const { Q_ASSERT( isFinished() && m_ast ); return m_ast.data(); } void ParseJob::run(ThreadWeaver::JobPointer /*self*/, ThreadWeaver::Thread* /*thread*/) { if ( abortRequested() || ICore::self()->shuttingDown() ) { return abortJob(); } - qDebug() << " ====> PARSING ====> parsing file " << document().toUrl() << "; has priority" << parsePriority(); + qCDebug(KDEV_PYTHON) << " ====> PARSING ====> parsing file " << document().toUrl() << "; has priority" << parsePriority(); { QMutexLocker l(&Helper::projectPathLock); Helper::projectSearchPaths.clear(); foreach (IProject* project, ICore::self()->projectController()->projects() ) { Helper::projectSearchPaths.append(QUrl::fromLocalFile(project->path().path())); } } // lock the URL so no other parse job can run on this document QReadLocker parselock(languageSupport()->parseLock()); UrlParseLock urlLock(document()); readContents(); if ( !(minimumFeatures() & TopDUContext::ForceUpdate || minimumFeatures() & Rescheduled) ) { DUChainReadLocker lock(DUChain::lock()); static const IndexedString langString("python"); foreach(const ParsingEnvironmentFilePointer &file, DUChain::self()->allEnvironmentFiles(document())) { if ( file->language() != langString ) { continue; } if ( ! file->needsUpdate() && file->featuresSatisfied(minimumFeatures()) && file->topContext() ) { - qDebug() << " ====> NOOP ====> Already up to date:" << document().str(); + qCDebug(KDEV_PYTHON) << " ====> NOOP ====> Already up to date:" << document().str(); setDuChain(file->topContext()); if ( ICore::self()->languageController()->backgroundParser()->trackerForUrl(document()) ) { lock.unlock(); highlightDUChain(); } return; } break; } } ReferencedTopDUContext toUpdate = 0; { DUChainReadLocker lock; toUpdate = DUChainUtils::standardContextForUrl(document().toUrl()); } if ( toUpdate ) { translateDUChainToRevision(toUpdate); toUpdate->setRange(RangeInRevision(0, 0, INT_MAX, INT_MAX)); } m_currentSession = new ParseSession(); m_currentSession->setContents(QString::fromUtf8(contents().contents)); m_currentSession->setCurrentDocument(document()); // call the python API and the AST transformer to populate the syntax tree QPair parserResults = m_currentSession->parse(); m_ast = parserResults.first; auto editor = QSharedPointer(new PythonEditorIntegrator(m_currentSession.data())); // if parsing succeeded, continue and do semantic analysis if ( parserResults.second ) { // set up the declaration builder, it gets the parsePriority so it can re-schedule imported files with a better priority DeclarationBuilder builder(editor.data(), parsePriority()); builder.setCurrentlyParsedDocument(document()); builder.setFutureModificationRevision(contents().modification); // Run the declaration builder. If necessary, it will run itself again. m_duContext = builder.build(document(), m_ast.data(), toUpdate.data()); if ( abortRequested() ) { return abortJob(); } setDuChain(m_duContext); // gather uses of variables and functions on the document UseBuilder usebuilder(editor.data(), builder.missingModules()); usebuilder.setCurrentlyParsedDocument(document()); usebuilder.buildUses(m_ast.data()); // check whether any unresolved imports were encountered bool needsReparse = ! builder.unresolvedImports().isEmpty(); - qDebug() << "Document needs update because of unresolved identifiers: " << needsReparse; + qCDebug(KDEV_PYTHON) << "Document needs update because of unresolved identifiers: " << needsReparse; if ( needsReparse ) { // check whether one of the imports is queued for parsing, this is to avoid deadlocks // it's also ok if the duchain is now available (and thus has been parsed before already) bool dependencyInQueue = false; DUChainWriteLocker lock; foreach ( const IndexedString& url, builder.unresolvedImports() ) { dependencyInQueue = KDevelop::ICore::self()->languageController()->backgroundParser()->isQueued(url); dependencyInQueue = dependencyInQueue || DUChain::self()->chainForDocument(url); if ( dependencyInQueue ) { break; } } // we check whether this document already has been re-scheduled once and abort if that is the case // this prevents infinite loops in case something goes wrong (optimally, shouldn't reach here if // the document was already rescheduled, but there's many cases where this might still happen) if ( ! ( minimumFeatures() & Rescheduled ) && dependencyInQueue ) { KDevelop::ICore::self()->languageController()->backgroundParser()->addDocument(document(), static_cast(TopDUContext::ForceUpdate | Rescheduled), parsePriority(), nullptr, ParseJob::FullSequentialProcessing); } } // some internal housekeeping work { DUChainWriteLocker lock(DUChain::lock()); m_duContext->setFeatures(minimumFeatures()); ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile(); parsingEnvironmentFile->setModificationRevision(contents().modification); DUChain::self()->updateContextEnvironment(m_duContext, parsingEnvironmentFile.data()); } - qDebug() << "---- Parsing Succeeded ----"; + qCDebug(KDEV_PYTHON) << "---- Parsing Succeeded ----"; if ( abortRequested() ) { return abortJob(); } // start the code highlighter if parsing was successful. highlightDUChain(); } else { // No syntax tree was received from the parser, the expected reason for this is a syntax error in the document. qWarning() << "---- Parsing FAILED ----"; DUChainWriteLocker lock; m_duContext = toUpdate.data(); // if there's already a chain for the document, do some cleanup. if ( m_duContext ) { // m_duContext->parsingEnvironmentFile()->clearModificationRevisions(); // TODO why? ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile(); parsingEnvironmentFile->setModificationRevision(contents().modification); m_duContext->clearProblems(); } // otherwise, create a new, empty top context for the file. This serves as a placeholder until // the syntax is fixed; for example, it prevents the document from being reparsed again until it is modified. else { ParsingEnvironmentFile* file = new ParsingEnvironmentFile(document()); static const IndexedString langString("python"); file->setLanguage(langString); m_duContext = new TopDUContext(document(), RangeInRevision(0, 0, INT_MAX, INT_MAX), file); m_duContext->setType(DUContext::Global); DUChain::self()->addDocumentChain(m_duContext); Q_ASSERT(m_duContext->type() == DUContext::Global); } setDuChain(m_duContext); } if ( abortRequested() ) { return abortJob(); } // The parser might have given us some syntax errors, which are now added to the document. DUChainWriteLocker lock; foreach ( const ProblemPointer& p, m_currentSession->m_problems ) { m_duContext->addProblem(p); } // If enabled, and if the document is open, do PEP8 checking. eventuallyDoPEP8Checking(m_duContext); if ( minimumFeatures() & TopDUContext::AST ) { DUChainWriteLocker lock; m_currentSession->ast = m_ast; m_duContext->setAst(QExplicitlySharedDataPointer(m_currentSession.data())); } setDuChain(m_duContext); DUChain::self()->emitUpdateReady(document(), duChain()); } ControlFlowGraph* ParseJob::controlFlowGraph() { return nullptr; } DataAccessRepository* ParseJob::dataAccessInformation() { return nullptr; } void ParseJob::eventuallyDoPEP8Checking(TopDUContext* topContext) { KConfig config("kdevpythonsupportrc"); KConfigGroup configGroup = config.group("pep8"); if ( !PEP8KCModule::isPep8Enabled(configGroup) ) { return; } auto ls = static_cast(languageSupport()); QMetaObject::invokeMethod(ls, "updateStyleChecking", Q_ARG(KDevelop::ReferencedTopDUContext, topContext)); } } // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on diff --git a/pythonstylechecking.cpp b/pythonstylechecking.cpp index 6e65b778..d3a569ca 100644 --- a/pythonstylechecking.cpp +++ b/pythonstylechecking.cpp @@ -1,203 +1,204 @@ /* * Copyright 2016 Sven Brauch * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ #include "pythonstylechecking.h" #include #include #include #include #include #include +#include "pythondebug.h" #include "pythonparsejob.h" #include "helpers.h" namespace Python { StyleChecking::StyleChecking(QObject* parent) : QObject(parent) { qRegisterMetaType("KDevelop::ReferencedTopDUContext"); connect(&m_checkerProcess, &QProcess::readyReadStandardOutput, this, &StyleChecking::processOutputStarted); connect(&m_checkerProcess, &QProcess::readyReadStandardError, [this]() { qWarning() << "python code checker error:" << m_checkerProcess.readAllStandardError(); }); auto config = KSharedConfig::openConfig("kdevpythonsupportrc"); m_pep8Group = config->group("pep8"); } StyleChecking::~StyleChecking() { if ( m_checkerProcess.state() == QProcess::Running ) { m_checkerProcess.terminate(); m_checkerProcess.waitForFinished(100); } } void StyleChecking::startChecker(const QString& text, const QString& select, const QString& ignore, const int maxLineLength) { // start up the server if ( m_checkerProcess.state() == QProcess::NotRunning ) { auto python = Helper::getPythonExecutablePath(nullptr); auto serverPath = QStandardPaths::locate(QStandardPaths::GenericDataLocation, "kdevpythonsupport/codestyle.py"); if ( serverPath.isEmpty() ) { qWarning() << "setup problem: codestyle.py not found"; return; } m_checkerProcess.start(python, {serverPath}); m_checkerProcess.waitForStarted(30); if ( m_checkerProcess.state() != QProcess::Running ) { qWarning() << "failed to start code checker process"; return; } } // send input QByteArray data = text.toUtf8(); QByteArray header; header.append(select.toUtf8()); header.append("\n"); header.append(ignore.toUtf8()); header.append("\n"); header.append(QByteArray::number(maxLineLength)); header.append("\n"); // size, always 10 bytes header.insert(0, QString::number(header.size() + data.size()).leftJustified(10)); m_checkerProcess.write(header); m_checkerProcess.write(data); } void StyleChecking::addErrorsToContext(const QVector& errors) { static QRegularExpression errorFormat("(.*):(\\d*):(\\d*): (.*)", QRegularExpression::CaseInsensitiveOption); DUChainWriteLocker lock; auto document = m_currentlyChecking->url(); for ( const auto& error : errors ) { QRegularExpressionMatch match; if ( (match = errorFormat.match(error)).hasMatch() ) { bool lineno_ok = false; bool colno_ok = false; int lineno = match.captured(2).toInt(&lineno_ok); int colno = match.captured(3).toInt(&colno_ok); if ( ! lineno_ok || ! colno_ok ) { - qDebug() << "invalid line / col number"; + qCDebug(KDEV_PYTHON) << "invalid line / col number"; continue; } QString error = match.captured(4); KDevelop::Problem* p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(document, KTextEditor::Range(lineno - 1, qMax(colno - 4, 0), lineno - 1, colno + 4))); p->setSource(KDevelop::IProblem::Preprocessor); p->setSeverity(error.startsWith('W') ? KDevelop::IProblem::Hint : KDevelop::IProblem::Warning); p->setDescription(i18n("PEP8 checker error: %1", error)); ProblemPointer ptr(p); m_currentlyChecking->addProblem(ptr); } else { - qDebug() << "invalid pep8 error line:" << error; + qCDebug(KDEV_PYTHON) << "invalid pep8 error line:" << error; } } m_currentlyChecking->setFeatures((TopDUContext::Features) ( m_currentlyChecking->features() | ParseJob::PEP8Checking )); } void StyleChecking::processOutputStarted() { // read output size QByteArray size_d; size_d = m_checkerProcess.read(10); bool ok; auto size = size_d.toInt(&ok); if ( !ok || size < 0 ) { addSetupErrorToContext("Got invalid size: " + size_d); m_mutex.unlock(); return; } // read and process actual output QByteArray buf; QVector errors; QTimer t; t.start(100); while ( size > 0 && t.isActive() ) { auto d = m_checkerProcess.read(size); buf.append(d); size -= d.size(); auto ofs = -1; auto prev = ofs; while ( prev = ofs, (ofs = buf.indexOf('\n', ofs+1)) != -1 ) { errors.append(buf.mid(prev+1, ofs-prev)); } } if ( !t.isActive() ) { addSetupErrorToContext("Output took longer than 100 ms."); } addErrorsToContext(errors); // done, unlock mutex m_currentlyChecking = nullptr; m_mutex.unlock(); } void StyleChecking::updateStyleChecking(const KDevelop::ReferencedTopDUContext& top) { if ( !top ) { return; } auto url = top->url(); IDocument* idoc = ICore::self()->documentController()->documentForUrl(url.toUrl()); if ( !idoc || !idoc->textDocument() || top->features() & ParseJob::PEP8Checking ) { return; } auto text = idoc->textDocument()->text(); if ( !m_mutex.tryLock(1000) ) { return; } m_currentlyChecking = top; // default empty is ok, it will never be used, because the config has to be written at least once // to even enable this feature. auto select = m_pep8Group.readEntry("enableErrors", ""); auto ignore = m_pep8Group.readEntry("disableErrors", ""); auto maxLineLength = m_pep8Group.readEntry("maxLineLength", 80); startChecker(text, select, ignore, maxLineLength); } void StyleChecking::addSetupErrorToContext(const QString& error) { DUChainWriteLocker lock; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(m_currentlyChecking->url(), KTextEditor::Range(0, 0, 0, 0))); p->setSource(KDevelop::IProblem::Preprocessor); p->setSeverity(KDevelop::IProblem::Warning); p->setDescription(i18n("The PEP8 syntax checker does not seem to work correctly.") + "\n" + error); ProblemPointer ptr(p); m_currentlyChecking->addProblem(ptr); } };