diff --git a/parser/astbuilder.cpp b/parser/astbuilder.cpp index 1d02d659..4d053c92 100644 --- a/parser/astbuilder.cpp +++ b/parser/astbuilder.cpp @@ -1,766 +1,703 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astbuilder.h" #include "ast.h" #include #include #include #include #include #include #include #include #include #include "python_header.h" #include "astdefaultvisitor.h" #include "cythonsyntaxremover.h" #include #include #include #include #include #include "parserdebug.h" using namespace KDevelop; extern grammar _PyParser_Grammar; namespace Python { class NextAstFindVisitor : public AstDefaultVisitor { public: KTextEditor::Cursor findNext(Python::Ast* node) { m_root = node; auto parent = node; while ( parent->parent && parent->parent->isExpression() ) { parent = parent->parent; } visitNode(parent); while ( ! m_next.isValid() && parent->parent ) { // no next expression found in that statement, advance to the next statement parent = parent->parent; visitNode(parent); } return m_next; }; void visitNode(Python::Ast* node) override { if ( ! node ) { return; } AstDefaultVisitor::visitNode(node); if ( node->start() > m_root->start() && ! node->isChildOf(m_root) ) { m_next = (m_next < node->start() && m_next.isValid()) ? m_next : node->start(); } } private: KTextEditor::Cursor m_next{-1, -1}; Ast* m_root; }; // This class is used to fix some of the remaining issues // with the ranges of objects obtained from the python parser. // Issues addressed are: // 1) decorators and "def" / "class" statements for classes / functions // 2) ranges of aliases // Both issues are easy to correct since the possible syntax is very restricted // (no bracket matching, no strings, no nesting, ...) // For the aliases, fortunately only imports and excepthandlers are affected; // the "with" statement, which has more complicated syntax, provides // the necessary information already. // 3) attribute rangees // Not so easy, but when starting from the end of the expression it works okay class RangeFixVisitor : public AstDefaultVisitor { public: RangeFixVisitor(const QString& contents) : lines(contents.split('\n')) { }; void visitNode(Ast* node) override { AstDefaultVisitor::visitNode(node); if ( node && node->parent && node->parent->astType != Ast::AttributeAstType ) { if ( ( node->parent->endLine <= node->endLine && node->parent->endCol <= node->endCol ) || node->parent->endLine < node->endLine ) { node->parent->endLine = node->endLine; node->parent->endCol = node->endCol; } } }; void visitFunctionDefinition(FunctionDefinitionAst* node) override { cutDefinitionPreamble(node->name, node->async ? "asyncdef" : "def"); AstDefaultVisitor::visitFunctionDefinition(node); }; void visitClassDefinition(ClassDefinitionAst* node) override { cutDefinitionPreamble(node->name, "class"); AstDefaultVisitor::visitClassDefinition(node); }; void visitAttribute(AttributeAst* node) override { // Work around the weird way to count columns in Python's AST module. // Find where the next expression (of any kind) behind this one starts NextAstFindVisitor v; auto next_start = v.findNext(node); if ( ! next_start.isValid() ) { // use end of document as reference next_start = {lines.size() - 1, lines.last().size() - 1}; } // take only the portion of the line up to that next expression auto endLine = next_start.line(); auto endCol = next_start.column(); if ( ! (next_start > node->start()) ) { endLine = node->startLine; endCol = -1; } const QString& name(node->attribute->value); QString line; for ( int n = node->startLine, pos = node->value->endCol + 1, dotFound = false, nameFound = false; n <= endLine; ++n, pos = 0 ) { line = lines.at(n); if ( n == endLine && endCol != -1 ) { // Never look at the next expression. line = line.left(endCol); } if ( !dotFound ) { // The real attr name can never be before a dot. // Nor can the start of a comment. // (Don't be misled by `foo["bar"].bar` or `foo["#"].bar`) pos = line.indexOf('.', pos); if ( pos == -1 ) continue; dotFound = true; } if ( !nameFound ) { // Track if the attr name has appeared at least once. // This helps avoid interpreting '#'s in strings as comments - // there can never be a comment before the real attr name. pos = line.indexOf(name, pos + 1); if ( pos == -1 ) continue; nameFound = true; } if ( dotFound && nameFound && (pos = line.indexOf('#', pos + name.length())) != -1) { // Remove the comment after a '#' iff we're certain it can't // be inside a string literal (e.g. `foo["#"].bar`). line = line.left(pos); } // Take the last occurrence, any others are in string literals. pos = line.lastIndexOf(name); if ( pos != -1 ) { node->startLine = n; node->startCol = pos; } // N.B. we do this for all lines, the last non-comment occurrence // is the real one. } // This fails (only, AFAIK) in a very limited case: // If the value expression (`foo` in `foo.bar`) contains a dot, the // attr name, _and_ a hash in that order (may not be consecutive), // and the hash is on the same line as the real attr name, // we wrongly interpret the hash as the start of a comment. // e.g `foo["...barrier#"].bar` will highlight part of the string. node->endLine = node->startLine; node->endCol = node->startCol + name.length() - 1; node->attribute->copyRange(node); AstDefaultVisitor::visitAttribute(node); }; // alias for imports (import foo as bar, baz as bang) // no strings, brackets, or whatever are allowed here, so the "parser" // can be very straightforward. void visitImport(ImportAst* node) override { AstDefaultVisitor::visitImport(node); int aliasIndex = 0; foreach ( AliasAst* alias, node->names ) { fixAlias(alias->name, alias->asName, node->startLine, aliasIndex); aliasIndex += 1; } }; // alias for exceptions (except FooBarException as somethingterriblehappened: ...) void visitExceptionHandler(ExceptionHandlerAst* node) override { AstDefaultVisitor::visitExceptionHandler(node); if ( ! node->name ) { return; } const QString& line = lines.at(node->startLine); const int end = line.count() - 1; int back = backtrackDottedName(line, end); node->name->startCol = end - back; node->name->endCol = end; } void visitString(Python::StringAst* node) override { AstDefaultVisitor::visitString(node); auto match = findString.match(lines.at(node->startLine), node->startCol); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength() - 1; // Ranges are inclusive. } } void visitBytes(Python::BytesAst* node) override { AstDefaultVisitor::visitBytes(node); auto match = findString.match(lines.at(node->startLine), node->startCol + 1); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength(); // -1 then +1, because of the 'b'. } } void visitFormattedValue(Python::FormattedValueAst * node) override { AstDefaultVisitor::visitFormattedValue(node); auto match = findString.match(lines.at(node->startLine), node->startCol + 1); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength(); } } void visitNumber(Python::NumberAst* node) override { AstDefaultVisitor::visitNumber(node); auto match = findNumber.match(lines.at(node->startLine), node->startCol); if ( match.capturedLength() > 0 ) { node->endCol += match.capturedLength() - 1; // Ranges are inclusive. } } // Add one column after the last child to cover the closing bracket: `[1,2,3]` // TODO This is still wrong if the last child is followed by parens or whitespace. // endCol matters most in single-line expressions, so this isn't a huge problem. void visitSubscript(Python::SubscriptAst* node) override { AstDefaultVisitor::visitSubscript(node); node->endCol++; } void visitComprehension(Python::ComprehensionAst* node) override { AstDefaultVisitor::visitComprehension(node); node->endCol++; } void visitList(Python::ListAst* node) override { AstDefaultVisitor::visitList(node); node->endCol++; } void visitTuple(Python::TupleAst* node) override { AstDefaultVisitor::visitTuple(node); node->endCol++; } private: const QStringList lines; QVector dots; KTextEditor::Cursor attributeStart; static const QRegularExpression findString; static const QRegularExpression findNumber; // skip the decorators and the "def" at the beginning // of a class or function declaration and modify @arg node // example: // @decorate(foo) // @decorate(bar) // class myclass(parent): pass // before: start of class->name is [0, 0] // after: start of class->name is [2, 5] // line continuation characters are not supported, // because code needing those in this case is not worth being supported. void cutDefinitionPreamble(Ast* fixNode, const QString& defKeyword) { if ( ! fixNode ) { return; } int currentLine = fixNode->startLine; // cut away decorators while ( currentLine < lines.size() ) { if ( lines.at(currentLine).trimmed().remove(' ').remove('\t').startsWith(defKeyword) ) { // it's not a decorator, so stop skipping lines. break; } currentLine += 1; } // qDebug() << "FIX:" << fixNode->range(); fixNode->startLine = currentLine; fixNode->endLine = currentLine; // qDebug() << "FIXED:" << fixNode->range() << fixNode->astType; // cut away the "def" / "class" int currentColumn = -1; if ( currentLine > lines.size() ) { // whops? return; } const QString& lineData = lines.at(currentLine); bool keywordFound = false; while ( currentColumn < lineData.size() - 1 ) { currentColumn += 1; if ( lineData.at(currentColumn).isSpace() ) { // skip space at the beginning of the line continue; } else if ( keywordFound ) { // if the "def" / "class" was already found, and the current char is // non space, then this is indeed the start of the identifier we're looking for. break; } else { keywordFound = true; currentColumn += defKeyword.size(); } } const int previousLength = fixNode->endCol - fixNode->startCol; fixNode->startCol = currentColumn; fixNode->endCol = currentColumn + previousLength; }; int backtrackDottedName(const QString& data, const int start) { bool haveDot = true; bool previousWasSpace = true; for ( int i = start - 1; i >= 0; i-- ) { if ( data.at(i).isSpace() ) { previousWasSpace = true; continue; } if ( data.at(i) == ':' ) { // excepthandler continue; } if ( data.at(i) == '.' ) { haveDot = true; } else if ( haveDot ) { haveDot = false; previousWasSpace = false; continue; } if ( previousWasSpace && ! haveDot ) { return start-i-2; } previousWasSpace = false; } return 0; } void fixAlias(Ast* dotted, Ast* asname, const int startLine, int aliasIndex) { if ( ! asname && ! dotted ) { return; } QString line = lines.at(startLine); int lineno = startLine; for ( int i = 0; i < line.size(); i++ ) { const QChar& current = line.at(i); if ( current == '\\' ) { // line continuation character // splitting like "import foo as \ \n bar" is not supported. lineno += 1; line = lines.at(lineno); i = 0; continue; } if ( current == ',' ) { if ( aliasIndex == 0 ) { // nothing found, continue below line = line.left(i); break; } // next alias expression aliasIndex -= 1; } if ( i > line.length() - 3 ) { continue; } if ( current.isSpace() && line.mid(i+1).startsWith("as") && ( line.at(i+3).isSpace() || line.at(i+3) == '\\' ) ) { // there's an "as" if ( aliasIndex == 0 ) { // it's the one we're looking for // find the expression if ( dotted ) { int dottedNameLength = backtrackDottedName(line, i); dotted->startLine = lineno; dotted->endLine = lineno; dotted->startCol = i-dottedNameLength; dotted->endCol = i; } // find the asname if ( asname ) { bool atStart = true; int textStart = i+3; for ( int j = i+3; j < line.size(); j++ ) { if ( atStart && ! line.at(j).isSpace() ) { atStart = false; textStart = j; } if ( ! atStart && ( line.at(j).isSpace() || j == line.size() - 1 ) ) { // found it asname->startLine = lineno; asname->endLine = lineno; asname->startCol = textStart - 1; asname->endCol = j; } } } return; } } } // no "as" found, use last dotted name in line const int end = line.count() - whitespaceAtEnd(line); int back = backtrackDottedName(line, end); dotted->startLine = lineno; dotted->endLine = lineno; dotted->startCol = end - back; dotted->endCol = end; }; int whitespaceAtEnd(const QString& line) { for ( int i = 0; i < line.size(); i++ ) { if ( ! line.at(line.size() - i - 1).isSpace() ) { return i; } } return 0; }; }; // FIXME This doesn't work for triple-quoted strings // (it gives length 2, which is no worse than before). const QRegularExpression RangeFixVisitor::findString = QRegularExpression("\\G(['\"]).*?(?(PyObject_Str(obj), pyObjectCleanup); const auto str = strOwner.get(); if (PyUnicode_READY(str) < 0) { qWarning("PyUnicode_READY(%p) returned false!", (void*)str); return QString(); } const auto length = PyUnicode_GET_LENGTH(str); switch(PyUnicode_KIND(str)) { case PyUnicode_1BYTE_KIND: return QString::fromLatin1((const char*)PyUnicode_1BYTE_DATA(str), length); case PyUnicode_2BYTE_KIND: return QString::fromUtf16(PyUnicode_2BYTE_DATA(str), length); case PyUnicode_4BYTE_KIND: return QString::fromUcs4(PyUnicode_4BYTE_DATA(str), length); case PyUnicode_WCHAR_KIND: qWarning("PyUnicode_KIND(%p) returned PyUnicode_WCHAR_KIND, this should not happen!", (void*)str); return QString::fromWCharArray(PyUnicode_AS_UNICODE(str), length); } Q_UNREACHABLE(); } -QPair fileHeaderHack(QString& contents, const QUrl& filename) -{ - IProject* proj = ICore::self()->projectController()->findProjectForUrl(filename); - // the file is not in a project, don't apply hack - if ( ! proj ) { - return QPair(contents, 0); - } - const QUrl headerFileUrl = QUrl::fromLocalFile(proj->path().path() + "/.kdev_python_header"); - QFile headerFile(headerFileUrl.path()); - QString headerFileContents; - if ( headerFile.exists() ) { - headerFile.open(QIODevice::ReadOnly); - headerFileContents = headerFile.readAll(); - headerFile.close(); - qCDebug(KDEV_PYTHON_PARSER) << "Found header file, applying hack"; - int insertAt = 0; - bool endOfCommentsReached = false; - bool commentSignEncountered = false; -// bool atLineBeginning = true; - int lastLineBeginning = 0; - int newlineCount = 0; - int l = contents.length(); - do { - if ( insertAt >= l ) { - qCDebug(KDEV_PYTHON_PARSER) << "File consist only of comments, not applying hack"; - return QPair(contents, 0); - } - if ( contents.at(insertAt) == '#' ) { - commentSignEncountered = true; - } - if ( !contents.at(insertAt).isSpace() ) { -// atLineBeginning = false; - if ( !commentSignEncountered ) { - endOfCommentsReached = true; - } - } - if ( contents.at(insertAt) == '\n' ) { -// atLineBeginning = true; - commentSignEncountered = false; - lastLineBeginning = insertAt; - newlineCount += 1; - } - if ( newlineCount == 2 ) { - endOfCommentsReached = true; - } - insertAt += 1; - } while ( !endOfCommentsReached ); - qCDebug(KDEV_PYTHON_PARSER) << "Inserting contents at char" << lastLineBeginning << "of file"; - contents = contents.left(lastLineBeginning) - + "\n" + headerFileContents + "\n#\n" - + contents.right(contents.length() - lastLineBeginning); - qCDebug(KDEV_PYTHON_PARSER) << contents; - return QPair(contents, - ( headerFileContents.count('\n') + 3 )); - } - else { - return QPair(contents, 0); - } -} - namespace { struct PythonInitializer : private QMutexLocker { PythonInitializer(QMutex& pyInitLock): QMutexLocker(&pyInitLock), arena(nullptr) { Py_InitializeEx(0); Q_ASSERT(Py_IsInitialized()); arena = PyArena_New(); Q_ASSERT(arena); // out of memory } ~PythonInitializer() { if (arena) PyArena_Free(arena); if (Py_IsInitialized()) Py_Finalize(); } PyArena* arena; }; } CodeAst::Ptr AstBuilder::parse(const QUrl& filename, QString &contents) { qCDebug(KDEV_PYTHON_PARSER) << " ====> AST ====> building abstract syntax tree for " << filename.path(); Py_NoSiteFlag = 1; contents.append('\n'); - QPair hacked = fileHeaderHack(contents, filename); - contents = hacked.first; - int lineOffset = hacked.second; - PythonInitializer pyIniter(pyInitLock); PyArena* arena = pyIniter.arena; PyCompilerFlags flags = {PyCF_SOURCE_IS_UTF8 | PyCF_IGNORE_COOKIE}; PyObject *exception, *value, *backtrace; PyErr_Fetch(&exception, &value, &backtrace); CythonSyntaxRemover cythonSyntaxRemover; if (filename.fileName().endsWith(".pyx", Qt::CaseInsensitive)) { qCDebug(KDEV_PYTHON_PARSER) << filename.fileName() << "is probably Cython file."; contents = cythonSyntaxRemover.stripCythonSyntax(contents); } mod_ty syntaxtree = PyParser_ASTFromString(contents.toUtf8().data(), "", file_input, &flags, arena); if ( ! syntaxtree ) { qCDebug(KDEV_PYTHON_PARSER) << " ====< parse error, trying to fix"; PyErr_Fetch(&exception, &value, &backtrace); qCDebug(KDEV_PYTHON_PARSER) << "Error objects: " << exception << value << backtrace; if ( ! value ) { qCWarning(KDEV_PYTHON_PARSER) << "Internal parser error: exception value is null, aborting"; return CodeAst::Ptr(); } PyObject_Print(value, stderr, Py_PRINT_RAW); PyObject* errorMessage_str = PyTuple_GetItem(value, 0); PyObject* errorDetails_tuple = PyTuple_GetItem(value, 1); if ( ! errorDetails_tuple ) { qCWarning(KDEV_PYTHON_PARSER) << "Error retrieving error message, not displaying, and not doing anything"; return CodeAst::Ptr(); } PyObject* linenoobj = PyTuple_GetItem(errorDetails_tuple, 1); errorMessage_str = PyTuple_GetItem(value, 0); errorDetails_tuple = PyTuple_GetItem(value, 1); PyObject_Print(errorMessage_str, stderr, Py_PRINT_RAW); PyObject* colnoobj = PyTuple_GetItem(errorDetails_tuple, 2); int lineno = PyLong_AsLong(linenoobj) - 1; int colno = PyLong_AsLong(colnoobj); ProblemPointer p(new Problem()); - KTextEditor::Cursor start(lineno + lineOffset, (colno-4 > 0 ? colno-4 : 0)); - KTextEditor::Cursor end(lineno + lineOffset, (colno+4 > 4 ? colno+4 : 4)); + KTextEditor::Cursor start(lineno, (colno-4 > 0 ? colno-4 : 0)); + KTextEditor::Cursor end(lineno, (colno+4 > 4 ? colno+4 : 4)); KTextEditor::Range range(start, end); qCDebug(KDEV_PYTHON_PARSER) << "Problem range: " << range; DocumentRange location(IndexedString(filename.path()), range); p->setFinalLocation(location); p->setDescription(PyUnicodeObjectToQString(errorMessage_str)); p->setSource(IProblem::Parser); m_problems.append(p); // try to recover. // Currently the following is tired: // * If the last non-space char before the error reported was ":", it's most likely an indent error. // The common easy-to-fix and annoying indent error is "for item in foo: ". In that case, just add "pass" after the ":" token. // * If it's not, we will just comment the line with the error, fixing problems like "foo = ". // * If both fails, everything including the first non-empty line before the one with the error will be deleted. int len = contents.length(); int currentLine = 0; QString currentLineContents; QChar c; QChar newline('\n'); int emptySince = 0; int emptySinceLine = 0; int emptyLinesSince = 0; int emptyLinesSinceLine = 0; unsigned short currentLineIndent = 0; bool atLineBeginning = true; QList indents; int errline = qMax(0, lineno); int currentLineBeginning = 0; for ( int i = 0; i < len; i++ ) { c = contents.at(i); if ( ! c.isSpace() ) { emptySince = i; emptySinceLine = currentLine; atLineBeginning = false; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); } else if ( c == newline ) { if ( currentLine == errline ) { atLineBeginning = false; } else { currentLine += 1; currentLineBeginning = i+1; // this line has had content, so reset the "empty lines since" counter if ( ! atLineBeginning ) { // lastNonemptyLineBeginning = emptyLinesSince; emptyLinesSince = i; emptyLinesSinceLine = currentLine; } atLineBeginning = true; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); currentLineIndent = 0; } } else if ( atLineBeginning ) { currentLineIndent += 1; } if ( currentLine == errline && ! atLineBeginning ) { // if the last non-empty char before the error opens a new block, it's likely an "empty block" problem // we can easily fix that by adding in a "pass" statement. However, we want to add that in the next line, if possible // so context ranges for autocompletion stay intact. if ( contents[emptySince] == QChar(':') ) { qCDebug(KDEV_PYTHON_PARSER) << indents.length() << emptySinceLine + 1 << indents; if ( indents.length() > emptySinceLine + 1 && indents.at(emptySinceLine) < indents.at(emptySinceLine + 1) ) { qCDebug(KDEV_PYTHON_PARSER) << indents.at(emptySinceLine) << indents.at(emptySinceLine + 1); contents.insert(emptyLinesSince + 1 + indents.at(emptyLinesSinceLine), "\tpass#"); } else { contents.insert(emptySince + 1, "\tpass#"); } } else if ( indents.length() >= currentLine && currentLine > 0 ) { qCDebug(KDEV_PYTHON_PARSER) << indents << currentLine; contents[i+1+indents.at(currentLine - 1)] = QChar('#'); contents.insert(i+1+indents.at(currentLine - 1), "pass"); } break; } } syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); // 3rd try: discard everything after the last non-empty line, but only until the next block start currentLineBeginning = qMin(contents.length() - 1, currentLineBeginning); errline = qMax(0, qMin(indents.length()-1, errline)); if ( ! syntaxtree ) { qCWarning(KDEV_PYTHON_PARSER) << "Discarding parts of the code to be parsed because of previous errors"; qCDebug(KDEV_PYTHON_PARSER) << indents; int indentAtError = indents.at(errline); QChar c; bool atLineBeginning = true; int currentIndent = -1; int currentLineBeginning_end = currentLineBeginning; int currentLineContentBeginning = currentLineBeginning; for ( int i = currentLineBeginning; i < len; i++ ) { c = contents.at(i); qCDebug(KDEV_PYTHON_PARSER) << c; if ( c == '\n' ) { if ( currentIndent <= indentAtError && currentIndent != -1 ) { qCDebug(KDEV_PYTHON_PARSER) << "Start of error code: " << currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "End of error block (current position): " << currentLineBeginning_end; qCDebug(KDEV_PYTHON_PARSER) << "Length: " << currentLineBeginning_end - currentLineBeginning; qCDebug(KDEV_PYTHON_PARSER) << "indent at error <> current indent:" << indentAtError << "<>" << currentIndent; // contents.remove(currentLineBeginning, currentLineBeginning_end-currentLineBeginning); break; } contents.insert(currentLineContentBeginning - 1, "pass#"); i += 5; i = qMin(i, contents.length()); len = contents.length(); atLineBeginning = true; currentIndent = 0; currentLineBeginning_end = i + 1; currentLineContentBeginning = i + 1; continue; } if ( ! c.isSpace() && atLineBeginning ) { currentLineContentBeginning = i; atLineBeginning = false; } if ( c.isSpace() && atLineBeginning ) currentIndent += 1; } qCDebug(KDEV_PYTHON_PARSER) << "This is what is left: " << contents; syntaxtree = PyParser_ASTFromString(contents.toUtf8(), "", file_input, &flags, arena); } if ( ! syntaxtree ) { return CodeAst::Ptr(); // everything fails, so we abort. } } qCDebug(KDEV_PYTHON_PARSER) << "Got syntax tree from python parser:" << syntaxtree->kind << Module_kind; - PythonAstTransformer t(lineOffset); + PythonAstTransformer t; t.run(syntaxtree, filename.fileName().replace(".py", "")); RangeFixVisitor fixVisitor(contents); fixVisitor.visitNode(t.ast); cythonSyntaxRemover.fixAstRanges(t.ast); return CodeAst::Ptr(t.ast); } } diff --git a/parser/generated.h b/parser/generated.h index 1e6e3a51..1087ac2e 100644 --- a/parser/generated.h +++ b/parser/generated.h @@ -1,794 +1,792 @@ /* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ #include #include "kdevpythonversion.h" class PythonAstTransformer { public: CodeAst* ast; - PythonAstTransformer(int lineOffset) : m_lineOffset(lineOffset) {}; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } // Shift lines by some fixed amount inline int tline(int line) { if ( line == -99999 ) { // don't touch the marker return -99999; } - return line + m_lineOffset; + return line; }; private: QStack nodeStack; - int m_lineOffset; - + Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } Ast* visitNode(_alias* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; AliasAst* v = new AliasAst(parent()); v->name = node->name ? new Python::Identifier(PyUnicodeObjectToQString(node->name)) : nullptr; v->asName = node->asname ? new Python::Identifier(PyUnicodeObjectToQString(node->asname)) : nullptr; return v; } Ast* visitNode(_arg* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgAst* v = new ArgAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; if ( v->argumentName ) { v->argumentName->startCol = node->col_offset; v->startCol = v->argumentName->startCol; v->argumentName->startLine = tline(node->lineno - 1); v->startLine = v->argumentName->startLine; v->argumentName->endCol = node->col_offset + v->argumentName->value.length() - 1; v->endCol = v->argumentName->endCol; v->argumentName->endLine = tline(node->lineno - 1); v->endLine = v->argumentName->endLine; ranges_copied = true; } nodeStack.push(v); v->annotation = static_cast(visitNode(node->annotation)); nodeStack.pop(); return v; } Ast* visitNode(_arguments* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ArgumentsAst* v = new ArgumentsAst(parent()); nodeStack.push(v); v->vararg = static_cast(visitNode(node->vararg)); nodeStack.pop(); nodeStack.push(v); v->kwarg = static_cast(visitNode(node->kwarg)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_arg, ArgAst>(node->args); nodeStack.pop(); nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); nodeStack.push(v); v->kwonlyargs = visitNodeList<_arg, ArgAst>(node->kwonlyargs); nodeStack.pop(); return v; } Ast* visitNode(_comprehension* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; ComprehensionAst* v = new ComprehensionAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->iter)); nodeStack.pop(); nodeStack.push(v); v->conditions = visitNodeList<_expr, ExpressionAst>(node->ifs); nodeStack.pop(); return v; } Ast* visitNode(_excepthandler* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case ExceptHandler_kind: { ExceptionHandlerAst* v = new ExceptionHandlerAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.ExceptHandler.type)); nodeStack.pop(); v->name = node->v.ExceptHandler.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ExceptHandler.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ExceptHandler.body); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _excepthandler AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_expr* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Await_kind: { AwaitAst* v = new AwaitAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Await.value)); nodeStack.pop(); result = v; break; } #endif case BoolOp_kind: { BooleanOperationAst* v = new BooleanOperationAst(parent()); v->type = (ExpressionAst::BooleanOperationTypes) node->v.BoolOp.op; nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.BoolOp.values); nodeStack.pop(); result = v; break; } case BinOp_kind: { BinaryOperationAst* v = new BinaryOperationAst(parent()); v->type = (ExpressionAst::OperatorTypes) node->v.BinOp.op; nodeStack.push(v); v->lhs = static_cast(visitNode(node->v.BinOp.left)); nodeStack.pop(); nodeStack.push(v); v->rhs = static_cast(visitNode(node->v.BinOp.right)); nodeStack.pop(); result = v; break; } case UnaryOp_kind: { UnaryOperationAst* v = new UnaryOperationAst(parent()); v->type = (ExpressionAst::UnaryOperatorTypes) node->v.UnaryOp.op; nodeStack.push(v); v->operand = static_cast(visitNode(node->v.UnaryOp.operand)); nodeStack.pop(); result = v; break; } case Lambda_kind: { LambdaAst* v = new LambdaAst(parent()); nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.Lambda.args)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Lambda.body)); nodeStack.pop(); result = v; break; } case IfExp_kind: { IfExpressionAst* v = new IfExpressionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.IfExp.test)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.IfExp.body)); nodeStack.pop(); nodeStack.push(v); v->orelse = static_cast(visitNode(node->v.IfExp.orelse)); nodeStack.pop(); result = v; break; } case Dict_kind: { DictAst* v = new DictAst(parent()); nodeStack.push(v); v->keys = visitNodeList<_expr, ExpressionAst>(node->v.Dict.keys); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Dict.values); nodeStack.pop(); result = v; break; } case Set_kind: { SetAst* v = new SetAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Set.elts); nodeStack.pop(); result = v; break; } case ListComp_kind: { ListComprehensionAst* v = new ListComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.ListComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.ListComp.generators); nodeStack.pop(); result = v; break; } case SetComp_kind: { SetComprehensionAst* v = new SetComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.SetComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.SetComp.generators); nodeStack.pop(); result = v; break; } case DictComp_kind: { DictionaryComprehensionAst* v = new DictionaryComprehensionAst(parent()); nodeStack.push(v); v->key = static_cast(visitNode(node->v.DictComp.key)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.DictComp.value)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.DictComp.generators); nodeStack.pop(); result = v; break; } case GeneratorExp_kind: { GeneratorExpressionAst* v = new GeneratorExpressionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.GeneratorExp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.GeneratorExp.generators); nodeStack.pop(); result = v; break; } case Yield_kind: { YieldAst* v = new YieldAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Yield.value)); nodeStack.pop(); result = v; break; } case Compare_kind: { CompareAst* v = new CompareAst(parent()); nodeStack.push(v); v->leftmostElement = static_cast(visitNode(node->v.Compare.left)); nodeStack.pop(); for ( int _i = 0; _i < node->v.Compare.ops->size; _i++ ) { v->operators.append((ExpressionAst::ComparisonOperatorTypes) node->v.Compare.ops->elements[_i]); } nodeStack.push(v); v->comparands = visitNodeList<_expr, ExpressionAst>(node->v.Compare.comparators); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION < QT_VERSION_CHECK(3, 5, 0) case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); /* Convert 3.4 unpacked-args AST to match the new format from 3.5+ */if (node->v.Call.starargs) { nodeStack.push(v); auto starred = new StarredAst(v); starred->context = ExpressionAst::Context::Load; nodeStack.push(starred); starred->value = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->arguments.append(starred); nodeStack.pop();};if (node->v.Call.kwargs) { nodeStack.push(v); auto kwargs = new KeywordAst(v); nodeStack.push(kwargs); kwargs->value = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); v->keywords.append(kwargs); nodeStack.pop();}; result = v; break; } #endif case Num_kind: { NumberAst* v = new NumberAst(parent()); v->isInt = PyLong_Check(node->v.Num.n); v->value = PyLong_AsLong(node->v.Num.n); result = v; break; } case Str_kind: { StringAst* v = new StringAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Str.s); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case JoinedStr_kind: { JoinedStringAst* v = new JoinedStringAst(parent()); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.JoinedStr.values); nodeStack.pop(); result = v; break; } #endif #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case FormattedValue_kind: { FormattedValueAst* v = new FormattedValueAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.FormattedValue.value)); nodeStack.pop(); v->conversion = node->v.FormattedValue.conversion; nodeStack.push(v); v->formatSpec = static_cast(visitNode(node->v.FormattedValue.format_spec)); nodeStack.pop(); result = v; break; } #endif case Bytes_kind: { BytesAst* v = new BytesAst(parent()); v->value = PyUnicodeObjectToQString(node->v.Bytes.s); result = v; break; } case Attribute_kind: { AttributeAst* v = new AttributeAst(parent()); v->attribute = node->v.Attribute.attr ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Attribute.attr)) : nullptr; if ( v->attribute ) { v->attribute->startCol = node->col_offset; v->startCol = v->attribute->startCol; v->attribute->startLine = tline(node->lineno - 1); v->startLine = v->attribute->startLine; v->attribute->endCol = node->col_offset + v->attribute->value.length() - 1; v->endCol = v->attribute->endCol; v->attribute->endLine = tline(node->lineno - 1); v->endLine = v->attribute->endLine; ranges_copied = true; } nodeStack.push(v); v->value = static_cast(visitNode(node->v.Attribute.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Attribute.ctx; result = v; break; } case Subscript_kind: { SubscriptAst* v = new SubscriptAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Subscript.value)); nodeStack.pop(); nodeStack.push(v); v->slice = static_cast(visitNode(node->v.Subscript.slice)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Subscript.ctx; result = v; break; } case Starred_kind: { StarredAst* v = new StarredAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Starred.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Starred.ctx; result = v; break; } case Name_kind: { NameAst* v = new NameAst(parent()); v->identifier = node->v.Name.id ? new Python::Identifier(PyUnicodeObjectToQString(node->v.Name.id)) : nullptr; if ( v->identifier ) { v->identifier->startCol = node->col_offset; v->startCol = v->identifier->startCol; v->identifier->startLine = tline(node->lineno - 1); v->startLine = v->identifier->startLine; v->identifier->endCol = node->col_offset + v->identifier->value.length() - 1; v->endCol = v->identifier->endCol; v->identifier->endLine = tline(node->lineno - 1); v->endLine = v->identifier->endLine; ranges_copied = true; } v->context = (ExpressionAst::Context) node->v.Name.ctx; result = v; break; } case List_kind: { ListAst* v = new ListAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.List.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.List.ctx; result = v; break; } case Tuple_kind: { TupleAst* v = new TupleAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Tuple.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Tuple.ctx; result = v; break; } case Ellipsis_kind: { EllipsisAst* v = new EllipsisAst(parent()); result = v; break; } case NameConstant_kind: { NameConstantAst* v = new NameConstantAst(parent()); v->value = node->v.NameConstant.value == Py_None ? NameConstantAst::None : node->v.NameConstant.value == Py_False ? NameConstantAst::False : NameConstantAst::True; result = v; break; } case YieldFrom_kind: { YieldFromAst* v = new YieldFromAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.YieldFrom.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _expr AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_keyword* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; KeywordAst* v = new KeywordAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyUnicodeObjectToQString(node->arg)) : nullptr; nodeStack.push(v); v->value = static_cast(visitNode(node->value)); nodeStack.pop(); return v; } Ast* visitNode(_slice* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Slice_kind: { SliceAst* v = new SliceAst(parent()); nodeStack.push(v); v->lower = static_cast(visitNode(node->v.Slice.lower)); nodeStack.pop(); nodeStack.push(v); v->upper = static_cast(visitNode(node->v.Slice.upper)); nodeStack.pop(); nodeStack.push(v); v->step = static_cast(visitNode(node->v.Slice.step)); nodeStack.pop(); result = v; break; } case ExtSlice_kind: { ExtendedSliceAst* v = new ExtendedSliceAst(parent()); nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop(); result = v; break; } case Index_kind: { IndexAst* v = new IndexAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Index.value)); nodeStack.pop(); result = v; break; } default: qWarning() << "Unsupported _slice AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_stmt* node) { if ( ! node ) return nullptr; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = nullptr; switch ( node->kind ) { case Expr_kind: { ExpressionAst* v = new ExpressionAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Expr.value)); nodeStack.pop(); result = v; break; } case FunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.FunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.FunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.FunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.FunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.FunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.FunctionDef.returns)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.AsyncFunctionDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.AsyncFunctionDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.AsyncFunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.AsyncFunctionDef.decorator_list); nodeStack.pop(); nodeStack.push(v); v->returns = static_cast(visitNode(node->v.AsyncFunctionDef.returns)); nodeStack.pop(); v->async = true; result = v; break; } #endif case ClassDef_kind: { ClassDefinitionAst* v = new ClassDefinitionAst(parent()); v->name = node->v.ClassDef.name ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ClassDef.name)) : nullptr; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->baseClasses = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.bases); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ClassDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.decorator_list); nodeStack.pop(); result = v; break; } case Return_kind: { ReturnAst* v = new ReturnAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Return.value)); nodeStack.pop(); result = v; break; } case Delete_kind: { DeleteAst* v = new DeleteAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Delete.targets); nodeStack.pop(); result = v; break; } case Assign_kind: { AssignmentAst* v = new AssignmentAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Assign.targets); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Assign.value)); nodeStack.pop(); result = v; break; } case AugAssign_kind: { AugmentedAssignmentAst* v = new AugmentedAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AugAssign.target)); nodeStack.pop(); v->op = (ExpressionAst::OperatorTypes) node->v.AugAssign.op; nodeStack.push(v); v->value = static_cast(visitNode(node->v.AugAssign.value)); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0) case AnnAssign_kind: { AnnotationAssignmentAst* v = new AnnotationAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AnnAssign.target)); nodeStack.pop(); nodeStack.push(v); v->annotation = static_cast(visitNode(node->v.AnnAssign.annotation)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.AnnAssign.value)); nodeStack.pop(); result = v; break; } #endif case For_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.For.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.For.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.For.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.For.orelse); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncFor_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AsyncFor.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.AsyncFor.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncFor.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.AsyncFor.orelse); nodeStack.pop(); result = v; break; } #endif case While_kind: { WhileAst* v = new WhileAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.While.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.While.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.While.orelse); nodeStack.pop(); result = v; break; } case If_kind: { IfAst* v = new IfAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.If.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.If.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.If.orelse); nodeStack.pop(); result = v; break; } case With_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.With.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.With.items); nodeStack.pop(); result = v; break; } #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0) case AsyncWith_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.AsyncWith.body); nodeStack.pop(); nodeStack.push(v); v->items = visitNodeList<_withitem, WithItemAst>(node->v.AsyncWith.items); nodeStack.pop(); result = v; break; } #endif case Raise_kind: { RaiseAst* v = new RaiseAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.Raise.exc)); nodeStack.pop(); result = v; break; } case Try_kind: { TryAst* v = new TryAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.Try.body); nodeStack.pop(); nodeStack.push(v); v->handlers = visitNodeList<_excepthandler, ExceptionHandlerAst>(node->v.Try.handlers); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.Try.orelse); nodeStack.pop(); nodeStack.push(v); v->finally = visitNodeList<_stmt, Ast>(node->v.Try.finalbody); nodeStack.pop(); result = v; break; } case Assert_kind: { AssertionAst* v = new AssertionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.Assert.test)); nodeStack.pop(); nodeStack.push(v); v->message = static_cast(visitNode(node->v.Assert.msg)); nodeStack.pop(); result = v; break; } case Import_kind: { ImportAst* v = new ImportAst(parent()); nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.Import.names); nodeStack.pop(); result = v; break; } case ImportFrom_kind: { ImportFromAst* v = new ImportFromAst(parent()); v->module = node->v.ImportFrom.module ? new Python::Identifier(PyUnicodeObjectToQString(node->v.ImportFrom.module)) : nullptr; if ( v->module ) { v->module->startCol = node->col_offset; v->startCol = v->module->startCol; v->module->startLine = tline(node->lineno - 1); v->startLine = v->module->startLine; v->module->endCol = node->col_offset + v->module->value.length() - 1; v->endCol = v->module->endCol; v->module->endLine = tline(node->lineno - 1); v->endLine = v->module->endLine; ranges_copied = true; } nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.ImportFrom.names); nodeStack.pop(); v->level = node->v.ImportFrom.level; result = v; break; } case Global_kind: { GlobalAst* v = new GlobalAst(parent()); for ( int _i = 0; _i < node->v.Global.names->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyUnicodeObjectToQString( static_cast(node->v.Global.names->elements[_i]) )); v->names.append(id); } result = v; break; } case Break_kind: { BreakAst* v = new BreakAst(parent()); result = v; break; } case Continue_kind: { ContinueAst* v = new ContinueAst(parent()); result = v; break; } case Pass_kind: { PassAst* v = new PassAst(parent()); result = v; break; } case Nonlocal_kind: { NonlocalAst* v = new NonlocalAst(parent()); result = v; break; } default: qWarning() << "Unsupported _stmt AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return nullptr; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } if ( result && result->astType == Ast::NameAstType ) { NameAst* r = static_cast(result); r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_withitem* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return nullptr; WithItemAst* v = new WithItemAst(parent()); nodeStack.push(v); v->contextExpression = static_cast(visitNode(node->context_expr)); nodeStack.pop(); nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->optional_vars)); nodeStack.pop(); return v; } }; /* * End generated code */