diff --git a/documentation_files/__future__.py b/documentation_files/__future__.py index 96b73282..f5e12f2e 100644 --- a/documentation_files/__future__.py +++ b/documentation_files/__future__.py @@ -1,86 +1,88 @@ #!/usr/bin/env python2.7 # -*- coding: utf-8 -*- """:synopsis: Future statement definitions :mod:`__future__` is a real module, and serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to find the modules they're importing. * To ensure that :ref:`future statements ` run under releases prior to 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will fail, because there was no module of that name prior to 2.1). * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and can be inspected programmatically via importing :mod:`__future__` and examining its contents. Each statement in :file:`__future__.py` is of the form:: FeatureName = _Feature(OptionalRelease, MandatoryRelease, CompilerFlag) where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are 5-tuples of the same form as ``sys.version_info``:: (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int PY_MINOR_VERSION, # the 1; an int PY_MICRO_VERSION, # the 0; an int PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string PY_RELEASE_SERIAL # the 3; an int ) *OptionalRelease* records the first release in which the feature was accepted. In the case of a *MandatoryRelease* that has not yet occurred, *MandatoryRelease* predicts the release in which the feature will become part of the language. Else *MandatoryRelease* records when the feature became part of the language; in releases at or after that, modules no longer need a future statement to use the feature in question, but may continue to use such imports. *MandatoryRelease* may also be ``None``, meaning that a planned feature got dropped. Instances of class :class:`_Feature` have two corresponding methods, :meth:`getOptionalRelease` and :meth:`getMandatoryRelease`. *CompilerFlag* is the (bitfield) flag that should be passed in the fourth argument to the built-in function :func:`compile` to enable the feature in dynamically compiled code. This flag is stored in the :attr:`compiler_flag` attribute on :class:`_Feature` instances. No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: +------------------+-------------+--------------+---------------------------------------------+ | feature | optional in | mandatory in | effect | +==================+=============+==============+=============================================+ | nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | | | | | *Statically Nested Scopes* | +------------------+-------------+--------------+---------------------------------------------+ | generators | 2.2.0a1 | 2.3 | :pep:`255`: | | | | | *Simple Generators* | +------------------+-------------+--------------+---------------------------------------------+ | division | 2.2.0a2 | 3.0 | :pep:`238`: | | | | | *Changing the Division Operator* | +------------------+-------------+--------------+---------------------------------------------+ | absolute_import | 2.5.0a1 | 2.7 | :pep:`328`: | | | | | *Imports: Multi-Line and Absolute/Relative* | +------------------+-------------+--------------+---------------------------------------------+ | with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | | | | | *The "with" Statement* | +------------------+-------------+--------------+---------------------------------------------+ | print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | | | | | *Make print a function* | +------------------+-------------+--------------+---------------------------------------------+ | unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | | | | | *Bytes literals in Python 3000* | +------------------+-------------+--------------+---------------------------------------------+ """ + +def print_function(): pass diff --git a/duchain/declarationbuilder.cpp b/duchain/declarationbuilder.cpp index 36102906..bacf3e5d 100644 --- a/duchain/declarationbuilder.cpp +++ b/duchain/declarationbuilder.cpp @@ -1,1155 +1,1155 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2007 Andreas Pakulat * * Copyright 2010 Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "duchain/declarations/decorator.h" #include "contextbuilder.h" #include "declarationbuilder.h" #include "pythoneditorintegrator.h" #include "expressionvisitor.h" #include "helpers.h" #include "types/variablelengthcontainer.h" #include "types/hintedtype.h" #include "types/unsuretype.h" #include "duchain/declarations/functiondeclaration.h" #include "duchain/declarations/classdeclaration.h" using namespace KTextEditor; using namespace KDevelop; namespace Python { DeclarationBuilder::DeclarationBuilder() : DeclarationBuilderBase(), m_prebuilding(false) { kDebug() << "Building Declarations"; } DeclarationBuilder::DeclarationBuilder( PythonEditorIntegrator* editor ) : DeclarationBuilderBase( ), m_prebuilding(false) { setEditor(editor); kDebug() << "Building Declarations"; } DeclarationBuilder:: ~DeclarationBuilder() { } void DeclarationBuilder::setPrebuilding(bool prebuilding) { m_prebuilding = prebuilding; } ReferencedTopDUContext DeclarationBuilder::build(const IndexedString& url, Ast* node, ReferencedTopDUContext updateContext) { if ( ! m_prebuilding ) { kDebug() << "building, but running pre-builder first"; DeclarationBuilder* prebuilder = new DeclarationBuilder(editor()); prebuilder->m_currentlyParsedDocument = currentlyParsedDocument(); prebuilder->setPrebuilding(true); prebuilder->m_futureModificationRevision = m_futureModificationRevision; updateContext = prebuilder->build(url, node, updateContext); kDebug() << "pre-builder finished"; } else { kDebug() << "prebuilding"; } return DeclarationBuilderBase::build(url, node, updateContext); } void DeclarationBuilder::closeDeclaration() { if ( lastContext() ) { DUChainReadLocker lock( DUChain::lock() ); currentDeclaration()->setKind( Declaration::Type ); } Q_ASSERT(currentDeclaration()->alwaysForceDirect()); eventuallyAssignInternalContext(); DeclarationBuilderBase::closeDeclaration(); } template T* DeclarationBuilder::visitVariableDeclaration(Ast* node, Declaration* previous) { NameAst* currentVariableDefinition = dynamic_cast(node); // those contexts can invoke a variable declaration // this prevents "bar" from being declared in something like "foo = bar" QList declaringContexts; declaringContexts << ExpressionAst::Store << ExpressionAst::Parameter << ExpressionAst::AugStore; if ( ! declaringContexts.contains(currentVariableDefinition->context) ) { return 0; } Identifier* id = currentVariableDefinition->identifier; return visitVariableDeclaration(id, currentVariableDefinition, previous); } template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, RangeInRevision range) { Ast* pseudo = new Ast(); pseudo->startLine = range.start.line; pseudo->startCol = range.start.column; pseudo->endLine = range.end.line; pseudo->endCol = range.end.column; T* result = visitVariableDeclaration(node, pseudo, 0); delete pseudo; return result; } typedef QPair p; /* * WARNING: This will return a nullpointer if another than the expected type of variable was found! * */ template T* DeclarationBuilder::visitVariableDeclaration(Identifier* node, Ast* originalAst, Declaration* previous) { DUChainWriteLocker lock(DUChain::lock()); Q_ASSERT(node); kDebug() << "Parsing variable declaration: " << node->value; Declaration* dec = 0; QList existingDeclarations; // specified from outside if ( previous ) { existingDeclarations << previous; kDebug() << previous->toString(); } // find decls by ourselves else { /**DBG**/ kDebug() << "Current context: " << currentContext()->scopeIdentifier() << currentContext()->range().castToSimpleRange(); kDebug() << "Looking for node identifier:" << identifierForNode(node) << identifierForNode(node).last(); /** /DBG **/ existingDeclarations = currentContext()->findDeclarations(identifierForNode(node).last(), // <- WARNING first / last? CursorInRevision::invalid(), 0, DUContext::DontSearchInParent); // append arguments context if ( m_mostRecentArgumentsContext ) { QList args = m_mostRecentArgumentsContext->findDeclarations(identifierForNode(node).last(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent); existingDeclarations.append(args); } kDebug() << "Found " << existingDeclarations.length() << "declarations"; } QList remainingDeclarations; bool declarationOpened = false; foreach ( Declaration* d, existingDeclarations ) { Declaration* fitting = dynamic_cast(d); kDebug() << "last one: " << d << d->toString() << dynamic_cast(d) << wasEncountered(d); bool invalidType = d && d->abstractType() && lastType() && lastType()->whichType() != AbstractType::TypeFunction && d->isFunctionDeclaration(); if ( fitting && ! wasEncountered(d) && ! invalidType ) { if ( d->topContext() == currentContext()->topContext() ) { kDebug() << "Opening previously existing declaration for " << d->toString(); openDeclarationInternal(d); d->setRange(editorFindRange(node, node)); declarationOpened = true; } else { kDebug() << "Not opening previously existing declaration because it's in another top context"; } setEncountered(d); dec = d; } else if ( fitting && ! invalidType ) { remainingDeclarations << fitting; } } existingDeclarations = remainingDeclarations; kDebug() << "VARIABLE CONTEXT: " << currentContext()->scopeIdentifier() << currentContext()->range().castToSimpleRange() << currentContext()->type() << DUContext::Class; bool noFittingDeclaration = existingDeclarations.isEmpty() || ( ! existingDeclarations.isEmpty() && ! dynamic_cast(existingDeclarations.last()) ); if ( currentContext() && currentContext()->type() == DUContext::Class && noFittingDeclaration ) { kDebug() << "Creating class member declaration for " << node->value << node->startLine << ":" << node->startCol; kDebug() << "Context type: " << currentContext()->scopeIdentifier() << currentContext()->range().castToSimpleRange(); if ( ! dec ) { dec = openDeclaration(node, originalAst ? originalAst : node); declarationOpened = true; } if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } dec->setType(lastType()); dec->setKind(KDevelop::Declaration::Instance); } else if ( noFittingDeclaration ) { kDebug() << "Creating variable declaration for " << node->value << node->startLine << ":" << node->startCol << "->" << node->endLine << ":" << node->endCol; kDebug() << "which has type" << ( dec && dec->abstractType() ? dec->abstractType()->toString() : "none" ); if ( ! dec ) { kDebug() << "This declaration is a new one"; dec = openDeclaration(node, originalAst ? originalAst : node); declarationOpened = true; } /* else { kDebug() << "This declaration is old, and has the following type: " << dec->abstractType()->toString(); } */ if ( declarationOpened ) { DeclarationBuilderBase::closeDeclaration(); } UnsureType::Ptr hints = Helper::extractTypeHints(dec->abstractType(), topContext()); kDebug() << "Type Hints: " << hints->toString(); AbstractType::Ptr newType = Helper::mergeTypes(hints.cast(), lastType(), topContext()); kDebug() << "Resulting type: " << newType->toString(); dec->setType(newType); dec->setKind(KDevelop::Declaration::Instance); // everything is an object in python } else { kDebug() << "Existing declarations are not empty. count: " << existingDeclarations.count(); dec = existingDeclarations.last(); AbstractType::Ptr currentType = dec->abstractType(); AbstractType::Ptr newType = lastType(); if ( newType ) { if ( currentType && !currentType->equals(newType.unsafeData()) ) { IntegralType::Ptr integral = IntegralType::Ptr::dynamicCast(currentType); if ( integral && integral->dataType() == IntegralType::TypeMixed ) { dec->setType(newType); } else { dec->setType(Helper::mergeTypes(currentType, newType, topContext())); } } else { kDebug() << "Existing declaration with no type from last declaration."; dec->setType(lastType()); } } else { kDebug() << "Existing declaration with no type."; } } T* result = dynamic_cast(dec); if ( ! result ) kWarning() << "variable declaration does not have the expected type"; return result; } void DeclarationBuilder::visitCode(CodeAst* node) { Q_ASSERT(currentlyParsedDocument().toUrl().isValid()); m_hasUnresolvedImports = false; DeclarationBuilderBase::visitCode(node); } void DeclarationBuilder::visitExceptionHandler(ExceptionHandlerAst* node) { if ( dynamic_cast(node->name) ) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->type); openType(v.lastType()); setLastType(v.lastType()); visitVariableDeclaration(node->name); // except Error as closeType(); } DeclarationBuilderBase::visitExceptionHandler(node); } void DeclarationBuilder::visitWith(WithAst* node) { if ( node->optionalVars ) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->contextExpression); setLastType(v.lastType()); visitVariableDeclaration(node->optionalVars); } Python::ContextBuilder::visitWith(node); } void DeclarationBuilder::visitFor(ForAst* node) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->iterator); VariableLengthContainer::Ptr type = v.lastType().cast(); if ( node->target->astType == Ast::NameAstType ) { if ( type && type->contentType() ) { setLastType(type->contentType().abstractType()); } else { setLastType(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); } visitVariableDeclaration(node->target); } else if ( node->target->astType == Ast::TupleAstType ) { short atElement = 0; foreach ( ExpressionAst* tupleMember, dynamic_cast(node->target)->elements ) { if ( tupleMember->astType == Ast::NameAstType ) { if ( atElement == 0 && type && type->keyType() ) { setLastType(type->keyType().abstractType()); } else if ( atElement == 1 && type && type->contentType() ) { setLastType(type->contentType().abstractType()); } else { setLastType(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); } visitVariableDeclaration(tupleMember); } ++atElement; } } Python::ContextBuilder::visitFor(node); } Declaration* DeclarationBuilder::findDeclarationInContext(QStringList dottedNameIdentifier, TopDUContext* ctx) const { DUChainReadLocker lock(DUChain::lock()); DUContext* currentContext = ctx; Q_ASSERT(currentContext); Declaration* lastAccessedDeclaration = 0; foreach ( QString currentIdentifier, dottedNameIdentifier ) { QList declarations = currentContext->findDeclarations(QualifiedIdentifier(currentIdentifier).first(), CursorInRevision::invalid(), 0, DUContext::DontSearchInParent); // break if the list of identifiers is not yet totally worked through and no declaration with an internal context was found if ( declarations.isEmpty() || ( ! declarations.first()->internalContext() && currentIdentifier != dottedNameIdentifier.last() ) ) { kWarning() << "Declaration not found: " << dottedNameIdentifier << "in top context" << ctx->url().toUrl().path(); return 0; } else { lastAccessedDeclaration = declarations.first(); currentContext = lastAccessedDeclaration->internalContext(); } } return lastAccessedDeclaration; } void DeclarationBuilder::visitImportFrom(ImportFromAst* node) { Python::AstDefaultVisitor::visitImportFrom(node); QString moduleName; foreach ( AliasAst* name, node->names ) { if ( node->module ) { moduleName = node->module->value + "." + name->name->value; } else { moduleName = name->name->value; } Identifier* declarationIdentifier = name->asName ? name->asName : name->name; createModuleImportDeclaration(moduleName, declarationIdentifier); } } void DeclarationBuilder::visitImport(ImportAst* node) { Python::ContextBuilder::visitImport(node); DUChainWriteLocker lock(DUChain::lock()); foreach ( AliasAst* name, node->names ) { QString moduleName = name->name->value; // use alias if available, name otherwise Identifier* declarationIdentifier = name->asName ? name->asName : name->name; createModuleImportDeclaration(moduleName, declarationIdentifier); } } Declaration* DeclarationBuilder::createModuleImportDeclaration(QString dottedName, Identifier* declarationIdentifier, Ast* rangeNode) { QPair moduleInfo = findModulePath(dottedName); kDebug() << dottedName; RangeInRevision range(RangeInRevision::invalid()); if ( rangeNode ) { range = rangeForNode(rangeNode, false); } else { range = rangeForNode(declarationIdentifier, true); } Q_ASSERT(range.isValid()); kDebug() << "Found module path [path/path in file]: " << moduleInfo; kDebug() << "Declaration identifier:" << declarationIdentifier->value; DUChainWriteLocker lock(DUChain::lock()); ReferencedTopDUContext moduleContext = DUChain::self()->chainForDocument(IndexedString(moduleInfo.first)); lock.unlock(); Declaration* resultingDeclaration = 0; if ( ! moduleInfo.first.isValid() ) { kWarning() << "invalid or non-existent URL:" << moduleInfo; KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Warning); p->setDescription(i18n("Module \"%1\" not found", dottedName)); { DUChainWriteLocker wlock(DUChain::lock()); ProblemPointer ptr(p); topContext()->addProblem(ptr); } return 0; } if ( ! moduleContext ) { // schedule the include file for parsing, and schedule the current one for reparsing after that is done - qDebug() << "No module context, recompiling"; + kDebug() << "No module context, recompiling"; m_hasUnresolvedImports = true; DUChain::self()->updateContextForUrl(IndexedString(moduleInfo.first), TopDUContext::AllDeclarationsContextsAndUses, 0, -20); return 0; } if ( moduleInfo.second.isEmpty() ) { // import the whole module kDebug() << "Got module, importing it"; StructureType::Ptr moduleType(new StructureType()); openType(moduleType); DUChainWriteLocker lock(DUChain::lock()); setLastType(AbstractType::Ptr(0)); resultingDeclaration = visitVariableDeclaration(declarationIdentifier, range); Q_ASSERT(resultingDeclaration); closeType(); // create a wrapper context, import the context for the file into that context, and add it using a declaration DUContext* wrapperContext = openContext(declarationIdentifier, KDevelop::DUContext::Other); closeContext(); wrapperContext->addImportedParentContext(moduleContext); moduleType->setDeclaration(resultingDeclaration); resultingDeclaration->setType(moduleType); if ( m_builtinFunctionsContext ) wrapperContext->removeImportedParentContext(m_builtinFunctionsContext.data()); resultingDeclaration->setInternalContext(wrapperContext); } else { // import a specific declaration from the given file lock.lock(); if ( declarationIdentifier->value == "*" ) { kDebug() << "Importing * from module"; currentContext()->addImportedParentContext(moduleContext); } else { kDebug() << "Got module, importing declaration: " << moduleInfo.second; Declaration* originalDeclaration = findDeclarationInContext(moduleInfo.second, moduleContext); kDebug() << "Result: " << originalDeclaration; if ( originalDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); resultingDeclaration = visitVariableDeclaration(declarationIdentifier, range); if ( dynamic_cast(resultingDeclaration) ) { static_cast(resultingDeclaration)->setAliasedDeclaration(originalDeclaration); kDebug() << "Resulting alias: " << resultingDeclaration->toString(); } else kWarning() << "import declaration is being overwritten!"; } else { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), range.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Warning); p->setDescription(i18n("Declaration for \"%1\" not found in specified module", moduleInfo.second.join("."))); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } } return resultingDeclaration; } void DeclarationBuilder::visitYield(YieldAst* node) { kDebug() << "visiting yield statement"; ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->value); setLastType(v.lastType()); if ( node->value ) { if ( TypePtr t = currentType() ) { AbstractType::Ptr encountered = v.lastType(); if ( VariableLengthContainer::Ptr previous = t->returnType().cast() ) { previous->addContentType(encountered); t->setReturnType(previous.cast()); } else { VariableLengthContainer::Ptr container = ExpressionVisitor::typeObjectForIntegralType("list", currentContext()); if ( container ) { openType(container); container->addContentType(encountered); t->setReturnType(Helper::mergeTypes(t->returnType(), container.cast())); closeType(); } } } } setLastType(AbstractType::Ptr(0)); } void DeclarationBuilder::visitCall(CallAst* node) { Python::AstDefaultVisitor::visitCall(node); KDEBUG_BLOCK kDebug() << "Visiting call"; ExpressionVisitor functionVisitor(currentContext(), editor()); functionVisitor.visitNode(node); kDebug() << functionVisitor.lastFunctionDeclaration(); if ( node->function && node->function->astType == Ast::AttributeAstType && functionVisitor.lastFunctionDeclaration() ) { kDebug() << "Checking for list content updates..."; ExpressionVisitor v(currentContext(), editor()); v.visitNode(static_cast(node->function)->value); if ( VariableLengthContainer::Ptr container = v.lastType().cast() ) { if ( v.lastDeclaration() ) { // /// DEBUG // kDebug() << "Got container type for eventual update: " << container->toString(); // kDebug() << "Eventual function declaration: " << functionVisitor.lastFunctionDeclaration()->toString(); // kDebug() << functionVisitor.lastFunctionDeclaration()->isFunctionDeclaration(); // /// END DEBUG if ( functionVisitor.lastFunctionDeclaration()->isFunctionDeclaration() ) { FunctionDeclaration* f = static_cast(functionVisitor.lastFunctionDeclaration().data()); if ( const Decorator* d = Helper::findDecoratorByName(f, "addsTypeOfArg") ) { register const int offset = d->additionalInformation().str().toInt(); if ( node->arguments.length() > offset ) { ExpressionVisitor argVisitor(currentContext(), editor()); argVisitor.visitNode(node->arguments.at(offset)); if ( argVisitor.lastType() ) { DUChainWriteLocker lock(DUChain::lock()); kDebug() << "Adding content type: " << argVisitor.lastType()->toString(); container->addContentType(argVisitor.lastType()); v.lastDeclaration()->setType(container); } } } if ( const Decorator* d = Helper::findDecoratorByName(f, "addsTypeOfArgContent") ) { register const int offset = d->additionalInformation().str().toInt(); if ( node->arguments.length() > offset ) { ExpressionVisitor argVisitor(currentContext(), editor()); argVisitor.visitNode(node->arguments.at(offset)); DUChainWriteLocker lock(DUChain::lock()); if ( argVisitor.lastType() ) { if ( VariableLengthContainer::Ptr sourceContainer = argVisitor.lastType().cast() ) { if ( AbstractType::Ptr contentType = sourceContainer->contentType().abstractType() ) { kDebug() << "Adding content type: " << contentType->toString(); container->addContentType(contentType); v.lastDeclaration()->setType(container); } } else if ( argVisitor.lastType()->whichType() == AbstractType::TypeUnsure ) { UnsureType::Ptr sourceUnsure = argVisitor.lastType().cast(); FOREACH_FUNCTION ( const IndexedType& type, sourceUnsure->types ) { if ( AbstractType::Ptr p = type.abstractType() ) { if ( VariableLengthContainer::Ptr sourceContainer = p.cast() ) { if ( AbstractType::Ptr contentType = sourceContainer->contentType().abstractType() ) { kDebug() << "Adding content type: " << contentType->toString(); container->addContentType(contentType); v.lastDeclaration()->setType(container); } } } } v.lastDeclaration()->setType(container); } } } } } } } } if ( ! m_prebuilding ) { return; } kDebug() << "--"; kDebug() << "Trying to update function argument types based on call"; bool isConstructor = false; FunctionDeclarationPointer lastFunctionDeclaration = functionVisitor.lastFunctionDeclaration(); DUChainWriteLocker lock(DUChain::lock()); if ( ! lastFunctionDeclaration && functionVisitor.lastClassDeclaration() ) { kDebug() << "No function declaration, looking for class constructor"; DUChainPointer eventualClassDeclaration = functionVisitor.lastClassDeclaration(); kDebug() << "Class declaration: " << eventualClassDeclaration; if ( eventualClassDeclaration && eventualClassDeclaration->internalContext() ) { kDebug() << "ok, looking for constructor"; QList constructors = eventualClassDeclaration->internalContext()->findDeclarations(KDevelop::Identifier("__init__")); kDebug() << "Found constructors: " << constructors; if ( ! constructors.isEmpty() ) { lastFunctionDeclaration = dynamic_cast(constructors.first()); isConstructor = true; kDebug() << "new function declaration: " << lastFunctionDeclaration; } } } if ( lastFunctionDeclaration ) { kDebug() << "got declaration:" << lastFunctionDeclaration->toString(); if ( lastFunctionDeclaration->topContext()->url() == IndexedString(Helper::getDocumentationFile()) ) { kDebug() << "in documentation file, not modifying args"; return; } kDebug() << "... and yep, it's a function declaration"; DUContext* args = DUChainUtils::getArgumentContext(lastFunctionDeclaration.data()); FunctionType::Ptr functiontype = lastFunctionDeclaration->type(); if ( args && functiontype ) { kDebug() << "got arguments"; QVector parameters = args->localDeclarations(); if ( ( lastFunctionDeclaration->context()->type() == DUContext::Class || isConstructor ) && ! parameters.isEmpty() ) { parameters.remove(0); } int atParam = 0; if ( parameters.size() >= node->arguments.size() && functiontype->arguments().length() + lastFunctionDeclaration->defaultParametersSize() >= (uint) node->arguments.size() ) { kDebug() << "... and they match the parameter size"; foreach ( ExpressionAst* arg, node->arguments ) { if ( atParam >= functiontype->arguments().size() || atParam >= parameters.size() ) { break; } ExpressionVisitor argumentVisitor(currentContext(), editor()); argumentVisitor.visitNode(arg); kDebug() << "Got type for function argument: " << argumentVisitor.lastType(); if ( argumentVisitor.lastType() && Helper::isUsefulType(argumentVisitor.lastType().cast()) ) { kDebug() << "last type: " << argumentVisitor.lastType()->toString(); HintedType::Ptr addType = HintedType::Ptr(new HintedType()); openType(addType); addType->setType(argumentVisitor.lastType()); addType->setCreatedBy(topContext(), m_futureModificationRevision); closeType(); AbstractType::Ptr newType = Helper::mergeTypes(parameters.at(atParam)->abstractType(), addType.cast(), topContext()); kDebug() << "new type: " << newType->toString(); functiontype->removeArgument(atParam); functiontype->addArgument(newType, atParam); lastFunctionDeclaration->setAbstractType(functiontype.cast()); parameters.at(atParam)->setType(newType); } atParam++; } } else { kWarning() << "Arguments size mismatch, not updating type" << parameters << node->arguments; } } } else kDebug() << "No declaration for function, not setting arg types"; } void DeclarationBuilder::visitAssignment(AssignmentAst* node) { KDEBUG_BLOCK kDebug(); AstDefaultVisitor::visitAssignment(node); QList realTargets; QList realValues; QList realDeclarations; foreach ( ExpressionAst* target, node->targets ) { if ( target->astType == Ast::TupleAstType ) { TupleAst* tuple = static_cast(target); foreach ( ExpressionAst* ast, tuple->elements ) { realTargets << ast; } } else { realTargets << target; } } if ( node->value && node->value->astType == Ast::TupleAstType ) { foreach ( ExpressionAst* value, static_cast(node->value)->elements ) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(value); realValues << v.lastType(); realDeclarations << v.lastDeclaration(); } } else { ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->value); realValues << v.lastType(); realDeclarations << v.lastDeclaration(); if ( node->value && node->value->astType == Ast::CallAstType && ! node->targets.isEmpty() ) { if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeIntegral && v.lastType().cast()->dataType() == (uint) IntegralType::TypeVoid ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), simpleRangeForNode(node->targets.at(0), true))); p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Hint); p->setDescription(i18n("Assignment to call returning nothing")); ProblemPointer ptr(p); kDebug() << "Not adding return hint, documentation data is not good enough yet"; // topContext()->addProblem(ptr); } } DUChainReadLocker lock(DUChain::lock()); kDebug() << ( v.lastType() ? v.lastType()->toString() : "< no last type >" ) << ( v.lastDeclaration() ? v.lastDeclaration()->toString() : "< no last declaration >" ); } AbstractType::Ptr tupleElementType(0); DeclarationPointer tupleElementDeclaration(0); bool canUnpack = realTargets.length() == realValues.length(); int i = 0; foreach ( ExpressionAst* target, realTargets ) { if ( canUnpack ) { tupleElementType = realValues.at(i); tupleElementDeclaration = realDeclarations.at(i); } else if (realTargets.length() == 1) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->value); tupleElementType = v.lastType(); } else { tupleElementType = AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed)); tupleElementDeclaration = 0; } /** DEBUG **/ if ( tupleElementType ) { VariableLengthContainer* d = dynamic_cast(tupleElementType.unsafeData()); if ( d ) { DUChainReadLocker lock(DUChain::lock()); kDebug() << "Got container type for declaration creation: " << tupleElementType << d->contentType(); if ( d->contentType() ) { kDebug() << "Content type: " << d->contentType().abstractType()->toString(); } } } /** END DEBUG **/ i += 1; setLastType(tupleElementType); // TODO fix this for x, y = a, b, i.e. if node->value->astType == TupleAstType if ( target->astType == Ast::NameAstType ) { if ( tupleElementType && tupleElementType->whichType() == AbstractType::TypeFunction ) { if ( tupleElementDeclaration ) { DUChainWriteLocker lock(DUChain::lock()); AliasDeclaration* decl = openDeclaration(static_cast(target)->identifier, target); decl->setAliasedDeclaration(tupleElementDeclaration.data()); } } else { DUChainWriteLocker lock(DUChain::lock()); Declaration* dec = visitVariableDeclaration(target); /** DEBUG **/ if ( tupleElementType && dec ) { VariableLengthContainer* type = dynamic_cast(dec->abstractType().unsafeData()); kDebug() << "type is: " << dec->abstractType().unsafeData() << type << dynamic_cast(tupleElementType.unsafeData()); kDebug() << "Container: " << dynamic_cast(dec->abstractType().unsafeData()); Q_ASSERT(dec->abstractType()); } /** END DEBUG **/ } } if ( target->astType == Ast::AttributeAstType ) { AttributeAst* attrib = static_cast(target); kDebug() << "Visiting attribute: " << attrib->attribute->value; // check whether the current attribute is undeclared, but the previos ones known // like in X.Y.Z = 3 where X and Y are defined, but Z isn't; then declare Z. ExpressionVisitor checkForUnknownAttribute(currentContext(), editor()); checkForUnknownAttribute.visitNode(attrib); DeclarationPointer unknown = checkForUnknownAttribute.lastDeclaration(); // declare the attribute. // however, if there's an earlier declaration which does not match the current position // (so it's really a different declaration) we skip this. Declaration* haveDeclaration = 0; if ( unknown ) { kDebug() << "Declaration is already created"; haveDeclaration = unknown.data(); } ExpressionVisitor checkPreviousAttributes(currentContext(), editor()); checkPreviousAttributes.visitNode(attrib->value); // go "down one level", so only visit "X.Y" DUContextPointer internal(0); DeclarationPointer parentObjectDeclaration = checkPreviousAttributes.lastDeclaration(); AbstractType::Ptr type = checkPreviousAttributes.lastType(); if ( ! parentObjectDeclaration ) { kWarning() << "No declaration for attribute base, aborting creation of attribute"; continue; } // if foo is a class, this is like foo.bar = 3 if ( parentObjectDeclaration->internalContext() ) { kDebug() << "Accessing class type directly"; internal = parentObjectDeclaration->internalContext(); } // while this is like A = foo(); A.bar = 3 else { kDebug() << "Accessing class type through an instance, searching original declaration of type..."; type = parentObjectDeclaration->abstractType(); StructureType::Ptr structure(dynamic_cast(type.unsafeData())); if ( ! structure || ! structure->declaration(topContext()) ) continue; parentObjectDeclaration = structure->declaration(topContext()); internal = parentObjectDeclaration->internalContext(); kDebug() << "... ok!"; } if ( ! internal ) { kWarning() << "No internal context for structure type, aborting creation of attribute declaration"; continue; } kDebug() << "Fine, got an internal context."; DUContext* previousContext = currentContext(); bool isAlreadyOpen = contextAlreayOpen(internal); if ( isAlreadyOpen ) { activateAlreadyOpenedContext(internal); visitVariableDeclaration(attrib->attribute, target, haveDeclaration); closeAlreadyOpenedContext(internal); } else { injectContext(internal.data()); Declaration* dec = visitVariableDeclaration(attrib->attribute, target, haveDeclaration); if ( dec ) { dec->setRange(RangeInRevision(internal->range().start, internal->range().start)); dec->setAutoDeclaration(true); DUChainWriteLocker lock(DUChain::lock()); previousContext->createUse(dec->ownIndex(), editorFindRange(attrib, attrib)); lock.unlock(); } else kWarning() << "No declaration created for " << attrib->attribute << "as parent is not a class"; closeInjectedContext(); } } } } void DeclarationBuilder::visitClassDefinition( ClassDefinitionAst* node ) { kDebug() << "opening class definition"; DUChainWriteLocker lock(DUChain::lock()); ClassDeclaration* dec = openDeclaration( node->name, node ); visitDecorators(node->decorators, dec); eventuallyAssignInternalContext(); dec->setKind(KDevelop::Declaration::Type); dec->clearBaseClasses(); dec->setClassType(ClassDeclarationData::Class); foreach ( ExpressionAst* c, node->baseClasses ) { ExpressionVisitor v(currentContext(), editor()); v.visitNode(c); if ( v.lastType() && v.lastType()->whichType() == AbstractType::TypeStructure ) { StructureType::Ptr baseClassType = v.lastType().cast(); BaseClassInstance base; base.baseClass = baseClassType->indexed(); base.access = KDevelop::Declaration::Public; dec->addBaseClass(base); } } // check whether this is a type container (list, dict, ...) or just a "normal" class StructureType::Ptr type(0); const Decorator* d = Helper::findDecoratorByName(dec, "TypeContainer"); if ( d ) { type = StructureType::Ptr(new VariableLengthContainer()); } if ( ! type ) { type = StructureType::Ptr(new StructureType()); } type->setDeclaration(dec); dec->setType(type); lock.unlock(); openType(type); // needs to be done here, so the assignment of the internal context happens before visiting the body openContextForClassDefinition(node); lock.lock(); dec->setInternalContext(currentContext()); lock.unlock(); // yes, we do not call the context builder here, because contexts are already open AstDefaultVisitor::visitClassDefinition( node ); lock.lock(); kDebug() << " --- closing CLASS context: " << currentContext()->range().castToSimpleRange(); closeContext(); lock.unlock(); closeType(); closeDeclaration(); dec->setComment(getDocstring(node->body)); } template void DeclarationBuilder::visitDecorators(QList< Python::ExpressionAst* > decorators, T* addTo) { foreach ( ExpressionAst* decorator, decorators ) { kDebug() << "decorator type: " << decorator->astType << "(name: " << Ast::NameAstType << ", call: " << Ast::CallAstType << ")"; if ( decorator->astType == Ast::CallAstType ) { CallAst* call = static_cast(decorator); Decorator d; if ( call->function->astType != Ast::NameAstType ) { continue; } d.setName(*static_cast(call->function)->identifier); foreach ( ExpressionAst* arg, call->arguments ) { if ( arg->astType == Ast::NumberAstType ) { d.setAdditionalInformation(static_cast(arg)->value); } else if ( arg->astType == Ast::StringAstType ) { d.setAdditionalInformation(static_cast(arg)->value); } break; // we only need the first argument for documentation analysis } kDebug() << "call decorator identifier: " << d.name() << *static_cast(call->function)->identifier; addTo->addDecorator(d); } else if ( decorator->astType == Ast::NameAstType ) { NameAst* name = static_cast(decorator); Decorator d; d.setName(*(name->identifier)); kDebug() << "name decorator identifier: " << d.name() << *(name->identifier); addTo->addDecorator(d); } } } void DeclarationBuilder::visitListComprehension(ListComprehensionAst* node) { if ( ! node->generators.isEmpty() ) { DUChainWriteLocker lock(DUChain::lock()); RangeInRevision range = editorFindRange(node->element, node->generators.last()->iterator); openContext(node, range, KDevelop::DUContext::Other); currentContext()->setPropagateDeclarations(false); kDebug() << "Opening context for list comprehension, with range" << range.castToSimpleRange(); foreach ( ComprehensionAst* comprehension, node->generators ) { if ( comprehension->target->astType == Ast::NameAstType ) { // TODO this is disabled because it doesn't work. kWarning() << "implement me: list generator"; // visitVariableDeclaration(static_cast(comprehension->target)->identifier); } else kDebug() << "List comprehension with non-name AST target, skipping"; } closeContext(); kDebug() << "Closing context for list comprehension"; } else kDebug() << "comprehension with empty generators list, skipping"; DeclarationBuilderBase::visitListComprehension(node); } void DeclarationBuilder::visitFunctionDefinition( FunctionDefinitionAst* node ) { kDebug() << "opening function definition" << node->startLine << node->endLine; DeclarationPointer eventualParentDeclaration(currentDeclaration()); // an eventual containing class declaration FunctionType::Ptr type; QList existing; FunctionDeclaration* dec = openDeclaration( node->name, node ); Q_ASSERT(dec->isFunctionDeclaration()); type = FunctionType::Ptr(new FunctionType()); { DUChainWriteLocker lock; dec->setType(type); } openType(type); kDebug() << " <<< open function type"; DUChainWriteLocker lock(DUChain::lock()); bool hasFirstArgument = false; visitDecorators(node->decorators, dec); visitFunctionArguments(node); // this must be done here, because the type of self must be known when parsing the body kDebug() << "Checking whether we have to change argument types..."; kDebug() << eventualParentDeclaration.data() << currentType()->arguments().length() << m_firstAttributeDeclaration.data() << currentContext()->type() << DUContext::Class; if ( eventualParentDeclaration.data() && currentType()->arguments().length() && m_firstAttributeDeclaration.data() && currentContext()->type() == DUContext::Class ) { kDebug() << "Changing self argument type"; kDebug() << "Arguments left: " << currentType()->arguments().count(); currentType()->removeArgument(0); kDebug() << "Arguments left: " << currentType()->arguments().count(); DUChainWriteLocker lock(DUChain::lock()); m_firstAttributeDeclaration->setAbstractType(eventualParentDeclaration->abstractType()); hasFirstArgument = true; } visitFunctionBody(node); closeDeclaration(); eventuallyAssignInternalContext(); kDebug() << " >>> close function type"; closeType(); // python methods don't have their parents attributes directly inside them if ( eventualParentDeclaration && eventualParentDeclaration->internalContext() && dec->internalContext() ) { dec->internalContext()->removeImportedParentContext(eventualParentDeclaration->internalContext()); } { DUChainWriteLocker lock(DUChain::lock()); if ( ! type->returnType() ) { type->setReturnType(AbstractType::Ptr(new IntegralType(IntegralType::TypeVoid))); } dec->setType(type); } DUContext* args = DUChainUtils::getArgumentContext(dec); if ( args ) { QVector parameters = args->localDeclarations(); if ( currentContext()->type() == DUContext::Class && ! parameters.isEmpty() ) { if ( parameters[0]->identifier().identifier() != IndexedString("self") ) { kDebug() << "argument is not called self, but instead:" << parameters[0]->identifier().identifier().str(); KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), SimpleRange(node->startLine, node->startCol, node->startLine, 10000))); p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Warning); p->setDescription(i18n("First argument of class method is not called self, this is deprecated")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } m_firstAttributeDeclaration = DeclarationPointer(0); } else if ( currentContext()->type() == DUContext::Class && parameters.isEmpty() ) { DUChainWriteLocker lock(DUChain::lock()); KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), SimpleRange(node->startLine, node->startCol, node->startLine, 10000))); // only mark first line p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Warning); p->setDescription(i18n("Non-static class method without arguments, must have at least one (self)")); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } // check for documentation dec->setComment(getDocstring(node->body)); } QString DeclarationBuilder::getDocstring(QList< Ast* > body) { if ( body.length() && body.first()->astType == Ast::ExpressionAstType && static_cast(body.first())->value->astType == Ast::StringAstType ) { StringAst* docstring = static_cast(static_cast(body.first())->value); kDebug() << "Got docstring for declaration"; return docstring->value; } return QString(""); } void DeclarationBuilder::visitReturn(ReturnAst* node) { kDebug() << "visiting return statement"; ExpressionVisitor v(currentContext(), editor()); v.visitNode(node->value); setLastType(v.lastType()); if ( node->value ) { if ( ! hasCurrentType() ) { DUChainWriteLocker lock(DUChain::lock()); KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), SimpleRange(node->startLine, node->startCol, node->endLine, node->endCol))); // only mark first line p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setDescription(i18n("Return statement not within function declaration")); ProblemPointer ptr(p); topContext()->addProblem(ptr); return; } TypePtr t = currentType(); AbstractType::Ptr encountered = v.lastType(); // kDebug() << "Found type: " << encountered->toString(); t->setReturnType(Helper::mergeTypes(t->returnType(), encountered)); } setLastType(AbstractType::Ptr(0)); } void DeclarationBuilder::visitArguments( ArgumentsAst* node ) { DUChainWriteLocker lock(DUChain::lock()); AbstractFunctionDeclaration* function = dynamic_cast(currentDeclaration()); kDebug() << "Current context for parameters: " << currentContext(); kDebug() << currentContext()->scopeIdentifier().toString(); if ( currentDeclaration() ) kDebug() << currentDeclaration()->identifier().toString(); if ( function ) { static_cast(currentDeclaration())->clearDefaultParameters(); if ( hasCurrentType() and currentType() ) { FunctionType::Ptr type = currentType(); NameAst* realParam = 0; bool isFirst = true; int defaultParametersCount = node->defaultValues.length(); int parametersCount = node->arguments.length(); int firstDefaultParameterOffset = parametersCount - defaultParametersCount; int currentIndex = 0; kDebug() << "variable argument ranges: " << node->arg_lineno << node->arg_col_offset << node->vararg_lineno << node->vararg_col_offset; foreach ( ExpressionAst* expression, node->arguments ) { currentIndex += 1; realParam = dynamic_cast(expression); if ( ! realParam || realParam->context != ExpressionAst::Parameter ) continue; setLastType(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); Declaration* paramDeclaration = visitVariableDeclaration(realParam); if ( type && paramDeclaration && currentIndex > firstDefaultParameterOffset ) { kDebug() << "Adding default argument: " << realParam->identifier->value << paramDeclaration->abstractType(); // find type of given default value ExpressionVisitor v(currentContext()); v.visitNode(node->defaultValues.at(currentIndex - firstDefaultParameterOffset - 1)); if ( v.lastType() ) { type->addArgument(v.lastType()); } else { type->addArgument(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); } static_cast(currentDeclaration())->addDefaultParameter(paramDeclaration->identifier().identifier()); kDebug() << "Arguments count: " << type->arguments().length(); } else { kDebug() << "Not a default argument: " << realParam->identifier->value; type->addArgument(AbstractType::Ptr(new IntegralType(IntegralType::TypeMixed))); } if ( isFirst ) { m_firstAttributeDeclaration = DeclarationPointer(paramDeclaration); isFirst = false; } } if ( node->vararg ) { AbstractType::Ptr listType = ExpressionVisitor::typeObjectForIntegralType("list", currentContext()); type->addArgument(listType); node->vararg->startCol = node->vararg_col_offset; node->vararg->endCol = node->vararg_col_offset + node->vararg->value.length() - 1; node->vararg->startLine = node->vararg_lineno - 1; node->vararg->endLine = node->vararg_lineno - 1; Declaration* d = visitVariableDeclaration(node->vararg); Q_ASSERT(d); d->setAbstractType(listType); } if ( node->kwarg ) { AbstractType::Ptr dictType = ExpressionVisitor::typeObjectForIntegralType("dict", currentContext()); type->addArgument(dictType); node->kwarg->startCol = node->arg_col_offset; node->kwarg->endCol = node->arg_col_offset + node->kwarg->value.length() - 1; node->kwarg->startLine = node->arg_lineno - 1; node->kwarg->endLine = node->arg_lineno - 1; Declaration* d = visitVariableDeclaration(node->kwarg); Q_ASSERT(d); d->setAbstractType(dictType); } } } DeclarationBuilderBase::visitArguments(node); } } diff --git a/duchain/usebuilder.cpp b/duchain/usebuilder.cpp index c736954b..3dc674b4 100644 --- a/duchain/usebuilder.cpp +++ b/duchain/usebuilder.cpp @@ -1,152 +1,152 @@ /***************************************************************************** * Copyright (c) 2007 Piyush verma * * Copyright 2010-2011 Sven Brauch * * * * Permission is hereby granted, free of charge, to any person obtaining * * a copy of this software and associated documentation files (the * * "Software"), to deal in the Software without restriction, including * * without limitation the rights to use, copy, modify, merge, publish, * * distribute, sublicense, and/or sell copies of the Software, and to * * permit persons to whom the Software is furnished to do so, subject to * * the following conditions: * * * * The above copyright notice and this permission notice shall be * * included in all copies or substantial portions of the Software. * * * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * *****************************************************************************/ #include "usebuilder.h" #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "pythoneditorintegrator.h" #include "ast.h" #include "expressionvisitor.h" #include "helpers.h" using namespace KTextEditor; using namespace KDevelop; namespace Python { UseBuilder::UseBuilder (PythonEditorIntegrator* editor) : UseBuilderBase(), m_errorReportingEnabled(true) { setEditor(editor); } void UseBuilder::visitName(NameAst* node) { Declaration* declaration = Helper::declarationForName(node, identifierForNode(node->identifier), editorFindRange(node, node), DUContextPointer(currentContext())); QStringList keywords; - keywords << "None" << "True" << "False" << "__name__"; + keywords << "None" << "True" << "False" << "__name__" << "print"; Q_ASSERT(node->identifier); Q_ASSERT(node->hasUsefulRangeInformation); // TODO remove this! RangeInRevision useRange = rangeForNode(node->identifier, true); if ( declaration && declaration->range() == useRange ) return; if ( ! declaration && ! keywords.contains(node->identifier->value) && m_errorReportingEnabled ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Hint); p->setDescription(i18n("Undefined variable: %1", node->identifier->value)); { DUChainWriteLocker wlock(DUChain::lock()); ProblemPointer ptr(p); topContext()->addProblem(ptr); } } /// debug kDebug() << " Registering use for " << node->identifier->value << " at " << useRange.castToSimpleRange() << "with dec" << declaration; { DUChainReadLocker lock(DUChain::lock()); Q_ASSERT( ! declaration || declaration->alwaysForceDirect() ); } /// end debug UseBuilderBase::newUse(node, useRange, DeclarationPointer(declaration)); // kDebug() << "USE FOUND:" << topContext()->findUseAt(useRange.start) << "for declaration" << declaration->toString(); } void UseBuilder::visitListComprehension(ListComprehensionAst* node) { // TODO fix this properly // due to duchain limitations, we currently cannot declare the "x" // in "[x for x in range(3)]" properly. Thus, it's always reported as an error; // we at least avoid this here, so it's displayed in plain black with no // language support whatsoever. disableErrorReporting(); AstDefaultVisitor::visitListComprehension(node); enableErrorReporting(); } void UseBuilder::visitDictionaryComprehension(DictionaryComprehensionAst* node) { disableErrorReporting(); AstDefaultVisitor::visitDictionaryComprehension(node); enableErrorReporting(); } void UseBuilder::visitGeneratorExpression(GeneratorExpressionAst* node) { disableErrorReporting(); Python::AstDefaultVisitor::visitGeneratorExpression(node); enableErrorReporting(); } void UseBuilder::visitAttribute(AttributeAst* node) { ExpressionVisitor v(currentContext(), editor()); kDebug() << "VisitAttribute start"; UseBuilderBase::visitAttribute(node); kDebug() << "Visit Attribute base end"; v.visitNode(node); RangeInRevision useRange(node->attribute->startLine, node->attribute->startCol, node->attribute->endLine, node->attribute->endCol + 1); DUChainWriteLocker lock(DUChain::lock()); DeclarationPointer declaration = v.lastDeclaration(); if ( declaration && declaration->range() == useRange ) return; if ( ! declaration && v.shouldBeKnown() ) { KDevelop::Problem *p = new KDevelop::Problem(); p->setFinalLocation(DocumentRange(currentlyParsedDocument(), useRange.castToSimpleRange())); // TODO ok? p->setSource(KDevelop::ProblemData::SemanticAnalysis); p->setSeverity(KDevelop::ProblemData::Hint); p->setDescription(i18n("Attribute \"%1\" not found on accessed object", node->attribute->value)); ProblemPointer ptr(p); topContext()->addProblem(ptr); } UseBuilderBase::newUse(node, useRange, declaration); // currentContext()->findUseAt(); } ParseSession *UseBuilder::parseSession() const { return m_session; } } // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on diff --git a/parser/ast.cpp b/parser/ast.cpp index 121ede39..bd2c2599 100644 --- a/parser/ast.cpp +++ b/parser/ast.cpp @@ -1,327 +1,327 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "ast.h" #include "astbuilder.h" namespace Python { // We never need actual constructors for AST nodes, but it seems to be required, at least for some platforms // so we provide pseudo implementations // there's nothing happening here, don't bother reading the code -Ast::Ast( Ast* parent, Ast::AstType type ) : parent(parent), astType( type ), startCol(0), startLine(tline(-5)), endCol(0), endLine(0), hasUsefulRangeInformation(false), context(0) { } +Ast::Ast( Ast* parent, Ast::AstType type ) : parent(parent), astType( type ), startCol(0), startLine(-99999), endCol(0), endLine(0), hasUsefulRangeInformation(false), context(0) { } Ast::Ast() : parent(0), startCol(0), startLine(-5), endCol(0), endLine(0), hasUsefulRangeInformation(false), context(0) { } Ast::~Ast() { } ArgumentsAst::ArgumentsAst(Ast* parent): Ast(parent, Ast::ArgumentsAstType), arg_lineno(0), arg_col_offset(0), vararg_lineno(0), vararg_col_offset(0) { } AssertionAst::AssertionAst(Ast* parent): StatementAst(parent, Ast::AssertionAstType) { } AssignmentAst::AssignmentAst(Ast* parent): StatementAst(parent, Ast::AssignmentAstType), value(0) { } AttributeAst::AttributeAst(Ast* parent): ExpressionAst(parent, Ast::AttributeAstType), value(0), depth(0) { } AugmentedAssignmentAst::AugmentedAssignmentAst(Ast* parent): StatementAst(parent, Ast::AugmentedAssignmentAstType), value(0) { } BinaryOperationAst::BinaryOperationAst(Ast* parent): ExpressionAst(parent, Ast::BinaryOperationAstType), lhs(0), rhs(0) { } BooleanOperationAst::BooleanOperationAst(Ast* parent): ExpressionAst(parent, Ast::BooleanOperationAstType) { } BreakAst::BreakAst(Ast* parent): StatementAst(parent, Ast::BreakAstType) { } CallAst::CallAst(Ast* parent): ExpressionAst(parent, Ast::CallAstType), function(0), keywordArguments(0), starArguments(0) { } ClassDefinitionAst::ClassDefinitionAst(Ast* parent): StatementAst(parent, Ast::ClassDefinitionAstType), name(0) { } CodeAst::CodeAst() : name(0) { astType = Ast::CodeAstType; } CompareAst::CompareAst(Ast* parent): ExpressionAst(parent, Ast::CompareAstType), leftmostElement(0) { } ComprehensionAst::ComprehensionAst(Ast* parent): Ast(parent, Ast::ComprehensionAstType), target(0), iterator(0) { } ContinueAst::ContinueAst(Ast* parent): StatementAst(parent, Ast::ContinueAstType) { } DeleteAst::DeleteAst(Ast* parent): StatementAst(parent, Ast::DeleteAstType) { } DictAst::DictAst(Ast* parent): ExpressionAst(parent, Ast::DictAstType) { } IndexAst::IndexAst(Ast* parent): SliceAstBase(parent, Ast::IndexAstType), value(0) { } SliceAst::SliceAst(Ast* parent): SliceAstBase(parent, Ast::SliceAstType), lower(0), upper(0), step(0) { } DictionaryComprehensionAst::DictionaryComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::DictionaryComprehensionAstType), key(0), value(0) { } EllipsisAst::EllipsisAst(Ast* parent): SliceAstBase(parent, Ast::EllipsisAstType) { } ExceptionHandlerAst::ExceptionHandlerAst(Ast* parent): Ast(parent, Ast::ExceptionHandlerAstType), type(0), name(0) { } ExecAst::ExecAst(Ast* parent): StatementAst(parent, Ast::ExecAstType), body(0), globals(0), locals(0) { } ListComprehensionAst::ListComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::ListComprehensionAstType), element(0) { } ExpressionAst::ExpressionAst(Ast* parent, AstType type): Ast(parent, type), value(0), belongsToCall(0) { } ExtendedSliceAst::ExtendedSliceAst(Ast* parent): SliceAstBase(parent, Ast::ExtendedSliceAstType) { } ForAst::ForAst(Ast* parent): StatementAst(parent, Ast::ForAstType), target(0), iterator(0) { } FunctionDefinitionAst::FunctionDefinitionAst(Ast* parent): StatementAst(parent, Ast::FunctionDefinitionAstType), name(0), arguments(0) { } GeneratorExpressionAst::GeneratorExpressionAst(Ast* parent): ExpressionAst(parent, Ast::GeneratorExpressionAstType), element(0) { } GlobalAst::GlobalAst(Ast* parent): StatementAst(parent, Ast::GlobalAstType) { } Identifier::Identifier(QString value) : value(value) { } IfAst::IfAst(Ast* parent): StatementAst(parent, Ast::IfAstType), condition(0) { } IfExpressionAst::IfExpressionAst(Ast* parent): ExpressionAst(parent, Ast::IfExpressionAstType), condition(0) { } ImportAst::ImportAst(Ast* parent): StatementAst(parent, Ast::ImportAstType) { } ImportFromAst::ImportFromAst(Ast* parent): StatementAst(parent, Ast::ImportFromAstType), module(0), level(0) { } KeywordAst::KeywordAst(Ast* parent): Ast(parent, Ast::KeywordAstType), argumentName(0), value(0) { } LambdaAst::LambdaAst(Ast* parent): ExpressionAst(parent, Ast::LambdaAstType), arguments(0) { } ListAst::ListAst(Ast* parent): ExpressionAst(parent, Ast::ListAstType) { } NameAst::NameAst(Ast* parent): ExpressionAst(parent, Ast::NameAstType), identifier(0) { } NumberAst::NumberAst(Ast* parent): ExpressionAst(parent, Ast::NumberAstType), value("0") { } PassAst::PassAst(Ast* parent): StatementAst(parent, Ast::PassAstType) { } PrintAst::PrintAst(Ast* parent): StatementAst(parent, Ast::PrintAstType), destination(0), newline(0) { } RaiseAst::RaiseAst(Ast* parent): StatementAst(parent, Ast::RaiseAstType), type(0) { } ReprAst::ReprAst(Ast* parent): ExpressionAst(parent, Ast::ReprAstType), value(0) { } ReturnAst::ReturnAst(Ast* parent): StatementAst(parent, Ast::ReturnAstType), value(0) { } SetAst::SetAst(Ast* parent): ExpressionAst(parent, Ast::SetAstType) { } SetComprehensionAst::SetComprehensionAst(Ast* parent): ExpressionAst(parent, Ast::SetComprehensionAstType), element(0) { } SliceAstBase::SliceAstBase(Ast* parent, AstType type): Ast(parent, type) { } StatementAst::StatementAst(Ast* parent, AstType type): Ast(parent, type) { } StringAst::StringAst(Ast* parent): ExpressionAst(parent, Ast::StringAstType), value("") { } SubscriptAst::SubscriptAst(Ast* parent): ExpressionAst(parent, Ast::SubscriptAstType), value(0), slice(0) { } TryExceptAst::TryExceptAst(Ast* parent): StatementAst(parent, Ast::TryExceptAstType) { } TryFinallyAst::TryFinallyAst(Ast* parent): StatementAst(parent, Ast::TryFinallyAstType) { } TupleAst::TupleAst(Ast* parent): ExpressionAst(parent, Ast::TupleAstType) { } UnaryOperationAst::UnaryOperationAst(Ast* parent): ExpressionAst(parent, Ast::UnaryOperationAstType), operand(0) { } WhileAst::WhileAst(Ast* parent): StatementAst(parent, Ast::WhileAstType), condition(0) { } WithAst::WithAst(Ast* parent): StatementAst(parent, Ast::WithAstType), contextExpression(0) { } YieldAst::YieldAst(Ast* parent): ExpressionAst(parent, Ast::YieldAstType), value(0) { } AliasAst::AliasAst(Ast* parent): Ast(parent, Ast::AliasAstType), name(0), asName(0) { } } diff --git a/parser/astbuilder.cpp b/parser/astbuilder.cpp index 5fd861f5..74c8255a 100644 --- a/parser/astbuilder.cpp +++ b/parser/astbuilder.cpp @@ -1,303 +1,316 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "astbuilder.h" #include "ast.h" #include #include #include #include #include #include #include #include #include #include #include "python_header.h" +#include +#include +#include + using namespace KDevelop; extern grammar _PyParser_Grammar; namespace Python { #include "generated.h" -#include -#include -#include QMutex AstBuilder::pyInitLock; QString AstBuilder::pyHomeDir = KStandardDirs::locate("data", ""); -QPair fileHeaderHack(const QString& contents, const KUrl& filename) +QPair fileHeaderHack(QString& contents, const KUrl& filename) { IProject* proj = ICore::self()->projectController()->findProjectForUrl(filename); // the file is not in a project, don't apply hack if ( ! proj ) { - return; + return QPair(contents, 0); } const KUrl headerFileUrl = proj->folder().path(KUrl::AddTrailingSlash) + ".kdev_python_header"; - const QFile headerFile(headerFileUrl.path()); + QFile headerFile(headerFileUrl.path()); + QString headerFileContents; if ( headerFile.exists() ) { headerFile.open(QIODevice::ReadOnly); - QString headerFileContents = headerFile.readAll(); + headerFileContents = headerFile.readAll(); headerFile.close(); kDebug() << "Found header file, applying hack"; int insertAt = 0; bool endOfCommentsReached = false; bool commentSignEncountered = false; bool atLineBeginning = true; int lastLineBeginning = 0; + int newlineCount = 0; int l = contents.length(); do { if ( insertAt >= l ) { kDebug() << "File consist only of comments, not applying hack"; - return contents; + return QPair(contents, 0); } if ( contents.at(insertAt) == '#' ) { commentSignEncountered = true; } if ( not contents.at(insertAt).isSpace() ) { atLineBeginning = false; if ( not commentSignEncountered ) { endOfCommentsReached = true; } } if ( contents.at(insertAt) == '\n' ) { atLineBeginning = true; commentSignEncountered = false; lastLineBeginning = insertAt; + newlineCount += 1; + } + if ( newlineCount == 2 ) { + endOfCommentsReached = true; } insertAt += 1; } while ( not endOfCommentsReached ); kDebug() << "Inserting contents at char" << lastLineBeginning << "of file"; contents = contents.left(lastLineBeginning) - + "\n" + headerFileContents + "\n" - + contents.right(lastLineBeginning + headerFileContents.length() + 2); + + "\n" + headerFileContents + "\n#\n" + + contents.right(contents.length() - lastLineBeginning); + kDebug() << contents; + return QPair(contents, - ( headerFileContents.count('\n') + 3 )); + } + else { + return QPair(contents, 0); } - } CodeAst* AstBuilder::parse(KUrl filename, QString& contents) { Py_NoSiteFlag = 1; - contents = fileHeaderHack(contents, filename); + QPair hacked = fileHeaderHack(contents, filename); + contents = hacked.first; + int lineOffset = hacked.second; AstBuilder::pyInitLock.lock(); Py_SetPythonHome(AstBuilder::pyHomeDir.toAscii().data()); kDebug() << "Not initialized, calling init func."; Py_Initialize(); QTimer timer; timer.start(1000); while ( ! Py_IsInitialized() && timer.isActive() ) { kWarning() << "Python doesn't say it is initialized yet, waiting -- should not happen!"; usleep(100000); } Q_ASSERT(Py_IsInitialized()); PyArena* arena = PyArena_New(); Q_ASSERT(arena); // out of memory PyCompilerFlags* flags = new PyCompilerFlags(); flags->cf_flags = 0; PyObject *exception, *value, *backtrace; PyErr_Fetch(&exception, &value, &backtrace); kDebug() << "Errors before starting parser:"; PyObject_Print(value, stderr, Py_PRINT_RAW); kDebug(); mod_ty syntaxtree = PyParser_ASTFromString(contents.toAscii(), "", file_input, flags, arena); if ( ! syntaxtree ) { kWarning() << "DID NOT RECEIVE A SYNTAX TREE -- probably parse error."; PyErr_Fetch(&exception, &value, &backtrace); kDebug() << "Error objects: " << exception << value << backtrace; PyObject_Print(value, stderr, Py_PRINT_RAW); PyObject* errorMessage_str = PyTuple_GetItem(value, 0); PyObject* errorDetails_tuple = PyTuple_GetItem(value, 1); qDebug() << "Eventual errors while extracting tuple: "; PyObject_Print(errorMessage_str, stderr, Py_PRINT_RAW); if ( ! errorDetails_tuple ) { kWarning() << "Error retrieving error message, not displaying, and not doing anything"; pyInitLock.unlock(); return 0; } PyObject* linenoobj = PyTuple_GetItem(errorDetails_tuple, 1); errorMessage_str = PyTuple_GetItem(value, 0); errorDetails_tuple = PyTuple_GetItem(value, 1); PyObject_Print(errorMessage_str, stderr, Py_PRINT_RAW); PyObject* colnoobj = PyTuple_GetItem(errorDetails_tuple, 2); int lineno = PyInt_AsLong(linenoobj) - 1; int colno = PyInt_AsLong(colnoobj); ProblemPointer p(new Problem()); - SimpleCursor start(lineno, (colno-4 > 0 ? colno-4 : 0)); - SimpleCursor end(lineno, (colno+4 > 4 ? colno+4 : 4)); + SimpleCursor start(lineno + lineOffset, (colno-4 > 0 ? colno-4 : 0)); + SimpleCursor end(lineno + lineOffset, (colno+4 > 4 ? colno+4 : 4)); SimpleRange range(start, end); kDebug() << "Problem range: " << range; DocumentRange location(IndexedString(filename.path()), range); p->setFinalLocation(location); p->setDescription(PyString_AsString(PyObject_Str(errorMessage_str))); p->setSource(ProblemData::Parser); m_problems.append(p); // try to recover. // Currently the following is tired: // * If the last non-space char before the error reported was ":", it's most likely an indent error. // The common easy-to-fix and annoying indent error is "for item in foo: ". In that case, just add "pass" after the ":" token. // * If it's not, we will just comment the line with the error, fixing problems like "foo = ". // * If both fails, everything including the first non-empty line before the one with the error will be deleted. int len = contents.length(); int currentLine = 0; QString currentLineContents; QChar c; QChar newline('\n'); int emptySince = 0; int emptySinceLine = 0; int emptyLinesSince = 0; int emptyLinesSinceLine = 0; int lastNonemptyLineBeginning = 0; unsigned short currentLineIndent = 0; bool atLineBeginning = true; QList indents; int errline = qMax(0, lineno); int currentLineBeginning = 0; for ( int i = 0; i < len; i++ ) { c = contents.at(i); if ( ! c.isSpace() ) { emptySince = i; emptySinceLine = currentLine; atLineBeginning = false; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); } else if ( c == newline ) { if ( currentLine == errline ) { atLineBeginning = false; } else { currentLine += 1; currentLineBeginning = i+1; // this line has had content, so reset the "empty lines since" counter if ( ! atLineBeginning ) { lastNonemptyLineBeginning = emptyLinesSince; emptyLinesSince = i; emptyLinesSinceLine = currentLine; } atLineBeginning = true; if ( indents.length() <= currentLine ) indents.append(currentLineIndent); currentLineIndent = 0; } } else if ( atLineBeginning ) { currentLineIndent += 1; } if ( currentLine == errline && ! atLineBeginning ) { // if the last non-empty char before the error opens a new block, it's likely an "empty block" problem // we can easily fix that by adding in a "pass" statement. However, we want to add that in the next line, if possible // so context ranges for autocompletion stay intact. if ( contents[emptySince] == QChar(':') ) { kDebug() << indents.length() << emptySinceLine + 1 << indents; if ( indents.length() > emptySinceLine + 1 && indents.at(emptySinceLine) < indents.at(emptySinceLine + 1) ) { kDebug() << indents.at(emptySinceLine) << indents.at(emptySinceLine + 1); contents.insert(emptyLinesSince + 1 + indents.at(emptyLinesSinceLine), "\tpass#"); } else { contents.insert(emptySince + 1, "\tpass#"); } } else if ( indents.length() >= currentLine && currentLine > 0 ) { kDebug() << indents << currentLine; contents[i+1+indents.at(currentLine - 1)] = QChar('#'); contents.insert(i+1+indents.at(currentLine - 1), "pass"); } break; } } syntaxtree = PyParser_ASTFromString(contents.toAscii(), "", file_input, flags, arena); // 3rd try: discard everything after the last non-empty line, but only until the next block start currentLineBeginning = qMin(contents.length() - 1, currentLineBeginning); errline = qMax(0, qMin(indents.length()-1, errline)); if ( ! syntaxtree ) { kWarning() << "Discarding parts of the code to be parsed because of previous errors"; kDebug() << indents; int indentAtError = indents.at(errline); QChar c; bool atLineBeginning = true; int currentIndent = -1; int currentLineBeginning_end = currentLineBeginning; int currentLineContentBeginning = currentLineBeginning; for ( int i = currentLineBeginning; i < len; i++ ) { c = contents.at(i); kDebug() << c; if ( c == '\n' ) { if ( currentIndent <= indentAtError && currentIndent != -1 ) { kDebug() << "Start of error code: " << currentLineBeginning; kDebug() << "End of error block (current position): " << currentLineBeginning_end; kDebug() << "Length: " << currentLineBeginning_end - currentLineBeginning; kDebug() << "indent at error <> current indent:" << indentAtError << "<>" << currentIndent; // contents.remove(currentLineBeginning, currentLineBeginning_end-currentLineBeginning); break; } contents.insert(currentLineContentBeginning - 1, "pass#"); i += 5; i = qMin(i, contents.length()); len = contents.length(); atLineBeginning = true; currentIndent = 0; currentLineBeginning_end = i + 1; currentLineContentBeginning = i + 1; continue; } if ( ! c.isSpace() && atLineBeginning ) { currentLineContentBeginning = i; atLineBeginning = false; } if ( c.isSpace() && atLineBeginning ) currentIndent += 1; } kDebug() << "This is what is left: " << contents; syntaxtree = PyParser_ASTFromString(contents.toAscii(), "", file_input, flags, arena); } if ( ! syntaxtree ) { Py_Finalize(); pyInitLock.unlock(); return 0; // everything fails, so we abort. } } kDebug() << "Got syntax tree from python parser:" << syntaxtree->kind << Module_kind; - PythonAstTransformer* t = new PythonAstTransformer(); + PythonAstTransformer* t = new PythonAstTransformer(lineOffset); t->run(syntaxtree, filename.fileName().replace(".py", "")); kDebug() << t->ast; PyArena_Free(arena); Py_Finalize(); AstBuilder::pyInitLock.unlock(); return t->ast; } } diff --git a/parser/astbuilder.h b/parser/astbuilder.h index 861b43a3..d20249f6 100644 --- a/parser/astbuilder.h +++ b/parser/astbuilder.h @@ -1,66 +1,61 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2007 Andreas Pakulat * * Copyright 2010-2011 Sven Brauch * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef ASTBUILDER_H #define ASTBUILDER_H #include "ast.h" #include #include #include "kdebug.h" #include "QXmlStreamReader" #include #include "parserexport.h" namespace PythonParser { class Parser; class AstNode; } namespace Python { class Ast; class CodeAst; typedef QMap stringDictionary; -// Shift lines by some fixed amount -inline int tline(int line) { - return line; -}; - -QString fileHeaderHack(const QString& contents, const KUrl& filename); +QPair fileHeaderHack(QString& contents, const KUrl& filename); class KDEVPYTHONPARSER_EXPORT AstBuilder { public: CodeAst* parse(KUrl filename, QString& contents); QList m_problems; private: static QMutex pyInitLock; static QString pyHomeDir; }; } #endif \ No newline at end of file diff --git a/parser/conversionGenerator.py b/parser/conversionGenerator.py index 99414dbe..ee4ffe0d 100644 --- a/parser/conversionGenerator.py +++ b/parser/conversionGenerator.py @@ -1,274 +1,284 @@ #!/usr/bin/env python # Transforms a conversion definition file (.sdef) into C++ code. To be copied over manually. :) # sdef example line: # RULE_FOR _stmt;KIND Expr_kind;ACTIONS create|ExpressionAst set|value->ExpressionAst,value;CODE;; contents = open('python26.sdef').read().replace("\n", "").split(';;') func_structure = ''' Ast* visitNode(%{RULE_FOR}* node) { if ( ! node ) return 0; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = 0; switch ( node->kind ) { %{SWITCH_LINES} default: kWarning() << "Unsupported statement AST type: " << node->kind; Q_ASSERT(false); } %{APPENDIX} NameAst* r = dynamic_cast(result); if ( r ) { r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } ''' simple_func_structure = ''' Ast* visitNode(%{RULE_FOR}* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return 0; // return a nullpointer if no node is set, that's fine, everyone else will check for that. %{SWITCH_LINES} return v; } ''' switch_line = ''' case %{KIND}: { %{ACTIONS} result = v; break; }''' create_ast_line = ''' %{AST_TYPE}* v = new %{AST_TYPE}(parent());''' create_identifier_line = ''' v->%{TARGET} = node->v.%{KIND_W/O_SUFFIX}.%{VALUE} ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}))) : 0;''' set_attribute_line = ''' nodeStack.push(v); v->%{TARGET} = static_cast<%{AST_TYPE}*>(visitNode(node->v.%{KIND_W/O_SUFFIX}.%{VALUE})); nodeStack.pop();''' resolve_list_line = ''' nodeStack.push(v); v->%{TARGET} = visitNodeList<%{PYTHON_AST_TYPE}, %{AST_TYPE}>(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}); nodeStack.pop();''' create_identifier_line_any = ''' v->%{TARGET} = node->%{VALUE} ? new Python::Identifier(PyString_AsString(PyObject_Str(node->%{VALUE}))) : 0;''' set_attribute_line_any = ''' nodeStack.push(v); v->%{TARGET} = static_cast<%{AST_TYPE}*>(visitNode(node->%{VALUE})); nodeStack.pop();''' resolve_list_line_any = ''' nodeStack.push(v); v->%{TARGET} = visitNodeList<%{PYTHON_AST_TYPE}, %{AST_TYPE}>(node->%{VALUE}); nodeStack.pop();''' direct_assignment_line = ''' v->%{TARGET} = node->v.%{KIND_W/O_SUFFIX}.%{VALUE};''' direct_assignment_line_any = ''' v->%{TARGET} = node->v.%{VALUE};''' cast_operator_line = ''' v->%{TARGET} = (ExpressionAst::%{AST_TYPE}) node->v.%{KIND_W/O_SUFFIX}.%{VALUE};''' resolve_string = ''' v->%{TARGET} = PyString_AsString(PyObject_Str(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}));''' assign_mindless = ''' v->%{TARGET} = node->%{VALUE};''' resolve_oplist_block = ''' for ( int _i = 0; _i < node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->size; _i++ ) { v->%{TARGET}.append((ExpressionAst::%{AST_TYPE}) node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->elements[_i]); } ''' resolve_identifier_block = ''' for ( int _i = 0; _i < node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyString_AsString(PyObject_Str( static_cast(node->v.%{KIND_W/O_SUFFIX}.%{VALUE}->elements[_i]) ))); v->%{TARGET}.append(id); } ''' copy_ident_ranges = ''' if ( v->%{TARGET} ) { v->%{TARGET}->startCol = node->col_offset; v->startCol = v->%{TARGET}->startCol; v->%{TARGET}->startLine = tline(node->lineno - 1); v->startLine = v->%{TARGET}->startLine; v->%{TARGET}->endCol = node->col_offset + v->%{TARGET}->value.length() - 1; v->endCol = v->%{TARGET}->endCol; v->%{TARGET}->endLine = tline(node->lineno - 1); v->endLine = v->%{TARGET}->endLine; ranges_copied = true; }''' results = dict() does_match_any = dict() def pluginAstToPythonAstType(plugintypestr): if plugintypestr == 'ExpressionAst': return '_expr' if plugintypestr == 'StatementAst' : return '_stmt' if plugintypestr == 'NameAst': return '_expr' if plugintypestr == 'ExceptionHandlerAst': return '_excepthandler' if plugintypestr == 'ComprehensionAst': return '_comprehension' if plugintypestr == 'KeywordAst': return '_keyword' if plugintypestr == 'ArgumentsAst': return '_arguments' if plugintypestr == 'AliasAst': return '_alias' if plugintypestr == 'SliceAst': return '_slice' if plugintypestr == 'Ast': return '_stmt' # not sure about this else: return '' for rule in contents: outline = rule.split(';') command = outline[0] if command[:7] == 'COMMENT' or command == '': continue elif command[:7] != 'RULE_FO': raise SyntaxError('Invalid syntax in sdef file, line: ' + rule) rule_for = outline[0].split(' ')[1] kind = outline[1].split(' ')[1] kind_wo_suffix = kind.replace('_kind', '') actions = outline[2].split(' ')[1:] code = False if len(outline) > 3: code = ' '.join(outline[3].split(' ')[1:]) + ";" if not results.has_key(rule_for): results[rule_for] = list() current_actions = list() for action in actions: command = action.split('|')[0] try: arguments = action.split('|')[1] except IndexError: continue action = '' if command == 'set': s = arguments.split('>') commandType = s[0][-1] # -, ~, =, : , *, # target = s[0][:-1] s = s[1].split(',') raw = False if kind == 'any': any = True else: any = False # commands with one argument if commandType in ['~', ':', '$', '+']: if commandType == ':': raw = direct_assignment_line if not any else direct_assignment_line_any if commandType == '~': raw = create_identifier_line if not any else create_identifier_line_any if rule_for in ['_expr', '_stmt', '_excepthandler', '_alias']: raw += copy_ident_ranges if commandType == '$': raw = resolve_string if commandType == '+': raw = assign_mindless; value = s[0] # commands with two arguments else: astType = s[0] try: value = s[1] except IndexError: raise SyntaxError('Missing argument for operator ' + commandType + ' in rule: ' + rule) if commandType == '=': if astType == 'Identifier': raw = resolve_identifier_block else: raw = resolve_list_line if not any else resolve_list_line_any if commandType == '-': raw = set_attribute_line if not any else set_attribute_line_any if commandType == '*': raw = cast_operator_line if commandType == '#': raw = resolve_oplist_block if raw: command = raw.replace('%{AST_TYPE}', astType).replace('%{TARGET}', target) \ .replace('%{PYTHON_AST_TYPE}', pluginAstToPythonAstType(astType)) \ .replace('%{KIND_W/O_SUFFIX}', kind_wo_suffix).replace('%{VALUE}', value) else: command = '' current_actions.append(command) elif command == 'create': astType = arguments current_actions.append(create_ast_line.replace('%{AST_TYPE}', astType)) if code: current_actions.append(code); current_actions = "\n".join(current_actions) if kind == 'any': current_stmt = current_actions else: current_stmt = switch_line.replace('%{KIND}', kind).replace('%{ACTIONS}', current_actions) results[rule_for].append(current_stmt) does_match_any[rule_for] = any print '''/* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ class PythonAstTransformer { public: CodeAst* ast; + PythonAstTransformer(int lineOffset) : m_lineOffset(lineOffset) {}; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } + // Shift lines by some fixed amount + inline int tline(int line) { + if ( line == -99999 ) { + // don't touch the marker + return -99999; + } + return line + m_lineOffset; + }; private: QStack nodeStack; + int m_lineOffset; Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Q_ASSERT(currentNode); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } ''' for index, lines in results.iteritems(): current_switch_lines = "\n".join(lines) appendix = '' if index == '_expr' or index == '_stmt': appendix = ''' if ( ! result ) return 0; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } ''' appendix += ''' // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } - if ( ! parent->hasUsefulRangeInformation && parent->startLine == tline(-5) ) { + if ( ! parent->hasUsefulRangeInformation && parent->startLine == -99999 ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } ''' if not does_match_any[index]: func = func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines).replace('%{APPENDIX}', appendix) else: func = simple_func_structure.replace('%{RULE_FOR}', index).replace('%{SWITCH_LINES}', current_switch_lines) print func print '''}; /* * End generated code */ ''' diff --git a/parser/generated.h b/parser/generated.h index 75aa8583..a94c10a3 100644 --- a/parser/generated.h +++ b/parser/generated.h @@ -1,670 +1,680 @@ /* This code is generated by conversiongenerator.py. * I do not recommend editing it. * To update, run: python2 conversionGenerator.py > generated.h */ class PythonAstTransformer { public: CodeAst* ast; + PythonAstTransformer(int lineOffset) : m_lineOffset(lineOffset) {}; void run(mod_ty syntaxtree, QString moduleName) { ast = new CodeAst(); ast->name = new Identifier(moduleName); nodeStack.push(ast); ast->body = visitNodeList<_stmt, Ast>(syntaxtree->v.Module.body); nodeStack.pop(); Q_ASSERT(nodeStack.isEmpty()); } + // Shift lines by some fixed amount + inline int tline(int line) { + if ( line == -5 ) { + // don't touch the marker + return -5; + } + return line + m_lineOffset; + }; private: QStack nodeStack; + int m_lineOffset; Ast* parent() { return nodeStack.top(); } template QList visitNodeList(asdl_seq* node) { QList nodelist; if ( ! node ) return nodelist; for ( int i=0; i < node->size; i++ ) { T* currentNode = static_cast(node->elements[i]); Q_ASSERT(currentNode); Ast* result = visitNode(currentNode); K* transformedNode = static_cast(result); nodelist.append(transformedNode); } return nodelist; } Ast* visitNode(_expr* node) { if ( ! node ) return 0; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = 0; switch ( node->kind ) { case BoolOp_kind: { BooleanOperationAst* v = new BooleanOperationAst(parent()); v->type = (ExpressionAst::BooleanOperationTypes) node->v.BoolOp.op; nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.BoolOp.values); nodeStack.pop(); result = v; break; } case BinOp_kind: { BinaryOperationAst* v = new BinaryOperationAst(parent()); v->type = (ExpressionAst::OperatorTypes) node->v.BinOp.op; nodeStack.push(v); v->lhs = static_cast(visitNode(node->v.BinOp.left)); nodeStack.pop(); nodeStack.push(v); v->rhs = static_cast(visitNode(node->v.BinOp.right)); nodeStack.pop(); result = v; break; } case UnaryOp_kind: { UnaryOperationAst* v = new UnaryOperationAst(parent()); v->type = (ExpressionAst::UnaryOperatorTypes) node->v.UnaryOp.op; nodeStack.push(v); v->operand = static_cast(visitNode(node->v.UnaryOp.operand)); nodeStack.pop(); result = v; break; } case Lambda_kind: { LambdaAst* v = new LambdaAst(parent()); nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.Lambda.args)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Lambda.body)); nodeStack.pop(); result = v; break; } case IfExp_kind: { IfExpressionAst* v = new IfExpressionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.IfExp.test)); nodeStack.pop(); nodeStack.push(v); v->body = static_cast(visitNode(node->v.IfExp.body)); nodeStack.pop(); nodeStack.push(v); v->orelse = static_cast(visitNode(node->v.IfExp.orelse)); nodeStack.pop(); result = v; break; } case Dict_kind: { DictAst* v = new DictAst(parent()); nodeStack.push(v); v->keys = visitNodeList<_expr, ExpressionAst>(node->v.Dict.keys); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Dict.values); nodeStack.pop(); result = v; break; } case ListComp_kind: { ListComprehensionAst* v = new ListComprehensionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.ListComp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.ListComp.generators); nodeStack.pop(); result = v; break; } case DictComp_kind: { DictionaryComprehensionAst* v = new DictionaryComprehensionAst(parent()); nodeStack.push(v); v->key = static_cast(visitNode(node->v.DictComp.key)); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.DictComp.value)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.DictComp.generators); nodeStack.pop(); result = v; break; } case GeneratorExp_kind: { GeneratorExpressionAst* v = new GeneratorExpressionAst(parent()); nodeStack.push(v); v->element = static_cast(visitNode(node->v.GeneratorExp.elt)); nodeStack.pop(); nodeStack.push(v); v->generators = visitNodeList<_comprehension, ComprehensionAst>(node->v.GeneratorExp.generators); nodeStack.pop(); result = v; break; } case Yield_kind: { YieldAst* v = new YieldAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Yield.value)); nodeStack.pop(); result = v; break; } case Compare_kind: { CompareAst* v = new CompareAst(parent()); nodeStack.push(v); v->leftmostElement = static_cast(visitNode(node->v.Compare.left)); nodeStack.pop(); for ( int _i = 0; _i < node->v.Compare.ops->size; _i++ ) { v->operators.append((ExpressionAst::ComparisonOperatorTypes) node->v.Compare.ops->elements[_i]); } nodeStack.push(v); v->comparands = visitNodeList<_expr, ExpressionAst>(node->v.Compare.comparators); nodeStack.pop(); result = v; break; } case Call_kind: { CallAst* v = new CallAst(parent()); nodeStack.push(v); v->function = static_cast(visitNode(node->v.Call.func)); nodeStack.pop(); nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->v.Call.args); nodeStack.pop(); nodeStack.push(v); v->keywords = visitNodeList<_keyword, KeywordAst>(node->v.Call.keywords); nodeStack.pop(); nodeStack.push(v); v->keywordArguments = static_cast(visitNode(node->v.Call.kwargs)); nodeStack.pop(); nodeStack.push(v); v->starArguments = static_cast(visitNode(node->v.Call.starargs)); nodeStack.pop(); v->function->belongsToCall = v; result = v; break; } case Repr_kind: { ReprAst* v = new ReprAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Repr.value)); nodeStack.pop(); result = v; break; } case Num_kind: { NumberAst* v = new NumberAst(parent()); v->isInt = PyInt_Check(node->v.Num.n); result = v; break; } case Str_kind: { StringAst* v = new StringAst(parent()); v->value = PyString_AsString(PyObject_Str(node->v.Str.s)); result = v; break; } case Attribute_kind: { AttributeAst* v = new AttributeAst(parent()); v->attribute = node->v.Attribute.attr ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.Attribute.attr))) : 0; if ( v->attribute ) { v->attribute->startCol = node->col_offset; v->startCol = v->attribute->startCol; v->attribute->startLine = tline(node->lineno - 1); v->startLine = v->attribute->startLine; v->attribute->endCol = node->col_offset + v->attribute->value.length() - 1; v->endCol = v->attribute->endCol; v->attribute->endLine = tline(node->lineno - 1); v->endLine = v->attribute->endLine; ranges_copied = true; } nodeStack.push(v); v->value = static_cast(visitNode(node->v.Attribute.value)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Attribute.ctx; result = v; break; } case Subscript_kind: { SubscriptAst* v = new SubscriptAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Subscript.value)); nodeStack.pop(); nodeStack.push(v); v->slice = static_cast(visitNode(node->v.Subscript.slice)); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Subscript.ctx; result = v; break; } case Name_kind: { NameAst* v = new NameAst(parent()); v->identifier = node->v.Name.id ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.Name.id))) : 0; if ( v->identifier ) { v->identifier->startCol = node->col_offset; v->startCol = v->identifier->startCol; v->identifier->startLine = tline(node->lineno - 1); v->startLine = v->identifier->startLine; v->identifier->endCol = node->col_offset + v->identifier->value.length() - 1; v->endCol = v->identifier->endCol; v->identifier->endLine = tline(node->lineno - 1); v->endLine = v->identifier->endLine; ranges_copied = true; } v->context = (ExpressionAst::Context) node->v.Name.ctx; result = v; break; } case List_kind: { ListAst* v = new ListAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.List.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.List.ctx; result = v; break; } case Tuple_kind: { TupleAst* v = new TupleAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Tuple.elts); nodeStack.pop(); v->context = (ExpressionAst::Context) node->v.Tuple.ctx; result = v; break; } case Set_kind: { SetAst* v = new SetAst(parent()); nodeStack.push(v); v->elements = visitNodeList<_expr, ExpressionAst>(node->v.Set.elts); nodeStack.pop(); result = v; break; } default: kWarning() << "Unsupported statement AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return 0; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == tline(-5) ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } NameAst* r = dynamic_cast(result); if ( r ) { r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_excepthandler* node) { if ( ! node ) return 0; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = 0; switch ( node->kind ) { case ExceptHandler_kind: { ExceptionHandlerAst* v = new ExceptionHandlerAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.ExceptHandler.type)); nodeStack.pop(); nodeStack.push(v); v->name = static_cast(visitNode(node->v.ExceptHandler.name)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ExceptHandler.body); nodeStack.pop(); result = v; break; } default: kWarning() << "Unsupported statement AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == tline(-5) ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } NameAst* r = dynamic_cast(result); if ( r ) { r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_comprehension* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return 0; // return a nullpointer if no node is set, that's fine, everyone else will check for that. ComprehensionAst* v = new ComprehensionAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->iter)); nodeStack.pop(); nodeStack.push(v); v->conditions = visitNodeList<_expr, ExpressionAst>(node->ifs); nodeStack.pop(); return v; } Ast* visitNode(_alias* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return 0; // return a nullpointer if no node is set, that's fine, everyone else will check for that. AliasAst* v = new AliasAst(parent()); v->name = node->name ? new Python::Identifier(PyString_AsString(PyObject_Str(node->name))) : 0; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } v->asName = node->asname ? new Python::Identifier(PyString_AsString(PyObject_Str(node->asname))) : 0; if ( v->asName ) { v->asName->startCol = node->col_offset; v->startCol = v->asName->startCol; v->asName->startLine = tline(node->lineno - 1); v->startLine = v->asName->startLine; v->asName->endCol = node->col_offset + v->asName->value.length() - 1; v->endCol = v->asName->endCol; v->asName->endLine = tline(node->lineno - 1); v->endLine = v->asName->endLine; ranges_copied = true; } return v; } Ast* visitNode(_stmt* node) { if ( ! node ) return 0; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = 0; switch ( node->kind ) { case Expr_kind: { ExpressionAst* v = new ExpressionAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Expr.value)); nodeStack.pop(); result = v; break; } case FunctionDef_kind: { FunctionDefinitionAst* v = new FunctionDefinitionAst(parent()); v->name = node->v.FunctionDef.name ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.FunctionDef.name))) : 0; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->arguments = static_cast(visitNode(node->v.FunctionDef.args)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.FunctionDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.FunctionDef.decorator_list); nodeStack.pop(); result = v; break; } case ClassDef_kind: { ClassDefinitionAst* v = new ClassDefinitionAst(parent()); v->name = node->v.ClassDef.name ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.ClassDef.name))) : 0; if ( v->name ) { v->name->startCol = node->col_offset; v->startCol = v->name->startCol; v->name->startLine = tline(node->lineno - 1); v->startLine = v->name->startLine; v->name->endCol = node->col_offset + v->name->value.length() - 1; v->endCol = v->name->endCol; v->name->endLine = tline(node->lineno - 1); v->endLine = v->name->endLine; ranges_copied = true; } nodeStack.push(v); v->baseClasses = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.bases); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.ClassDef.body); nodeStack.pop(); nodeStack.push(v); v->decorators = visitNodeList<_expr, ExpressionAst>(node->v.ClassDef.decorator_list); nodeStack.pop(); result = v; break; } case Return_kind: { ReturnAst* v = new ReturnAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Return.value)); nodeStack.pop(); result = v; break; } case Delete_kind: { DeleteAst* v = new DeleteAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Delete.targets); nodeStack.pop(); result = v; break; } case Assign_kind: { AssignmentAst* v = new AssignmentAst(parent()); nodeStack.push(v); v->targets = visitNodeList<_expr, ExpressionAst>(node->v.Assign.targets); nodeStack.pop(); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Assign.value)); nodeStack.pop(); result = v; break; } case AugAssign_kind: { AugmentedAssignmentAst* v = new AugmentedAssignmentAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.AugAssign.target)); nodeStack.pop(); v->op = (ExpressionAst::OperatorTypes) node->v.AugAssign.op; nodeStack.push(v); v->value = static_cast(visitNode(node->v.AugAssign.value)); nodeStack.pop(); result = v; break; } case Print_kind: { PrintAst* v = new PrintAst(parent()); nodeStack.push(v); v->destination = static_cast(visitNode(node->v.Print.dest)); nodeStack.pop(); nodeStack.push(v); v->values = visitNodeList<_expr, ExpressionAst>(node->v.Print.values); nodeStack.pop(); v->newline = node->v.Print.nl; result = v; break; } case For_kind: { ForAst* v = new ForAst(parent()); nodeStack.push(v); v->target = static_cast(visitNode(node->v.For.target)); nodeStack.pop(); nodeStack.push(v); v->iterator = static_cast(visitNode(node->v.For.iter)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.For.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.For.orelse); nodeStack.pop(); result = v; break; } case While_kind: { WhileAst* v = new WhileAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.While.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.While.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.While.orelse); nodeStack.pop(); result = v; break; } case If_kind: { IfAst* v = new IfAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.If.test)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.If.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.If.orelse); nodeStack.pop(); result = v; break; } case With_kind: { WithAst* v = new WithAst(parent()); nodeStack.push(v); v->contextExpression = static_cast(visitNode(node->v.With.context_expr)); nodeStack.pop(); nodeStack.push(v); v->optionalVars = static_cast(visitNode(node->v.With.optional_vars)); nodeStack.pop(); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.With.body); nodeStack.pop(); result = v; break; } case Raise_kind: { RaiseAst* v = new RaiseAst(parent()); nodeStack.push(v); v->type = static_cast(visitNode(node->v.Raise.type)); nodeStack.pop(); result = v; break; } case TryExcept_kind: { TryExceptAst* v = new TryExceptAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.TryExcept.body); nodeStack.pop(); nodeStack.push(v); v->orelse = visitNodeList<_stmt, Ast>(node->v.TryExcept.orelse); nodeStack.pop(); nodeStack.push(v); v->handlers = visitNodeList<_excepthandler, ExceptionHandlerAst>(node->v.TryExcept.handlers); nodeStack.pop(); result = v; break; } case TryFinally_kind: { TryFinallyAst* v = new TryFinallyAst(parent()); nodeStack.push(v); v->body = visitNodeList<_stmt, Ast>(node->v.TryFinally.body); nodeStack.pop(); nodeStack.push(v); v->finalbody = visitNodeList<_stmt, Ast>(node->v.TryFinally.finalbody); nodeStack.pop(); result = v; break; } case Assert_kind: { AssertionAst* v = new AssertionAst(parent()); nodeStack.push(v); v->condition = static_cast(visitNode(node->v.Assert.test)); nodeStack.pop(); nodeStack.push(v); v->message = static_cast(visitNode(node->v.Assert.msg)); nodeStack.pop(); result = v; break; } case Import_kind: { ImportAst* v = new ImportAst(parent()); nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.Import.names); nodeStack.pop(); result = v; break; } case ImportFrom_kind: { ImportFromAst* v = new ImportFromAst(parent()); v->module = node->v.ImportFrom.module ? new Python::Identifier(PyString_AsString(PyObject_Str(node->v.ImportFrom.module))) : 0; if ( v->module ) { v->module->startCol = node->col_offset; v->startCol = v->module->startCol; v->module->startLine = tline(node->lineno - 1); v->startLine = v->module->startLine; v->module->endCol = node->col_offset + v->module->value.length() - 1; v->endCol = v->module->endCol; v->module->endLine = tline(node->lineno - 1); v->endLine = v->module->endLine; ranges_copied = true; } nodeStack.push(v); v->names = visitNodeList<_alias, AliasAst>(node->v.ImportFrom.names); nodeStack.pop(); v->level = node->v.ImportFrom.level; result = v; break; } case Exec_kind: { ExecAst* v = new ExecAst(parent()); nodeStack.push(v); v->body = static_cast(visitNode(node->v.Exec.body)); nodeStack.pop(); nodeStack.push(v); v->globals = static_cast(visitNode(node->v.Exec.globals)); nodeStack.pop(); nodeStack.push(v); v->locals = static_cast(visitNode(node->v.Exec.locals)); nodeStack.pop(); result = v; break; } case Global_kind: { GlobalAst* v = new GlobalAst(parent()); for ( int _i = 0; _i < node->v.Global.names->size; _i++ ) { Python::Identifier* id = new Python::Identifier(PyString_AsString(PyObject_Str( static_cast(node->v.Global.names->elements[_i]) ))); v->names.append(id); } result = v; break; } case Break_kind: { BreakAst* v = new BreakAst(parent()); result = v; break; } case Continue_kind: { ContinueAst* v = new ContinueAst(parent()); result = v; break; } case Pass_kind: { PassAst* v = new PassAst(parent()); result = v; break; } default: kWarning() << "Unsupported statement AST type: " << node->kind; Q_ASSERT(false); } if ( ! result ) return 0; if ( ! ranges_copied ) { result->startCol = node->col_offset; result->endCol = node->col_offset; result->startLine = tline(node->lineno - 1); result->endLine = tline(node->lineno - 1); result->hasUsefulRangeInformation = true; } else { result->hasUsefulRangeInformation = true; } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == tline(-5) ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } NameAst* r = dynamic_cast(result); if ( r ) { r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_slice* node) { if ( ! node ) return 0; bool ranges_copied = false; Q_UNUSED(ranges_copied); Ast* result = 0; switch ( node->kind ) { case Slice_kind: { SliceAst* v = new SliceAst(parent()); nodeStack.push(v); v->lower = static_cast(visitNode(node->v.Slice.lower)); nodeStack.pop(); nodeStack.push(v); v->upper = static_cast(visitNode(node->v.Slice.upper)); nodeStack.pop(); nodeStack.push(v); v->step = static_cast(visitNode(node->v.Slice.step)); nodeStack.pop(); result = v; break; } case ExtSlice_kind: { ExtendedSliceAst* v = new ExtendedSliceAst(parent()); nodeStack.push(v); v->dims = visitNodeList<_slice, SliceAst>(node->v.ExtSlice.dims); nodeStack.pop(); result = v; break; } case Index_kind: { IndexAst* v = new IndexAst(parent()); nodeStack.push(v); v->value = static_cast(visitNode(node->v.Index.value)); nodeStack.pop(); result = v; break; } case Ellipsis_kind: { EllipsisAst* v = new EllipsisAst(parent()); result = v; break; } default: kWarning() << "Unsupported statement AST type: " << node->kind; Q_ASSERT(false); } // Walk through the tree and set proper end columns and lines, as the python parser sadly does not do this for us if ( result->hasUsefulRangeInformation ) { Ast* parent = result->parent; while ( parent ) { if ( parent->endLine < result->endLine ) { parent->endLine = result->endLine; parent->endCol = result->endCol; } if ( ! parent->hasUsefulRangeInformation && parent->startLine == tline(-5) ) { parent->startLine = result->startLine; parent->startCol = result->startCol; } parent = parent->parent; } } NameAst* r = dynamic_cast(result); if ( r ) { r->startCol = r->identifier->startCol; r->endCol = r->identifier->endCol; r->startLine = r->identifier->startLine; r->endLine = r->identifier->endLine; } return result; } Ast* visitNode(_arguments* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return 0; // return a nullpointer if no node is set, that's fine, everyone else will check for that. ArgumentsAst* v = new ArgumentsAst(parent()); v->vararg = node->vararg ? new Python::Identifier(PyString_AsString(PyObject_Str(node->vararg))) : 0; v->kwarg = node->kwarg ? new Python::Identifier(PyString_AsString(PyObject_Str(node->kwarg))) : 0; nodeStack.push(v); v->arguments = visitNodeList<_expr, ExpressionAst>(node->args); nodeStack.pop(); nodeStack.push(v); v->defaultValues = visitNodeList<_expr, ExpressionAst>(node->defaults); nodeStack.pop(); v->arg_lineno = node->arg_lineno; v->arg_col_offset = node->arg_col_offset; v->vararg_lineno = node->vararg_lineno; v->vararg_col_offset = node->vararg_col_offset; return v; } Ast* visitNode(_keyword* node) { bool ranges_copied = false; Q_UNUSED(ranges_copied); if ( ! node ) return 0; // return a nullpointer if no node is set, that's fine, everyone else will check for that. KeywordAst* v = new KeywordAst(parent()); v->argumentName = node->arg ? new Python::Identifier(PyString_AsString(PyObject_Str(node->arg))) : 0; nodeStack.push(v); v->value = static_cast(visitNode(node->value)); nodeStack.pop(); return v; } }; /* * End generated code */