diff --git a/completion/context.cpp b/completion/context.cpp index ba8f548..c14b88c 100644 --- a/completion/context.cpp +++ b/completion/context.cpp @@ -1,1785 +1,1786 @@ /* Copyright 2007 David Nolden Copyright 2008 Hamish Rodda Copyright 2008 Niko Sams Copyright 2009 Milian Wolff This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "context.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../duchain/completioncodemodel.h" #include "../duchain/expressionparser.h" #include "../duchain/helper.h" #include "../duchain/declarations/variabledeclaration.h" #include "../duchain/declarations/classmethoddeclaration.h" #include "../duchain/types/structuretype.h" #include "../parser/phpparser.h" #include "../parser/phptokentext.h" #include "includefileitem.h" #include "codemodelitem.h" #include "completiondebug.h" #include "helpers.h" #include "implementationitem.h" #include "keyworditem.h" #include #define LOCKDUCHAIN DUChainReadLocker lock(DUChain::lock()) #define ifDebug(x) using namespace KDevelop; namespace Php { typedef QList TokenList; /** * Utility class which makes it easier to access the relevant parts * of the token stream for code completion. * * TODO: This class should be reviewed imo - I just hacked it together, quick'n'dirty */ class TokenAccess { public: /// Setup the token stream from the input code TokenAccess(const QString &code) : m_code(code) { Lexer lexer(&m_stream, code); int token; while ((token = lexer.nextTokenKind())) { Parser::Token &t = m_stream.push(); t.begin = lexer.tokenBegin(); t.end = lexer.tokenEnd(); t.kind = token; } // move to last token m_pos = m_stream.size() - 1; } /// returns Token_INVALID if the position is invalid /// else returns the type of the current token Parser::TokenType type() const { if ( m_pos == -1 ) { return Parser::Token_INVALID; } else { return (Parser::TokenType) m_stream.at(m_pos).kind; } } /// convenience comparison to a tokentype bool operator==(const Parser::TokenType& other) const { return other == type(); } /// move to previous token void pop() { if ( m_pos >= 0 ) { --m_pos; } } /// move relative to current token /// NOTE: make sure you honor the boundaries. void moveTo(const qint64 &relPos) { m_pos += relPos; Q_ASSERT(m_pos > 0); Q_ASSERT(m_pos < m_stream.size()); } /// get type of token relative to current position /// returns Token_INVALID if the position goes out of the boundaries int typeAt(const qint64 &relPos) const { const qint64 pos = m_pos + relPos; if ( pos >= 0 && pos < m_stream.size() ) { return m_stream.at(pos).kind; } else { return Parser::Token_INVALID; } } /// Get string for token at a given position relative to the current one. /// NOTE: Make sure you honor the boundaries. QString stringAt(const qint64 &relPos) const { Parser::Token token = at(relPos); return m_code.mid(token.begin, token.end - token.begin + 1); } /// check whether the current token is prepended by the list of tokens /// @return -1 when not prepended by the list, else the relative index-position qint64 prependedBy(const TokenList &list, bool skipWhitespace = false ) const { // this would be useless, hence forbid it Q_ASSERT ( !list.isEmpty() ); if ( m_pos < list.count() - 1 ) { // not enough tokens return -1; } else { uint pos = 1; foreach ( Parser::TokenType type, list ) { if ( skipWhitespace && m_stream.at( m_pos - pos).kind == Parser::Token_WHITESPACE ) { ++pos; } if ( m_stream.at( m_pos - pos).kind == type ) { ++pos; continue; } else { return -1; } } return pos; } } /// Get the token relative to the current one. /// NOTE: Make sure you honor the boundaries. Parser::Token at(const qint64 &relPos) const { const qint64 pos = m_pos + relPos; Q_ASSERT(pos >= 0); Q_ASSERT(pos < m_stream.size()); return m_stream.at(pos); } private: const QString m_code; TokenStream m_stream; qint64 m_pos; }; /** * Pops all tokens from the @p lastToken and stops at the LPAREN. */ void removeOtherArguments(TokenAccess &lastToken) { Q_ASSERT(lastToken.type() == Parser::Token_COMMA); // remove all other arguments int openLParen = 0; do { lastToken.pop(); if ( lastToken.type() == Parser::Token_RPAREN ) { ++openLParen; } else if ( lastToken.type() == Parser::Token_LPAREN ) { if ( openLParen == 0 ) { return; } else { --openLParen; } } } while ( lastToken.type() != Parser::Token_INVALID ); } /** * if token at @p pos is whitespace, decrease pos by one. */ inline void skipWhiteSpace(const TokenAccess &lastToken, qint64 &pos) { if ( lastToken.typeAt(pos) == Parser::Token_WHITESPACE ) { --pos; } } /// add keyword to list of completion items #define ADD_KEYWORD(x) items << CompletionTreeItemPointer( new KeywordItem( QStringLiteral(x), Php::CodeCompletionContext::Ptr(this) ) ) #define ADD_KEYWORD2(x, y) items << CompletionTreeItemPointer( new KeywordItem( QStringLiteral(x), Php::CodeCompletionContext::Ptr(this), QStringLiteral(y) ) ) int completionRecursionDepth = 0; CodeCompletionContext::CodeCompletionContext(KDevelop::DUContextPointer context, const QString& text, const QString& followingText, const KDevelop::CursorInRevision& position, int depth) : KDevelop::CodeCompletionContext(context, text, position, depth) , m_memberAccessOperation(NoMemberAccess), m_parentAccess(false), m_isFileCompletionAfterDirname(false) { // use other ctor for parents Q_ASSERT(depth == 0); ifDebug(qCDebug(COMPLETION) << "non-processed text: " + text;) if ( context->type() == DUContext::Class || context->type() == DUContext::Function || context->type() == DUContext::Other || context->type() == DUContext::Namespace ) { if ( !m_parentContext && !m_text.startsWith(QLatin1String(" don't offer completion if ( m_text.mid( lastTokenEnd, 2 ) == QLatin1String("/*") ) { ifDebug(qCDebug(COMPLETION) << "no completion in comments"); m_valid = false; return; } ifDebug(qCDebug(COMPLETION) << tokenText(lastToken.type());) ///TODO: REFACTOR: push some stuff into its own methods /// and call them from inside the big switch. /// Then we can forget about having additional checks /// beforehand and can handle it all in one place. // The following tokens require a whitespace after them for code-completion: if ( !lastWasWhitespace ) { switch ( lastToken.type() ) { case Parser::Token_EXTENDS: case Parser::Token_IMPLEMENTS: case Parser::Token_NEW: case Parser::Token_THROW: ifDebug(qCDebug(COMPLETION) << "need whitespace after token for completion";) m_valid = false; return; default: break; } } ifDebug(qCDebug(COMPLETION) << tokenText(lastToken.type());) switch ( lastToken.type() ) { case Parser::Token_COMMENT: // don't offer code completion in comments, i.e. single line comments that don't end on \n // multi-line comments are handled above if ( !lastWasWhitespace && !lastToken.stringAt(0).endsWith('\n') && !lastToken.stringAt(0).startsWith(QLatin1String("/*")) ) { ifDebug(qCDebug(COMPLETION) << "no completion in comments";) m_valid = false; } break; case Parser::Token_EXTENDS: if ( lastToken.prependedBy(TokenList() << Parser::Token_WHITESPACE << Parser::Token_STRING << Parser::Token_WHITESPACE << Parser::Token_CLASS) != -1 ) { m_memberAccessOperation = ClassExtendsChoose; forbidIdentifier(lastToken.stringAt(-2)); } else if ( lastToken.prependedBy(TokenList() << Parser::Token_WHITESPACE << Parser::Token_STRING << Parser::Token_WHITESPACE << Parser::Token_INTERFACE) != -1 ) { m_memberAccessOperation = InterfaceChoose; forbidIdentifier(lastToken.stringAt(-2)); } else { ifDebug(qCDebug(COMPLETION) << "token prepended by bad tokens, don't do completion";) m_valid = false; } break; case Parser::Token_IMPLEMENTS: if ( lastToken.prependedBy(TokenList() << Parser::Token_WHITESPACE << Parser::Token_STRING << Parser::Token_WHITESPACE << Parser::Token_CLASS) != -1 ) { m_memberAccessOperation = InterfaceChoose; forbidIdentifier(lastToken.stringAt(-2)); } else { ifDebug(qCDebug(COMPLETION) << "token prepended by bad tokens, don't do completion";) m_valid = false; } break; case Parser::Token_COMMA: { // check if we are in the list after Token_IMPLEMENTS: qint64 relPos = -1; QList identifierPositions; while ( true ) { skipWhiteSpace(lastToken, relPos); if ( lastToken.typeAt(relPos) == Parser::Token_STRING ) { identifierPositions << relPos; --relPos; skipWhiteSpace(lastToken, relPos); // interfaces may extend more than one interface if ( ( lastToken.typeAt(relPos) == Parser::Token_EXTENDS && lastToken.typeAt(relPos - 1) == Parser::Token_WHITESPACE && lastToken.typeAt(relPos - 2) == Parser::Token_STRING && lastToken.typeAt(relPos - 3) == Parser::Token_WHITESPACE && lastToken.typeAt(relPos - 4) == Parser::Token_INTERFACE ) || // classes may implement more than one interface ( lastToken.typeAt(relPos) == Parser::Token_IMPLEMENTS && lastToken.typeAt(relPos - 1) == Parser::Token_WHITESPACE && lastToken.typeAt(relPos - 2) == Parser::Token_STRING && lastToken.typeAt(relPos - 3) == Parser::Token_WHITESPACE && lastToken.typeAt(relPos - 4) == Parser::Token_CLASS ) ) { identifierPositions << (relPos - 2); m_memberAccessOperation = InterfaceChoose; break; } else if ( lastToken.typeAt(relPos) == Parser::Token_COMMA ) { // skip to next entry --relPos; continue; } } else { break; } } if ( m_memberAccessOperation == InterfaceChoose ) { ifDebug(qCDebug(COMPLETION) << "in implementation list";) m_memberAccessOperation = InterfaceChoose; foreach ( qint64 pos, identifierPositions ) { forbidIdentifier(lastToken.stringAt(pos)); } } else { // else do function call completion m_memberAccessOperation = FunctionCallAccess; ///TODO: global, static etc. enumerations. removeOtherArguments(lastToken); if ( lastToken.type() == Parser::Token_INVALID ) { m_valid = false; } } } break; case Parser::Token_OPEN_TAG: // don't do completion if no whitespace is given and there is some text following, // esp. for stuff like type() == DUContext::Class ) { // when we are inside a class context, give overloadable members as completion m_memberAccessOperation = ClassMemberChoose; } else { m_memberAccessOperation = NoMemberAccess; } break; case Parser::Token_ABSTRACT: case Parser::Token_CONST: case Parser::Token_FINAL: case Parser::Token_PUBLIC: case Parser::Token_PRIVATE: case Parser::Token_PROTECTED: case Parser::Token_STATIC: case Parser::Token_VAR: if ( duContext() && duContext()->type() == DUContext::Class ) { // when we are inside a class context, give overloadable members as completion m_memberAccessOperation = ClassMemberChoose; } else { m_valid = false; } break; case Parser::Token_NAMESPACE: case Parser::Token_BACKSLASH: { QString identifier; qint64 relPos = 0; while (lastToken.typeAt(relPos) == Parser::Token_STRING || lastToken.typeAt(relPos) == Parser::Token_BACKSLASH) { if (lastToken.typeAt(relPos) == Parser::Token_BACKSLASH) { identifier.prepend("::"); } else { identifier.prepend(lastToken.stringAt(relPos)); } --relPos; } if ( lastToken.typeAt(relPos) == Parser::Token_NAMESPACE ) { m_memberAccessOperation = NamespaceChoose; } else { m_memberAccessOperation = BackslashAccess; } m_namespace = QualifiedIdentifier(identifier); break; } case Parser::Token_ARRAY: case Parser::Token_AS: case Parser::Token_BACKTICK: case Parser::Token_BOOL: case Parser::Token_BREAK: case Parser::Token_CALLABLE: case Parser::Token_CASE: case Parser::Token_CATCH: case Parser::Token_CLASS: case Parser::Token_CLASS_C: case Parser::Token_CLONE: case Parser::Token_CLOSE_TAG: case Parser::Token_CONTINUE: case Parser::Token_DECLARE: case Parser::Token_DEFAULT: case Parser::Token_DNUMBER: case Parser::Token_DO: case Parser::Token_DOLLAR: case Parser::Token_ELSE: case Parser::Token_ELSEIF: case Parser::Token_EMPTY: case Parser::Token_ENDDECLARE: case Parser::Token_ENDFOR: case Parser::Token_ENDFOREACH: case Parser::Token_ENDIF: case Parser::Token_ENDSWITCH: case Parser::Token_ENDWHILE: case Parser::Token_END_HEREDOC: case Parser::Token_END_NOWDOC: case Parser::Token_EOF: case Parser::Token_EVAL: case Parser::Token_FILE: case Parser::Token_FINALLY: case Parser::Token_FLOAT: case Parser::Token_FOR: case Parser::Token_FOREACH: case Parser::Token_FUNCTION: case Parser::Token_FUNC_C: case Parser::Token_GLOBAL: case Parser::Token_HALT_COMPILER: case Parser::Token_IF: case Parser::Token_INCLUDE: case Parser::Token_INCLUDE_ONCE: case Parser::Token_INLINE_HTML: case Parser::Token_INSTEADOF: case Parser::Token_INT: case Parser::Token_INTERFACE: case Parser::Token_INVALID: case Parser::Token_ISSET: case Parser::Token_ITERABLE: case Parser::Token_LINE: case Parser::Token_LIST: case Parser::Token_LNUMBER: case Parser::Token_METHOD_C: case Parser::Token_NAMESPACE_C: case Parser::Token_NUM_STRING: case Parser::Token_REQUIRE: case Parser::Token_REQUIRE_ONCE: case Parser::Token_RBRACKET: case Parser::Token_RPAREN: case Parser::Token_STRING_TYPE: case Parser::Token_STRING_VARNAME: case Parser::Token_SWITCH: case Parser::Token_TRAIT: case Parser::Token_TRY: case Parser::Token_UNSET: case Parser::Token_USE: case Parser::Token_VARIABLE: + case Parser::Token_VOID: case Parser::Token_WHILE: case Parser::Token_WHITESPACE: /// TODO: code completion after goto case Parser::Token_GOTO: case Parser::TokenTypeSize: ifDebug(qCDebug(COMPLETION) << "no completion after this token";) m_valid = false; break; } ifDebug( switch ( m_memberAccessOperation ) { case FileChoose: qCDebug(COMPLETION) << "FileChoose"; break; case ExceptionInstanceChoose: qCDebug(COMPLETION) << "ExceptionInstanceChoose"; break; case ExceptionChoose: qCDebug(COMPLETION) << "ExceptionChoose"; break; case ClassMemberChoose: qCDebug(COMPLETION) << "ClassMemberChoose"; break; case NoMemberAccess: qCDebug(COMPLETION) << "NoMemberAccess"; break; case NewClassChoose: qCDebug(COMPLETION) << "NewClassChoose"; break; case FunctionCallAccess: qCDebug(COMPLETION) << "FunctionCallAccess"; break; case InterfaceChoose: qCDebug(COMPLETION) << "InterfaceChoose"; break; case ClassExtendsChoose: qCDebug(COMPLETION) << "ClassExtendsChoose"; break; case MemberAccess: qCDebug(COMPLETION) << "MemberAccess"; break; case StaticMemberAccess: qCDebug(COMPLETION) << "StaticMemberAccess"; break; case InstanceOfChoose: qCDebug(COMPLETION) << "InstanceOfChoose"; break; case NamespaceChoose: qCDebug(COMPLETION) << "NamespaceChoose"; break; case BackslashAccess: qCDebug(COMPLETION) << "BackslashAccess"; break; } ) ifDebug(qCDebug(COMPLETION) << tokenText(lastToken.type());) // if it's not valid, we should return early if ( !m_valid ) { ifDebug(qCDebug(COMPLETION) << "invalid completion";) return; } // trim the text to the end position of the current token m_text = m_text.left(lastToken.at(0).end + 1).trimmed(); ifDebug(qCDebug(COMPLETION) << "trimmed text: " << m_text;) // check whether we need the expression or have everything we need and can return early switch ( m_memberAccessOperation ) { // these access operations don't need the previous expression evaluated case FileChoose: case ClassMemberChoose: case InterfaceChoose: case NewClassChoose: case ExceptionChoose: case ExceptionInstanceChoose: case ClassExtendsChoose: case NoMemberAccess: case InstanceOfChoose: case NamespaceChoose: case BackslashAccess: ifDebug(qCDebug(COMPLETION) << "returning early";) return; case FunctionCallAccess: m_memberAccessOperation = NoMemberAccess; Q_ASSERT(lastToken.type() == Parser::Token_LPAREN); if ( lastToken.prependedBy(TokenList() << Parser::Token_STRING, true) == -1 && lastToken.prependedBy(TokenList() << Parser::Token_VARIABLE, true) == -1 ) { // handle for, foreach, while, etc. ifDebug(qCDebug(COMPLETION) << "NoMemberAccess (no function call)";) } else { //The first context should never be a function-call context, //so make this a NoMemberAccess context and the parent a function-call context. ifDebug(qCDebug(COMPLETION) << "NoMemberAccess (creating parentContext for function call)";) m_parentContext = new CodeCompletionContext(m_duContext, m_position, lastToken, depth + 1); } return; case MemberAccess: case StaticMemberAccess: // these types need the expression evaluated break; } evaluateExpression(lastToken); } CodeCompletionContext::CodeCompletionContext(KDevelop::DUContextPointer context, const KDevelop::CursorInRevision& position, TokenAccess& lastToken, int depth) : KDevelop::CodeCompletionContext(context, QString(), position, depth) , m_memberAccessOperation(NoMemberAccess), m_parentAccess(false), m_isFileCompletionAfterDirname(false) { switch ( lastToken.type() ) { case Parser::Token_LPAREN: m_memberAccessOperation = FunctionCallAccess; break; default: qCDebug(COMPLETION) << "unhandled token type for parent context" << tokenText(lastToken.typeAt(0)); Q_ASSERT(false); m_valid = false; return; } evaluateExpression(lastToken); } void CodeCompletionContext::evaluateExpression(TokenAccess& lastToken) { /// token pos qint64 startPos = 0; int openLParen = 0; if ( m_memberAccessOperation == FunctionCallAccess ) { Q_ASSERT(lastToken.type() == Parser::Token_LPAREN); // check ctor call qint64 pos = lastToken.prependedBy(TokenList() << Parser::Token_STRING << Parser::Token_NEW, true); if ( pos != -1 ) { startPos = -pos; ifDebug(qCDebug(COMPLETION) << "ctor call";) } else { // simple function call, get it's expression startPos = -1; ifDebug(qCDebug(COMPLETION) << "simple function call";) } } static const QList defaultStopTokens = QList() << Parser::Token_SEMICOLON << Parser::Token_INVALID << Parser::Token_OPEN_TAG << Parser::Token_OPEN_TAG_WITH_ECHO << Parser::Token_LBRACE << Parser::Token_RBRACE << Parser::Token_IF << Parser::Token_WHILE << Parser::Token_FOR << Parser::Token_FOREACH << Parser::Token_SWITCH << Parser::Token_ELSEIF; // find expression start while ( !defaultStopTokens.contains(lastToken.typeAt(startPos)) && (m_memberAccessOperation == FunctionCallAccess || lastToken.typeAt(startPos) != Parser::Token_COMMA) ) { if ( lastToken.typeAt(startPos) == Parser::Token_LPAREN ) { ++openLParen; if ( openLParen > 0 ) { break; } } else if ( lastToken.typeAt(startPos) == Parser::Token_RPAREN ) { --openLParen; } --startPos; } if ( openLParen < 0 ) { ifDebug(qCDebug(COMPLETION) << "too many closed parenthesis";) m_valid = false; return; } // we actually incorporate the not-wanted token, hence move forward ++startPos; if ( lastToken.typeAt(startPos) == Parser::Token_WHITESPACE ) { ++startPos; } if ( lastToken.typeAt(startPos) == Parser::Token_RETURN ) { ///TODO: match against function return type ++startPos; if ( lastToken.typeAt(startPos) == Parser::Token_WHITESPACE ) { ++startPos; } } if ( m_memberAccessOperation == StaticMemberAccess ) { if ( lastToken.typeAt(startPos) != Parser::Token_STRING ) { ifDebug(qCDebug(COMPLETION) << "unsupported token for start member access:" << tokenText(lastToken.typeAt(startPos));) m_valid = false; return; } const QString identifier(lastToken.stringAt(startPos).toLower()); if ( identifier == QLatin1String("self") || identifier == QLatin1String("parent") || identifier == QLatin1String("static") ) { // self and parent are only accessible from within a member function of a class if (DUContext* parent = m_duContext->parentContext()) { LOCKDUCHAIN; ClassDeclaration* classDec = dynamic_cast(parent->owner()); if (classDec) { if (identifier == QLatin1String("parent")) { FOREACH_FUNCTION(const BaseClassInstance& base, classDec->baseClasses) { if (StructureType::Ptr classType = base.baseClass.type()) { if (ClassDeclaration* baseClass = dynamic_cast(classType->declaration(m_duContext->topContext()))) { if (baseClass->classType() == ClassDeclarationData::Class && baseClass->classModifier() != ClassDeclarationData::Abstract) { ifDebug(qCDebug(COMPLETION) << "correction: parent can do MemberAccess"); m_parentAccess = true; m_memberAccessOperation = MemberAccess; m_expressionResult.setDeclaration(baseClass); break; } } } } if (!m_parentAccess) { ifDebug(qCDebug(COMPLETION) << "class has no accessible parent class"); m_valid = false; return; } } else { m_expressionResult.setDeclaration(parent->owner()); } } } } else { QualifiedIdentifier id(identifier); m_expressionResult.setDeclaration(findDeclarationImportHelper(duContext(), id, ClassDeclarationType)); } } else { // Now get the string of the expression and evaluate it Q_ASSERT(m_expression.isEmpty()); for (qint64 i = startPos; i <= 0; ++i ) { m_expression += lastToken.stringAt(i); } m_expression = m_expression.trimmed(); // make sure the expression is valid if (m_memberAccessOperation == FunctionCallAccess) { m_expression.append(')'); } for ( int i = openLParen; i > 0; --i ) { m_expression.append(')'); } ifDebug(qCDebug(COMPLETION) << "expression: " << m_expression;) if ( !m_expression.isEmpty() ) { ExpressionParser expressionParser; m_expressionResult = expressionParser.evaluateType(m_expression.toUtf8(), m_duContext, m_position); } if (m_expressionResult.type()) { LOCKDUCHAIN; ifDebug(qCDebug(COMPLETION) << "expression type: " << m_expressionResult.type()->toString();) } else { ifDebug(qCDebug(COMPLETION) << QString("expression could not be evaluated")); if ( m_memberAccessOperation == FunctionCallAccess ) { ifDebug(qCDebug(COMPLETION) << "function not found";) return; } m_valid = false; return; } } lastToken.moveTo(startPos); // Handle recursive contexts (Example: "ret = function1(param1, function2(" ) if ( lastToken.typeAt(-1) == Parser::Token_LPAREN || lastToken.typeAt(-1) == Parser::Token_COMMA ) { //Our expression is within a function-call. We need to find out the possible argument-types we need to match, and show an argument-hint. lastToken.moveTo(-1); if ( lastToken.type() == Parser::Token_COMMA ) { removeOtherArguments(lastToken); if ( lastToken.type() == Parser::Token_INVALID ) { ifDebug(qCDebug(COMPLETION) << QString("Could not find start position for parent function-call. Aborting.");) m_valid = false; return; } } if ( lastToken.prependedBy(TokenList() << Parser::Token_STRING, true) == -1 ) { // handle for, foreach, while, etc. ifDebug(qCDebug(COMPLETION) << "No real function call";) return; } ifDebug(qCDebug(COMPLETION) << QString("Recursive function-call: creating parent context")); m_parentContext = new CodeCompletionContext(m_duContext, m_position, lastToken, m_depth + 1); if (!m_parentContext->isValid()) { m_parentContext = nullptr; m_valid = false; return; } } } void CodeCompletionContext::forbidIdentifier(const QString& identifier) { QualifiedIdentifier id(identifier.toLower()); ClassDeclaration *dec = dynamic_cast( findDeclarationImportHelper(m_duContext.data(), id, ClassDeclarationType).data() ); if (dec) { forbidIdentifier(dec); } else { // might be a class we are currently writing, i.e. without a proper declaration m_forbiddenIdentifiers << id.index(); } } void CodeCompletionContext::forbidIdentifier(ClassDeclaration* klass) { uint id; { LOCKDUCHAIN; // TODO: qualifiedIdentifier is marked as expensive - any other way // we can do what we are doing here? // TODO: maybe we should clar the m_fobiddenIdentifiers after we got // our list of items... id = klass->qualifiedIdentifier().index(); } if (m_forbiddenIdentifiers.contains(id)) { // nothing to do return; } m_forbiddenIdentifiers << id; // add parents so those are excluded from the completion items as well if (klass->baseClassesSize() > 0) { FOREACH_FUNCTION(const BaseClassInstance& base, klass->baseClasses) { StructureType::Ptr type = base.baseClass.type(); if (type.data()) { ClassDeclaration* parent; { LOCKDUCHAIN; parent = dynamic_cast( type->declaration(m_duContext->topContext()) ); } if (parent) { forbidIdentifier(parent); } } } } } CodeCompletionContext::~CodeCompletionContext() { } CodeCompletionContext::MemberAccessOperation CodeCompletionContext::memberAccessOperation() const { return m_memberAccessOperation; } ExpressionEvaluationResult CodeCompletionContext::memberAccessContainer() const { return m_expressionResult; } CodeCompletionContext* CodeCompletionContext::parentContext() { return static_cast(KDevelop::CodeCompletionContext::parentContext()); } QList CodeCompletionContext::memberAccessContainers() const { QList ret; QList types; AbstractType::Ptr expressionTarget = m_expressionResult.type(); if (UnsureType::Ptr unsureType = UnsureType::Ptr::dynamicCast(m_expressionResult.type())) { FOREACH_FUNCTION(const IndexedType& t, unsureType->types) { types << t.abstractType(); } } else if (ReferenceType::Ptr referencedType = ReferenceType::Ptr::dynamicCast(m_expressionResult.type()) ) { types << referencedType->baseType(); } else { types << expressionTarget; } foreach (const AbstractType::Ptr &type, types) { const IdentifiedType* idType = dynamic_cast(type.data()); Declaration* declaration = nullptr; if (idType) { declaration = idType->declaration(m_duContext->topContext()); } DUContext* ctx = nullptr; if (declaration) { ctx = declaration->logicalInternalContext(m_duContext->topContext()); } if (ctx) { ret << ctx; } else if (declaration) { //Print some debug-output qCDebug(COMPLETION) << "Could not get internal context from" << declaration->toString(); } else { //Print some debug-output qCDebug(COMPLETION) << "Could not get declaration"; } } return ret; } QList CodeCompletionContext::completionItems(bool& abort, bool fullCompletion) { //TODO: how should this be handled? Q_UNUSED(fullCompletion) /// Indexed string for 'Php', identifies environment files from this language plugin static const IndexedString phpLangString("Php"); LOCKDUCHAIN; QList items; if (!m_duContext) return items; typedef QPair DeclarationDepthPair; if ( memberAccessOperation() == FileChoose ) { if ( !ICore::self() ) { // in unit tests we can't do anything qCDebug(COMPLETION) << "no core found"; return items; } // file completion const Path currentDocument(m_duContext->url().str()); Path path; Path base; if ( !m_isFileCompletionAfterDirname ) { path = Path(currentDocument.parent(), m_expression); base = path; if ( !m_expression.isEmpty() && !m_expression.endsWith('/') ) { base = base.parent(); } } else { if ( m_expression.startsWith('/') ) { path = Path(currentDocument.parent(), m_expression.mid(1)); } else { path = currentDocument.parent(); } base = path; if ( !m_expression.isEmpty() && !m_expression.endsWith('/') && m_expression != QLatin1String("/") ) { base = base.parent(); } } QList addedPaths; bool addedParentDir = false; const QUrl baseUrl = base.toUrl(); foreach ( ProjectBaseItem* item, ICore::self()->projectController()->projectModel()->itemsForPath(IndexedString(base.toUrl())) ) { if ( abort || !item->folder() ) { break; } auto folder = item->folder(); foreach ( ProjectFileItem* subFile, folder->fileList() ) { if ( abort ) { break; } if ( addedPaths.contains(subFile->path()) ) { continue; } else { addedPaths << subFile->path(); } IncludeItem item; item.isDirectory = false; item.basePath = baseUrl; item.name = subFile->fileName(); if ( m_isFileCompletionAfterDirname && !m_expression.startsWith('/') ) { item.name.prepend('/'); } items << CompletionTreeItemPointer(new IncludeFileItem(item)); } foreach ( ProjectFolderItem* subFolder, folder->folderList() ) { if ( abort ) { break; } if ( addedPaths.contains(subFolder->path()) ) { continue; } else { addedPaths << subFolder->path(); } IncludeItem item; item.isDirectory = true; item.basePath = baseUrl; item.name = subFolder->folderName(); if ( m_isFileCompletionAfterDirname && !m_expression.startsWith('/') ) { item.name.prepend('/'); } items << CompletionTreeItemPointer(new IncludeFileItem(item)); } if ( !folder->parent() && !addedParentDir && m_expression.isEmpty() ) { // expect a parent dir IncludeItem item; item.isDirectory = true; item.basePath = baseUrl; item.name = QStringLiteral(".."); items << CompletionTreeItemPointer(new IncludeFileItem(item)); } } return items; } else if (memberAccessOperation() == ClassMemberChoose) { // get current class if (ClassDeclaration * currentClass = dynamic_cast(m_duContext->owner())) { // whether we want to show a list of overloadable functions // i.e. not after we have typed one of the keywords var,const or abstract bool showOverloadable = true; // whether we want to remove static functions from the overloadable list // i.e. after we have typed "public function" bool filterStatic = false; // whether we want to remove non-static functions from the overloadable list // i.e. after we have typed "public static function" bool filterNonStatic = false; // private functions are always removed from the overloadable list // but when we type "protected function" only protected functions may be shown bool filterPublic = false; { // add typical keywords for class member definitions QStringList modifiers = getMethodTokens(m_text); // don't add keywords when "function" was already typed bool addKeywords = !modifiers.contains(QStringLiteral("function")); if (currentClass->classModifier() == ClassDeclarationData::Abstract) { // abstract is only allowed in abstract classes if (modifiers.contains(QStringLiteral("abstract"))) { // don't show overloadable functions when we are defining an abstract function showOverloadable = false; } else if (addKeywords) { ADD_KEYWORD("abstract"); } } else { // final is only allowed in non-abstract classes if (addKeywords && !modifiers.contains(QStringLiteral("final"))) { ADD_KEYWORD("final"); } } if (modifiers.contains(QStringLiteral("private"))) { // overloadable functions must not be declared private showOverloadable = false; } else if (modifiers.contains(QStringLiteral("protected"))) { // only show protected overloadable methods filterPublic = true; } else if (addKeywords && !modifiers.contains(QStringLiteral("public"))) { ADD_KEYWORD("public"); ADD_KEYWORD("protected"); ADD_KEYWORD("private"); } if (modifiers.contains(QStringLiteral("static"))) { filterNonStatic = true; } else { if (addKeywords) { ADD_KEYWORD("static"); } else { filterStatic = true; } } if (addKeywords) { ADD_KEYWORD("function"); } if (modifiers.isEmpty()) { // var and const may not have any modifiers ADD_KEYWORD("var"); ADD_KEYWORD("const"); } } ifDebug( qCDebug(COMPLETION) << "showOverloadable" << showOverloadable; ) // complete overloadable methods from parents if (showOverloadable) { // TODO: use m_duContext instead of ctx // overloadable choose is only possible inside classes which extend/implement others if (currentClass->baseClassesSize()) { DUContext* ctx = currentClass->internalContext(); if (!ctx) { qCDebug(COMPLETION) << "invalid class context"; return items; } QList alreadyImplemented; //TODO: use the token stream here as well //TODO: always add __construct, __destruct and maby other magic functions // get all visible declarations and add inherited to the completion items foreach(DeclarationDepthPair decl, ctx->allDeclarations(ctx->range().end, m_duContext->topContext(), false)) { ClassMemberDeclaration *member = dynamic_cast(decl.first); ClassFunctionDeclaration *classFunc = dynamic_cast(decl.first); if (member) { if (decl.second == 0) { // this function is already implemented alreadyImplemented << decl.first->indexedIdentifier().getIndex(); continue; } // skip already implemented functions if (alreadyImplemented.contains(decl.first->indexedIdentifier().getIndex())) { continue; } // skip non-static functions when requested if (filterNonStatic && !member->isStatic()) { continue; } // skip static functions when requested if (filterStatic && member->isStatic()) { continue; } // always skip private functions if (member->accessPolicy() == Declaration::Private) { continue; } // skip public functions when requested if (filterPublic && member->accessPolicy() == Declaration::Public) { // make sure no non-public base members are added alreadyImplemented << decl.first->indexedIdentifier().getIndex(); continue; } // skip final members if (classFunc && classFunc->isFinal()) { // make sure no non-final base members are added alreadyImplemented << decl.first->indexedIdentifier().getIndex(); continue; } // make sure we inherit or implement the base class of this member if (!member->context() || !member->context()->owner()) { qCDebug(COMPLETION) << "invalid parent context/owner:" << member->toString(); continue; } if (!currentClass->isPublicBaseClass(dynamic_cast(member->context()->owner()), m_duContext->topContext())) { continue; } ImplementationItem::HelperType itype; if (!member->isFunctionDeclaration()) { itype = ImplementationItem::OverrideVar; } else if (member->isAbstract()) { itype = ImplementationItem::Implement; } else { itype = ImplementationItem::Override; } ifDebug( qCDebug(COMPLETION) << "ImplementationItem" << itype; ) items << CompletionTreeItemPointer(new ImplementationItem(itype, DeclarationPointer(decl.first), Php::CodeCompletionContext::Ptr(this), decl.second)); // don't add identical items twice to the completion choices alreadyImplemented << decl.first->indexedIdentifier().getIndex(); } } } } else { qCDebug(COMPLETION) << "invalid owner declaration for overloadable completion"; } } } else if (m_memberAccessOperation == BackslashAccess || m_memberAccessOperation == NamespaceChoose) { DUContext* ctx = nullptr; if (m_namespace.isEmpty()) { ctx = m_duContext->topContext(); } else { foreach(Declaration* dec, m_duContext->topContext()->findDeclarations(m_namespace)) { if (dec->kind() == Declaration::Namespace) { ctx = dec->internalContext(); break; } } } if (!ctx) { qCDebug(COMPLETION) << "could not find namespace:" << m_namespace.toString(); return items; } foreach(Declaration* dec, ctx->localDeclarations()) { if (!isValidCompletionItem(dec)) { continue; } else { items << CompletionTreeItemPointer( new NormalDeclarationCompletionItem( DeclarationPointer(dec), Php::CodeCompletionContext::Ptr(this), depth() ) ); } } } else if (m_expressionResult.type()) { QList containers = memberAccessContainers(); qCDebug(COMPLETION) << "containers: " << containers.count(); if (!containers.isEmpty()) { // get the parent class when we are inside a method declaration ClassDeclaration* currentClass = nullptr; if (m_duContext->owner() && m_duContext->owner()->isFunctionDeclaration() && m_duContext->parentContext() && m_duContext->parentContext()->owner()) { currentClass = dynamic_cast(m_duContext->parentContext()->owner()); } foreach(DUContext* ctx, containers) { ClassDeclaration* accessedClass = dynamic_cast(ctx->owner()); if (abort) return items; foreach(DeclarationDepthPair decl, ctx->allDeclarations( ctx->range().end, m_duContext->topContext(), false)) { //If we have StaticMemberAccess, which means A::Bla, show only static members, //except if we're within a class that derives from the container ClassMemberDeclaration* classMember = dynamic_cast(decl.first); if (memberAccessOperation() != StaticMemberAccess) { if (classMember && classMember->isStatic()) continue; //Skip static class members when not doing static access } else { if (!classMember || !classMember->isStatic()) continue; //Skip static class members when not doing static access } // check access policy if (classMember && accessedClass) { // by default only show public declarations Declaration::AccessPolicy ap = Declaration::Public; if (currentClass) { // if we are inside a class, we might want to show protected or private members ClassDeclaration* memberClass = dynamic_cast(classMember->context()->owner()); if (memberClass) { if (currentClass == accessedClass) { if (currentClass == memberClass) { // we can show all members of the current class ap = Declaration::Private; } else if (currentClass->isPublicBaseClass(memberClass, m_duContext->topContext())) { // we can show all but private members of ancestors of the current class ap = Declaration::Protected; } } else if (currentClass->isPublicBaseClass(accessedClass, m_duContext->topContext()) && (accessedClass == memberClass || accessedClass->isPublicBaseClass(memberClass, m_duContext->topContext()))) { // we can show all but private members of ancestors of the current class ap = Declaration::Protected; } } } if (ap < classMember->accessPolicy()) { continue; } } if (!decl.first->identifier().isEmpty()) items << CompletionTreeItemPointer( new NormalDeclarationCompletionItem( DeclarationPointer( decl.first), Php::CodeCompletionContext::Ptr(this), decl.second ) ); } } } else { qCDebug(COMPLETION) << "setContext: no container-type"; } } else { //Show all visible declarations QSet existingIdentifiers; QList decls = m_duContext->allDeclarations( CursorInRevision::invalid(), m_duContext->topContext() ); qCDebug(COMPLETION) << "setContext: using all declarations visible:" << decls.size(); QListIterator i(decls); i.toBack(); while (i.hasPrevious()) { DeclarationDepthPair decl = i.previous(); Declaration* dec = decl.first; if (dec->kind() == Declaration::Instance) { // filter non-superglobal vars of other contexts if (dec->context() != m_duContext.data() && !m_duContext->imports(dec->context())) { VariableDeclaration* vDec = dynamic_cast(dec); if ( vDec && !vDec->isSuperglobal() ) { continue; } } if (existingIdentifiers.contains(dec->indexedIdentifier().getIndex())) continue; existingIdentifiers.insert(dec->indexedIdentifier().getIndex()); } if (abort) return items; if (!isValidCompletionItem(dec)) continue; items << CompletionTreeItemPointer( new NormalDeclarationCompletionItem( DeclarationPointer(dec), Php::CodeCompletionContext::Ptr(this), decl.second ) ); } foreach(QSet urlSets, completionFiles()) { foreach(const IndexedString &url, urlSets) { if (url == m_duContext->url()) { continue; } uint count = 0; const CodeModelItem* foundItems = nullptr; CodeModel::self().items(url, count, foundItems); for (uint i = 0; i < count; ++i) { CodeModelItem::Kind k = foundItems[i].kind; if (((k & CodeModelItem::Function) || (k & CodeModelItem::Variable)) && !(k & CodeModelItem::ClassMember)) { foreach(const ParsingEnvironmentFilePointer &env, DUChain::self()->allEnvironmentFiles(url)) { if (env->language() != phpLangString) continue; TopDUContext* top = env->topContext(); if(!top) continue; if (m_duContext->imports(top)) continue; QList decls = top->findDeclarations(foundItems[i].id); foreach(Declaration* decl, decls) { if (abort) return items; // we don't want to have class methods/properties, just normal functions // and other global stuff if ( decl->context() && decl->context()->type() == DUContext::Class ) { continue; } if (!isValidCompletionItem(decl)) continue; if ( VariableDeclaration* vDec = dynamic_cast(decl) ) { if ( !vDec->isSuperglobal() ) { continue; } } items << CompletionTreeItemPointer( new NormalDeclarationCompletionItem( DeclarationPointer(decl), Php::CodeCompletionContext::Ptr(this) ) ); } } } } } } foreach(QSet urlSets, completionFiles()) { foreach(const IndexedString &url, urlSets) { uint count = 0; const CompletionCodeModelItem* foundItems = nullptr; CompletionCodeModel::self().items(url, count, foundItems); for (uint i = 0; i < count; ++i) { if (abort) return items; if (m_memberAccessOperation == ExceptionChoose) { if (!(foundItems[i].kind & CompletionCodeModelItem::Exception)) continue; } auto files = DUChain::self()->allEnvironmentFiles(url); items.reserve(files.size()); foreach(const ParsingEnvironmentFilePointer &env, files) { Q_ASSERT(env->language() == phpLangString); items << CompletionTreeItemPointer ( new CodeModelCompletionItem(env, foundItems[i])); } } } } } ///Find all recursive function-calls that should be shown as call-tips CodeCompletionContext::Ptr parentContext(this); do { if (abort) return items; parentContext = parentContext->parentContext(); if (parentContext) { if (parentContext->memberAccessOperation() == CodeCompletionContext::FunctionCallAccess) { if (!parentContext->memberAccessContainer().allDeclarationIds().isEmpty()) { Declaration* decl = parentContext->memberAccessContainer().allDeclarationIds().first() .getDeclaration(m_duContext->topContext()); if (!isValidCompletionItem(decl)) { continue; } if ( !decl->isFunctionDeclaration() ) { if ( ClassDeclaration* classDec = dynamic_cast(decl) ) { // search for ctor decl = nullptr; foreach ( Declaration* dec, classDec->internalContext()->findDeclarations(Identifier("__construct")) ) { if ( dec->isFunctionDeclaration() ) { decl = dec; break; } } if ( !decl ) { foreach ( Declaration* dec, classDec->internalContext()->findDeclarations(classDec->identifier()) ) { if ( dec->isFunctionDeclaration() ) { decl = dec; break; } } } if ( !decl ) { continue; } } else if ( !decl->type() ) { qCDebug(COMPLETION) << "parent decl is neither function nor class nor closure, skipping" << decl->toString(); continue; } } items << CompletionTreeItemPointer( new NormalDeclarationCompletionItem( DeclarationPointer(decl), Php::CodeCompletionContext::Ptr(parentContext.data()) ) ); } } else { qCDebug(COMPLETION) << "parent-context has non function-call access type"; } } } while (parentContext); if ( m_memberAccessOperation == NoMemberAccess ) { ///TODO: function-like statements should just be handled as a function with declaration etc. /// e.g.: empty, eval, die, exit, isset, unset /// but _not_ echo, print, catch, include*, require* ///TODO: use user's style for indendation etc. ADD_KEYWORD2("abstract class", "abstract class %SELECT%NAME%ENDSELECT% {\n%INDENT%\n}\n"); ADD_KEYWORD2("final class", "final class %SELECT%NAME%ENDSELECT% {\n%INDENT%\n}\n"); ADD_KEYWORD2("class", "class %SELECT%NAME%ENDSELECT% {\n%INDENT%\n}\n"); ADD_KEYWORD2("interface", "interface %SELECT%NAME%ENDSELECT% {\n%INDENT%\n}\n"); ADD_KEYWORD2("array", "array(\n%INDENT%%CURSOR%\n)"); ADD_KEYWORD2("break", "break;\n"); ADD_KEYWORD2("case", "case %SELECT%CASE%ENDSELECT%:\n%INDENT%\n%INDENT%break;\n"); ADD_KEYWORD2("throw", "throw %CURSOR%;\n"); ADD_KEYWORD2("try", "try {\n%INDENT%%CURSOR%\n} catch() {\n$%INDENT%\n}\n"); ADD_KEYWORD2("catch", "catch(%CURSOR%) {\n%INDENT%\n}\n"); ADD_KEYWORD2("clone", "clone %CURSOR%;\n"); ADD_KEYWORD2("continue", "continue;\n"); ADD_KEYWORD2("declare", "declare(%CURSOR%);\n"); ADD_KEYWORD2("default", "default:\n%INDENT%%CURSOR%\n%INDENT%break;\n"); ADD_KEYWORD2("do", "do {\n%INDENT%%CURSOR%\n} while();\n"); ADD_KEYWORD2("echo", "echo %CURSOR%;\n"); ADD_KEYWORD2("else", "else {\n%INDENT%%CURSOR%\n}\n"); ADD_KEYWORD2("elseif", "elseif (%CURSOR%) {\n%INDENT%\n}\n"); ADD_KEYWORD2("endif", "endif;"); ADD_KEYWORD2("endforeach", "endforeach;"); ADD_KEYWORD2("endswitch", "endswitch;"); ADD_KEYWORD2("endwhile", "endwhile;"); ADD_KEYWORD2("endfor", "endfor;"); ADD_KEYWORD2("enddeclare", "enddeclare;"); ADD_KEYWORD2("empty", "empty(%CURSOR%)"); ADD_KEYWORD2("eval", "eval(%CURSOR%)"); ADD_KEYWORD2("die", "die(%CURSOR%);\n"); ADD_KEYWORD2("exit", "exit(%CURSOR%);\n"); ///TODO: only activate when after "class NAME " ADD_KEYWORD("extends"); ADD_KEYWORD("implements"); ADD_KEYWORD2("for", "for(%CURSOR%;;) {\n%INDENT%\n}\n"); ADD_KEYWORD2("foreach", "foreach(%CURSOR%) {\n%INDENT%\n}\n"); ADD_KEYWORD2("function", "function %SELECT%NAME%ENDSELECT%() {\n%INDENT%\n}\n"); ADD_KEYWORD2("global", "global $%CURSOR%;"); ADD_KEYWORD2("if", "if (%CURSOR%) {\n%INDENT%\n}\n"); ADD_KEYWORD2("include", "include '%CURSOR%';\n"); ADD_KEYWORD2("include_once", "include_once '%CURSOR%';\n"); ADD_KEYWORD2("require", "require '%CURSOR%';\n"); ADD_KEYWORD2("require_once", "require_once '%CURSOR%';\n"); ADD_KEYWORD2("isset", "isset(%CURSOR%)"); ADD_KEYWORD2("list", "list(%CURSOR%)"); ADD_KEYWORD2("print", "print %CURSOR%;\n"); ADD_KEYWORD2("return", "return %CURSOR%;\n"); ADD_KEYWORD2("static", "static $%CURSOR%%;\n"); ADD_KEYWORD2("unset", "unset(%CURSOR%);\n"); ADD_KEYWORD2("while", "while (%CURSOR%) {\n%INDENT%\n}\n"); ADD_KEYWORD2("switch", "switch (%CURSOR%) {\n%INDENT%\n}\n"); } return items; } inline bool CodeCompletionContext::isValidCompletionItem(Declaration* dec) { if ( !dec || dec->range().isEmpty() ) { // hack for included files return false; } if ( dec->kind() == Declaration::Type && dec->qualifiedIdentifier().isEmpty() ) { // filter closures return false; } static DUChainPointer exceptionDecl; if (!exceptionDecl) { /// Qualified identifier for 'exception' static const KDevelop::QualifiedIdentifier exceptionQId(QStringLiteral("exception")); QList decs = dec->context()->findDeclarations(exceptionQId); Q_ASSERT(decs.count()); if (!decs.isEmpty()) { // additional safe-guard, see e.g. https://bugs.kde.org/show_bug.cgi?id=294218 exceptionDecl = dynamic_cast(decs.first()); Q_ASSERT(exceptionDecl); } } if (!exceptionDecl) { // safe-guard, see: https://bugs.kde.org/show_bug.cgi?id=294218 qWarning() << "could not find PHP-Exception declaration, related code completion will be broken."; } if (m_memberAccessOperation == ExceptionChoose || m_memberAccessOperation == NewClassChoose || m_memberAccessOperation == InterfaceChoose || m_memberAccessOperation == ClassExtendsChoose || m_memberAccessOperation == InstanceOfChoose) { // filter current class if (!m_forbiddenIdentifiers.isEmpty() && m_forbiddenIdentifiers.contains(dec->qualifiedIdentifier().index())) { return false; } ClassDeclaration* classDec = dynamic_cast(dec); // filter non-classes if (!classDec) { return false; } // show non-interface and non-abstract else if (m_memberAccessOperation == NewClassChoose) { return !(classDec->classModifier() & ClassDeclarationData::Abstract) && classDec->classType() == ClassDeclarationData::Class; } // filter non-exception classes else if (m_memberAccessOperation == ExceptionChoose) { if (!exceptionDecl) { // safe-guard, see: https://bugs.kde.org/show_bug.cgi?id=294218 return false; } return classDec->equalQualifiedIdentifier(exceptionDecl.data()) || classDec->isPublicBaseClass(exceptionDecl.data(), m_duContext->topContext()); } // show interfaces else if (m_memberAccessOperation == InterfaceChoose) { return classDec->classType() == ClassDeclarationData::Interface; } // show anything but final classes and interfaces else if (m_memberAccessOperation == ClassExtendsChoose) { return !(classDec->classModifier() & ClassDeclarationData::Final) && classDec->classType() == ClassDeclarationData::Class; } else if (m_memberAccessOperation == InstanceOfChoose) { return true; } } if (m_memberAccessOperation == ExceptionInstanceChoose) { if (!exceptionDecl) { // safe-guard, see: https://bugs.kde.org/show_bug.cgi?id=294218 return false; } if (dec->kind() != Declaration::Instance) { return false; } StructureType::Ptr structType = dec->type(); if (!structType) { return false; } ClassDeclaration* classDec = dynamic_cast(structType->declaration(dec->topContext())); if (!classDec) { return false; } return classDec->isPublicBaseClass(exceptionDecl.data(), m_duContext->topContext()); } if (m_memberAccessOperation == NoMemberAccess) { // filter private methods and class members when doing a global completion // when we are inside a class function, don't filter the private stuff // of the current class // NOTE: ClassFunctionDeclaration inherits ClassMemberDeclaration // NOTE: both have to have a parent context with type class if ( dec->context() && dec->context()->type() == DUContext::Class && m_duContext->parentContext() != dec->context() ) { if ( ClassMemberDeclaration* memberDec = dynamic_cast(dec) ) { if ( memberDec->accessPolicy() == Declaration::Private ) { return false; } } } if ( !dec->isFunctionDeclaration() && m_duContext.data() == dec->context() && m_position < dec->range().start ) { return false; } } if (m_memberAccessOperation == NamespaceChoose) { return dec->kind() == Declaration::Namespace; } return true; } QList > CodeCompletionContext::completionFiles() { QList > ret; if (ICore::self()) { auto projects = ICore::self()->projectController()->projects(); ret.reserve(projects.size()); foreach(IProject* project, projects) { ret << project->fileSet(); } } return ret; } } diff --git a/parser/php.g b/parser/php.g index 71a3db4..9914d23 100644 --- a/parser/php.g +++ b/parser/php.g @@ -1,1220 +1,1237 @@ ------------------------------------------------------------------------------- -- Copyright (c) 2008 Niko Sams -- -- This grammar is free software; you can redistribute it and/or -- modify it under the terms of the GNU Library General Public -- License as published by the Free Software Foundation; either -- version 2 of the License, or (at your option) any later version. -- -- This grammar is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -- Lesser General Public License for more details. -- -- You should have received a copy of the GNU Library General Public License -- along with this library; see the file COPYING.LIB. If not, write to -- the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- Boston, MA 02110-1301, USA. ----------------------------------------------------------- ----------------------------------------------------------- -- Grammar for PHP 5.2 -- Modelled after the Zend Grammar shipped with PHP5.2 -- source, the PHP Language Reference documentation, -- and parts taken from KDevelop Java Grammar ----------------------------------------------------------- -- 4 first/first conflicts: -- - var_expression: variable vs. varExpressionNormal -- no problem because of ifs that allow always just one rule -- - classNameReference: STRING vs. staticMember (foo vs. foo::$bar) -- resolved by LA() -- - encapsVar: STRING_VARNAME LBRACKET vs. expr (expr allows STRING_VARNAME too - but not LBRACKET) -- resolved by LA() -- - constantOrClassConst: constant vs. class constant (FOO v.s Cls::FOO) -- resolved by LA() (could be avoided, but the Ast is much cleaner that way) -- 1 first/follow conflicts: -- - elseifList: dangling-else conflict - should be ok -- TODO: (post 1.0.0 release) -- 1) decrease memory consumption -- 1.1) use quint32 instead of qint64 for end/start tokens -- 1.2) investigate whether using a map/hash for the ducontext member of all -- ast nodes gives a significant memory decrease while not hampering performance -- 1.3) investigate how unions could be used for exclusive AST node members -- 1.4) see whether we can always use the expression lists instead of both -- single member pointer and list of members, esp. in expressions -- 2) better cope with invalid code, have at least a partial AST -- 3) investigate whether expanding the visitor lookup to a -- (albeit huge) switch() in KDev-PG-Qt gives a significant performance gain -- I have the gut feeling that the current lookup takes unnecessary much time -- ------------------------------------------------------------ -- Forward declaration in phpast.h ------------------------------------------------------------ [: #include namespace KDevelop { class DUContext; } :] ------------------------------------------------------------ -- Additional includes for the parser ------------------------------------------------------------ %parser_declaration_header "tokenstream.h" %parser_declaration_header "QtCore/QString" %parser_declaration_header "language/duchain/problem.h" %parser_declaration_header "phplexer.h" %parser_bits_header "parserdebug.h" ------------------------------------------------------------ -- Export macro to use the parser in a shared lib ------------------------------------------------------------ %export_macro "KDEVPHPPARSER_EXPORT" %export_macro_header "parserexport.h" ------------------------------------------------------------ -- Enumeration types for additional AST members, -- in the global "Php" namespace ------------------------------------------------------------ %namespace [: class Lexer; enum ModifierFlags { ModifierPrivate = 1, ModifierPublic = 1 << 1, ModifierProtected = 1 << 2, ModifierStatic = 1 << 3, ModifierFinal = 1 << 4, ModifierAbstract = 1 << 5 }; enum ClassModifier { NormalClass, AbstractClass, FinalClass }; enum ScalarTypes { ScalarTypeInt, ScalarTypeFloat, ScalarTypeString }; enum CastType { CastInt, CastDouble, CastString, CastArray, CastObject, CastBool, CastUnset }; enum OperationType { OperationPlus = 1, OperationMinus, OperationConcat, OperationMul, OperationDiv, OperationExp, OperationMod, OperationAnd, OperationOr, OperationXor, OperationSl, OperationSr }; :] ------------------------------------------------------------ -- Ast Node class members ------------------------------------------------------------ %ast_extra_members [: KDevelop::DUContext* ducontext; :] ------------------------------------------------------------ -- Parser class members ------------------------------------------------------------ %parserclass (public declaration) [: /** * Transform the raw input into tokens. * When this method returns, the parser's token stream has been filled * and any parse*() method can be called. */ void tokenize(const QString& contents, int initialState = Lexer::HtmlState); enum ProblemType { Error, Warning, Info, Todo }; KDevelop::ProblemPointer reportProblem( Parser::ProblemType type, const QString& message, int tokenOffset = -1 ); QList problems() { return m_problems; } QString tokenText(qint64 begin, qint64 end); void setDebug(bool debug); void setCurrentDocument(KDevelop::IndexedString url); void setTodoMarkers(const QStringList& markers); void extractTodosFromComment(const QString& comment, qint64 offset); enum InitialLexerState { HtmlState = 0, DefaultState = 1 }; :] %parserclass (private declaration) [: enum VarExpressionState { Normal, OnlyVariable, OnlyNewObject }; QString m_contents; bool m_debug; KDevelop::IndexedString m_currentDocument; QList m_problems; struct ParserState { VarExpressionState varExpressionState; bool varExpressionIsVariable; }; ParserState m_state; QRegularExpression m_todoMarkers; :] %parserclass (constructor) [: m_state.varExpressionState = Normal; m_state.varExpressionIsVariable = false; :] %token_stream TokenStream ;; ----------------------------------------------------------- -- List of defined tokens ----------------------------------------------------------- -- keywords: %token ABSTRACT ("abstract"), BREAK ("break"), CASE ("case"), CATCH ("catch"), CLASS ("class"), CONST ("const"), CONTINUE ("continue"), DEFAULT ("default"), DO ("do"), ELSE ("else"), EXTENDS ("extends"), FINAL ("final"), FOR ("for"), IF ("if"), IMPLEMENTS ("implements"), INSTANCEOF ("instanceof"), INTERFACE ("interface"), NEW ("new"), PRIVATE ("private"), PROTECTED ("protected"), PUBLIC ("public"), RETURN ("return"), STATIC ("static"), SWITCH ("switch"), THROW ("throw"), TRY ("try"), WHILE ("while"), ECHO ("echo"), PRINT ("print"), FINALLY ("finally"), CLONE ("clone"), EXIT ("exit"), ELSEIF ("elseif"), ENDIF ("endif"), ENDWHILE ("endwhile"), ENDFOR ("endfor"), FOREACH ("foreach"), ENDFOREACH ("endforeach"), DECLARE ("declare"), ENDDECLARE ("enddeclare"), AS ("as"), ENDSWITCH ("endswitch"), FUNCTION ("function"), USE ("use"), GLOBAL ("global"), VAR ("var "), UNSET ("unset"), ISSET ("isset"), EMPTY ("empty"), HALT_COMPILER ("halt compiler"), DOUBLE_ARROW ("=>"), LIST ("list"), ARRAY ("array"), CLASS_C ("__CLASS__"), METHOD_C ("__METHOD__"), FUNC_C ("__FUNCTION__"), LINE ("__LINE__"), FILE ("__FILE__"), COMMENT ("comment"), DOC_COMMENT ("doc comment"), PAAMAYIM_NEKUDOTAYIM ("::"), INCLUDE ("include"), INCLUDE_ONCE ("include_once"), EVAL ("eval"), REQUIRE ("require"), REQUIRE_ONCE ("require_once"), NAMESPACE ("namespace"), NAMESPACE_C("__NAMESPACE__"), USE("use"), GOTO ("goto"), TRAIT ("trait"), INSTEADOF ("insteadof"), CALLABLE ("callable"), - ITERABLE ("iterable"), BOOL ("bool"), FLOAT ("float"), INT ("int"), STRING_TYPE ("string") ;; + ITERABLE ("iterable"), BOOL ("bool"), FLOAT ("float"), INT ("int"), STRING_TYPE ("string"), + VOID ("void") ;; -- casts: %token INT_CAST ("int cast"), DOUBLE_CAST ("double cast"), STRING_CAST ("string cast"), ARRAY_CAST ("array cast"), OBJECT_CAST ("object cast"), BOOL_CAST ("bool cast"), UNSET_CAST ("unset cast") ;; -- seperators: %token SEMICOLON (";"), DOUBLE_QUOTE ("\""), LBRACKET ("["), RBRACKET ("]"), LPAREN ("("), RPAREN (")"), LBRACE ("{"), RBRACE ("}"), COMMA (","), AT ("@"), CURLY_OPEN ("curly open"), -- { in "{$foo}"; not the same as LBRACE DOLLAR_OPEN_CURLY_BRACES ("${"), START_HEREDOC ("start heredoc"), END_HEREDOC ("end heredoc"), BACKTICK ("`"), BACKSLASH ("\\"), START_NOWDOC("start nowdoc"), END_NOWDOC("end nowdoc") ;; -- operators: %token IS_EQUAL ("=="), IS_NOT_EQUAL ("!="), IS_IDENTICAL ("==="), IS_NOT_IDENTICAL ("!=="), IS_SMALLER ("<"), IS_GREATER (">"), IS_SMALLER_OR_EQUAL ("<="), IS_GREATER_OR_EQUAL (">="), BOOLEAN_OR ("||"), BOOLEAN_AND ("&&"), ASSIGN ("="), EXP_ASSIGN("**="), PLUS_ASSIGN ("+="), MINUS_ASSIGN ("-="), MUL_ASSIGN ("*="), DIV_ASSIGN ("/="), CONCAT_ASSIGN (".="), MOD_ASSIGN ("%="), AND_ASSIGN ("&="), OR_ASSIGN ("|="), XOR_ASSIGN ("^="), SL_ASSIGN ("<<="), SR_ASSIGN (">>="), OBJECT_OPERATOR ("->"), PLUS ("+"), MINUS("-"), CONCAT("."), INC ("++"), DEC ("--"), BANG ("!"), QUESTION ("?"), COLON (":"), BIT_AND ("&"), BIT_OR("|"), BIT_XOR ("^"), SL ("<<"), SR (">>"), MUL("*"), DIV("/"), MOD ("%"), TILDE ("~"), DOLLAR ("$"), EXP ("**"), ELLIPSIS ("..."), LOGICAL_OR ("logical or"), LOGICAL_AND ("logical and"), LOGICAL_XOR ("logical xor") ;; -- literals and identifiers: %token INLINE_HTML ("inline html"), WHITESPACE ("whitespace"), CONSTANT_ENCAPSED_STRING ("constant encapsed string"), VARIABLE ("variable"), ENCAPSED_AND_WHITESPACE ("encapsed and whitespace"), DNUMBER ("double number"), LNUMBER ("long number"), NUM_STRING ("num string"), STRING ("string"), STRING_VARNAME ("string varname") ;; -- when in "${varname}" -- open/close tags %token OPEN_TAG (""), OPEN_TAG_WITH_ECHO (" start ;; namespaceDeclaration=namespaceDeclarationStatement | statement=topStatement -> outerTopStatement ;; -- first/first conflict for FUNCTION (?[: (LA(1).kind == Token_FUNCTION && ((LA(2).kind == Token_BIT_AND && LA(3).kind == Token_LPAREN) || LA(2).kind == Token_LPAREN)) || LA(1).kind != Token_FUNCTION :] statement=statement ) | functionDeclaration=functionDeclarationStatement | classDeclaration=classDeclarationStatement | traitDeclaration=traitDeclarationStatement | interfaceDeclaration=interfaceDeclarationStatement | HALT_COMPILER LPAREN RPAREN SEMICOLON -- Lexer stops allready -> topStatement ;; [: bool reported = false; while ( true ) { :] try/recover(#statements=topStatement)* [: if (yytoken != Token_RBRACE && yytoken != Token_EOF && yytoken != Token_CLOSE_TAG && yytoken != Token_ELSEIF && yytoken != Token_ELSE && yytoken != Token_ENDIF && yytoken != Token_ENDFOREACH && yytoken != Token_ENDFOR && yytoken != Token_ENDWHILE && yytoken != Token_ENDSWITCH && yytoken != Token_ENDDECLARE && yytoken != Token_CASE && yytoken != Token_DEFAULT) { if (!reported) { qint64 index = tokenStream->index() - 1; Token &token = tokenStream->at(index); QString tokenValue = token.kind != 0 ? tokenText(token.begin, token.end) : QStringLiteral("EOF"); reportProblem(Error, QStringLiteral("Unexpected token \"%1\".").arg(tokenValue)); reported = true; } yylex(); } else { break; } } :] -> innerStatementList ;; --Operator Precedence, from PHP Manual --left or --left xor --left and --right print --right = += -= *= /= .= %= &= |= ^= <<= >>= assignment --left ? : ternary --left || logical --left && logical --left | bitwise --left ^ bitwise --left & bitwise and references --non-associative == != === !== comparison --non-associative < <= > >= comparison --left << >> bitwise --left + - . arithmetic and string --left * / % arithmetic --non-associative ! ~ - (int) (float) (string) (array) (object) @ types --non-associative ++ -- increment/decrement --left [ array() --non-associative new new expression=logicalOrExpression -> expr ;; #expression=logicalXorExpression @ LOGICAL_OR -> logicalOrExpression ;; #expression=logicalAndExpression @ LOGICAL_XOR -> logicalXorExpression ;; #expression=printExpression @ LOGICAL_AND -> logicalAndExpression ;; (print=PRINT*) expression=assignmentExpression -> printExpression ;; -- leftside must me a variable, we check afterwards if it was a variable and -- if not we report an error 0 --needed for line below [: m_state.varExpressionIsVariable = false; :] --reset flag expression=conditionalExpression ( assignmentExpressionEqual=assignmentExpressionEqual | ( ( PLUS_ASSIGN [: (*yynode)->operation = OperationPlus; :] | MINUS_ASSIGN [: (*yynode)->operation = OperationMinus; :] | MUL_ASSIGN [: (*yynode)->operation = OperationMul; :] | EXP_ASSIGN [: (*yynode)->operation = OperationExp; :] | DIV_ASSIGN [: (*yynode)->operation = OperationDiv; :] | CONCAT_ASSIGN [: (*yynode)->operation = OperationConcat; :] | MOD_ASSIGN [: (*yynode)->operation = OperationMod; :] | AND_ASSIGN [: (*yynode)->operation = OperationAnd; :] | OR_ASSIGN [: (*yynode)->operation = OperationOr; :] | XOR_ASSIGN [: (*yynode)->operation = OperationXor; :] | SL_ASSIGN [: (*yynode)->operation = OperationSl; :] | SR_ASSIGN [: (*yynode)->operation = OperationSr; :] ) assignmentExpressionCheckIfVariable assignmentExpression=assignmentExpression) | 0) -> assignmentExpression [ member variable operation: OperationType; ];; --=& is special: -- $foo =& $var; is allowed but not $foo =& 'static'; -- $foo =& new bar(); is allowed too but deprecated and reports a warning --we set a flag (varExpressionState) with that var_expression accepts only valid parts --this is done in such a strage way because we need the full expression to allow --things like $foo =& $bar || e(); ASSIGN assignmentExpressionCheckIfVariable --as in assignmentExpression (BIT_AND [: if (yytoken == Token_NEW) { reportProblem(Warning, QStringLiteral("=& new foo() is deprecated"), -2); m_state.varExpressionState = OnlyNewObject; } else { m_state.varExpressionState = OnlyVariable; }:] | 0) assignmentExpression=assignmentExpression [: m_state.varExpressionState = Normal; :] -> assignmentExpressionEqual ;; -- check if var_expression was a variable, if not report an error -- varExpressionIsVariable is set in var_expression 0 --to allow cpp-code [: if (!m_state.varExpressionIsVariable) { reportProblem(Error, QStringLiteral("Left side is not a variable")); return false; } :] -> assignmentExpressionCheckIfVariable ;; expression=booleanOrExpression ( QUESTION (ifExpression=expr|0) COLON elseExpression=conditionalExpression | 0 ) -> conditionalExpression ;; #expression=booleanAndExpression @ BOOLEAN_OR -> booleanOrExpression ;; #expression=bitOrExpression @ BOOLEAN_AND -> booleanAndExpression ;; #expression=bitXorExpression @ BIT_OR -> bitOrExpression ;; #expression=bitAndExpression @ BIT_XOR -> bitXorExpression ;; #expression=equalityExpression @ BIT_AND -> bitAndExpression ;; expression=relationalExpression (#additionalExpression=equalityExpressionRest)* -> equalityExpression ;; ( IS_EQUAL | IS_NOT_EQUAL | IS_IDENTICAL | IS_NOT_IDENTICAL ) expression=relationalExpression -> equalityExpressionRest ;; expression=shiftExpression ( (#additionalExpression=relationalExpressionRest)+ --instanceof as in java.g (correct??) | INSTANCEOF instanceofType=classNameReference | 0 ) -> relationalExpression ;; ( IS_SMALLER | IS_GREATER | IS_SMALLER_OR_EQUAL | IS_GREATER_OR_EQUAL ) expression=shiftExpression -> relationalExpressionRest ;; expression=additiveExpression (#additionalExpression=shiftExpressionRest)* -> shiftExpression ;; ( SL | SR ) expression=additiveExpression -> shiftExpressionRest ;; expression=multiplicativeExpression (#additionalExpression=additiveExpressionRest)* -> additiveExpression ;; ( PLUS [: (*yynode)->operation = OperationPlus; :] | MINUS [: (*yynode)->operation = OperationMinus; :] | CONCAT [: (*yynode)->operation = OperationConcat; :] ) expression=multiplicativeExpression -> additiveExpressionRest [ member variable operation: OperationType; ];; expression=unaryExpression (#additionalExpression=multiplicativeExpressionRest)* -> multiplicativeExpression ;; ( MUL [: (*yynode)->operation = OperationMul; :] | DIV [: (*yynode)->operation = OperationDiv; :] | EXP [: (*yynode)->operation = OperationExp; :] | MOD [: (*yynode)->operation = OperationMod; :] ) expression=unaryExpression -> multiplicativeExpressionRest [ member variable operation: OperationType; ];; ( MINUS unaryExpression=unaryExpression | PLUS unaryExpression=unaryExpression | BANG unaryExpression=unaryExpression | TILDE unaryExpression=unaryExpression | INT_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastInt; :] | DOUBLE_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastDouble; :] | STRING_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastString; :] | ARRAY_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastArray; :] | OBJECT_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastObject; :] | BOOL_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastBool; :] | UNSET_CAST unaryExpression=unaryExpression [: (*yynode)->castType = CastUnset; :] | AT unaryExpression=unaryExpression | LIST LPAREN assignmentList=assignmentList RPAREN ASSIGN unaryExpression=unaryExpression | EXIT (LPAREN (expression=expr | 0) RPAREN | 0) | EVAL LPAREN expression=expr RPAREN | INCLUDE includeExpression=unaryExpression | INCLUDE_ONCE includeExpression=unaryExpression | REQUIRE includeExpression=unaryExpression | REQUIRE_ONCE includeExpression=unaryExpression | unaryExpressionNotPlusminus=unaryExpressionNotPlusminus ) -> unaryExpression [ member variable castType: CastType; ];; (#prefixOperator=postprefixOperator)* varExpression=varExpression (#postfixOperator=postprefixOperator)* -> unaryExpressionNotPlusminus ;; op=INC | op=DEC -> postprefixOperator ;; --first/first conflict - no problem because of ifs ?[: m_state.varExpressionState == OnlyVariable :] 0 [: m_state.varExpressionState = Normal; :] variable=variable | ?[: m_state.varExpressionState == OnlyNewObject :] 0 [: m_state.varExpressionState = Normal; :] newObject=varExpressionNewObject | varExpressionNormal=varExpressionNormal | varExpressionArray=varExpressionArray arrayIndex=arrayIndexSpecifier* -> varExpression ;; (?[: LA(1).kind == Token_LPAREN && LA(2).kind == Token_FUNCTION && LA(3).kind == Token_LPAREN :] iife=iifeSyntax ) | LPAREN try/rollback (newObject=varExpressionNewObject RPAREN (#variableProperties=instantiationAccess*)) catch (expression=expr RPAREN) | BACKTICK encapsList=encapsList BACKTICK --try/rollback resolves conflict scalar vs. staticMember (foo::bar vs. foo::$bar) --varExpressionIsVariable flag is needed for assignmentExpression | try/rollback (variable=variable [: m_state.varExpressionIsVariable = true; :]) catch (scalar=scalar) | ISSET LPAREN (#issetVariable=variable @ COMMA) RPAREN | EMPTY LPAREN emptyVarialbe=variable RPAREN | newObject=varExpressionNewObject | CLONE cloneCar=varExpressionNormal | closure=closure -> varExpressionNormal ;; ARRAY LPAREN (#arrayValues=arrayPairValue -- break because array(1,) is allowed (solves FIRST/FOLLOW conflict) @ (COMMA [: if (yytoken == Token_RPAREN) { break; } :] ) | 0) RPAREN | LBRACKET (#arrayValues=arrayPairValue -- break because [1,] is allowed (solves FIRST/FOLLOW conflict) @ (COMMA [: if (yytoken == Token_RBRACKET) { break; } :] ) | 0) RBRACKET -> varExpressionArray ;; -- http://wiki.php.net/rfc/closures FUNCTION (isRef=BIT_AND|0) LPAREN parameters=parameterList RPAREN ( USE LPAREN lexicalVars=lexicalVarList RPAREN | 0) + ( COLON returnType=returnType | 0) LBRACE try/recover(functionBody=innerStatementList) RBRACE -> closure ;; LPAREN try/rollback (closure=closure RPAREN LPAREN parameterList=functionCallParameterList RPAREN) catch (expression=expr RPAREN) -> iifeSyntax ;; (#lexicalVars=lexicalVar @ COMMA) | 0 [: reportProblem(Error, QStringLiteral("Use list of closure must not be empty.")); :] -> lexicalVarList ;; (isRef=BIT_AND | 0) variable=variableIdentifier -> lexicalVar ;; NEW className=classNameReference ctor=ctorArguments -> varExpressionNewObject ;; LPAREN parameterList=functionCallParameterList RPAREN | 0 -> ctorArguments ;; #parameters=functionCallParameterListElement @ COMMA | 0 -> functionCallParameterList ;; (BIT_AND variable=variable) | (isVariadic=ELLIPSIS | 0) expr=expr -> functionCallParameterListElement ;; #element=assignmentListElement @COMMA -> assignmentList ;; variable=variable | LIST LPAREN assignmentList=assignmentList RPAREN | 0 -> assignmentListElement ;; expr=expr (DOUBLE_ARROW (exprValue=expr | BIT_AND varValue=variable) | 0) | BIT_AND variable=variable -> arrayPairValue ;; var=baseVariableWithFunctionCalls (#variableProperties=variableObjectProperty*) -> variable ;; OBJECT_OPERATOR | PAAMAYIM_NEKUDOTAYIM -> objectOperator ;; ( ?[: LA(1).kind == Token_DOLLAR:] LBRACE variable=variable RBRACE | objectProperty=objectProperty ) (isFunctionCall=LPAREN parameterList=functionCallParameterList RPAREN arrayIndex=arrayIndexSpecifier* | 0) -> variableProperty ;; objectOperator variableProperty=variableProperty -> variableObjectProperty ;; OBJECT_OPERATOR variableProperty=variableProperty -> instantiationAccess ;; --Conflict -- foo::$bar[0] (=baseVariable-staticMember) --vs.foo::$bar[0](); (=static function call) try/rollback (functionCall=functionCall arrayIndex=arrayIndexSpecifier*) catch (baseVariable=baseVariable) -> baseVariableWithFunctionCalls ;; LBRACKET (expr=expr | 0) RBRACKET -> arrayIndexSpecifier ;; LBRACKET (expr=expr) RBRACKET -> stringIndexSpecifier ;; stringFunctionNameOrClass=namespacedIdentifier ( LPAREN stringParameterList=functionCallParameterList RPAREN | PAAMAYIM_NEKUDOTAYIM ( stringFunctionName=identifier LPAREN stringParameterList=functionCallParameterList RPAREN | varFunctionName=variableWithoutObjects LPAREN stringParameterList=functionCallParameterList RPAREN | LBRACE (expr=expr) RBRACE LPAREN stringParameterList=functionCallParameterList RPAREN ) ) | varFunctionName=variableWithoutObjects LPAREN varParameterList=functionCallParameterList RPAREN -> functionCall ;; var=compoundVariableWithSimpleIndirectReference #offsetItems=dimListItem* | staticMember=staticMember -> baseVariable ;; variable=variableIdentifier | DOLLAR LBRACE expr=expr RBRACE -> compoundVariable ;; ( DOLLAR ( DOLLAR+ | 0 ) ( indirectVariable=variableIdentifier | LBRACE expr=expr RBRACE ) | variable=variableIdentifier ) -> compoundVariableWithSimpleIndirectReference ;; className=namespacedIdentifier PAAMAYIM_NEKUDOTAYIM variable=variableWithoutObjects -> staticMember ;; LBRACE try/recover(statements=innerStatementList) RBRACE | IF LPAREN ifExpr=expr RPAREN ( COLON statements=innerStatementList newElseifList newElseSingle ENDIF semicolonOrCloseTag | ifStatement=statement elseifList=elseifList elseSingle=elseSingle ) | WHILE LPAREN whileExpr=expr RPAREN whileStatement=whileStatement | FOR LPAREN forExpr1=forExpr SEMICOLON forExpr2=forExpr SEMICOLON forExpr3=forExpr RPAREN forStatement=forStatement | SWITCH LPAREN swtichExpr=expr RPAREN switchCaseList=switchCaseList | FOREACH LPAREN ( -- allow $var as &$i and not expr() as &$i try/rollback(foreachVar=variable AS foreachVarAsVar=foreachVariable) catch(foreachExpr=expr AS foreachExprAsVar=variable)) (DOUBLE_ARROW foreachVariable=foreachVariable | 0) RPAREN foreachStatement=foreachStatement | DECLARE LPAREN declareItem=declareItem @ COMMA RPAREN declareStatement | SEMICOLON -- empty statement | TRY LBRACE try/recover(statements=innerStatementList) RBRACE #catches=catchItem* (FINALLY LBRACE finallyBody=innerStatementList RBRACE | 0) | UNSET LPAREN #unsetVariables=variable @ COMMA RPAREN semicolonOrCloseTag -- fix first/follow with goto target | ( ?[: LA(1).kind != Token_STRING || LA(2).kind != Token_COLON :] expr=expr semicolonOrCloseTag ) | DO doStatement=statement WHILE LPAREN whileExpr=expr RPAREN semicolonOrCloseTag | BREAK (breakExpr=expr | 0) semicolonOrCloseTag | CONTINUE (continueExpr=expr | 0) semicolonOrCloseTag | RETURN (returnExpr=expr | 0) semicolonOrCloseTag | GLOBAL #globalVars=globalVar @ COMMA semicolonOrCloseTag | STATIC #staticVars=staticVar @ COMMA semicolonOrCloseTag | ECHO #echoExprs=expr @ COMMA semicolonOrCloseTag | THROW throwExpr=expr semicolonOrCloseTag -- throws error in zend parser, so ignored | USE use_filename semicolonOrCloseTag | CLOSE_TAG | OPEN_TAG | OPEN_TAG_WITH_ECHO expr=expr semicolonOrCloseTag | INLINE_HTML | CONST #consts=constantDeclaration @ COMMA SEMICOLON | USE #useNamespace=useNamespace @ COMMA SEMICOLON | GOTO gotoLabel=STRING SEMICOLON | gotoTarget=STRING COLON -> statement ;; identifier=namespacedIdentifier (AS aliasIdentifier=identifier | 0) -> useNamespace ;; identifier=identifier ASSIGN scalar=staticScalar -> constantDeclaration ;; SEMICOLON | CLOSE_TAG -> semicolonOrCloseTag ;; LBRACE (SEMICOLON | 0) try/recover(caseList=caseList) RBRACE | COLON (SEMICOLON | 0) caseList=caseList ENDSWITCH semicolonOrCloseTag -> switchCaseList ;; #caseItems=case_item* -> caseList ;; CASE expr=expr (COLON | SEMICOLON) statements=innerStatementList | def=DEFAULT (COLON | SEMICOLON) statements=innerStatementList -> case_item ;; CATCH LPAREN catchClass=namespacedIdentifier var=variableIdentifier RPAREN LBRACE try/recover(statements=innerStatementList) RBRACE -> catchItem ;; statement=statement | COLON statements=innerStatementList ENDDECLARE semicolonOrCloseTag -> declareStatement ;; STRING ASSIGN scalar=staticScalar -> declareItem ;; (BIT_AND | 0) variable=variable -> foreachVariable ;; statement=statement | COLON statements=innerStatementList ENDFOREACH semicolonOrCloseTag -> foreachStatement ;; var=variableIdentifier (ASSIGN value=staticScalar | 0) -> staticVar ;; var=variableIdentifier | DOLLAR (dollarVar=variable | LBRACE expr=expr RBRACE) -> globalVar ;; #exprs=expr @ COMMA | 0 -> forExpr ;; statement=statement | COLON statements=innerStatementList ENDFOR semicolonOrCloseTag -> forStatement ;; statement=statement | COLON statements=innerStatementList ENDWHILE semicolonOrCloseTag -> whileStatement ;; --first/follow conflict; todo check if this is a problem #elseifListItem=elseifListItem* -> elseifList ;; ELSEIF LPAREN expr=expr RPAREN statement=statement -> elseifListItem ;; ELSE statement=statement | 0 -> elseSingle ;; #newElseifListItem=newelseifListItem* -> newElseifList ;; ELSEIF LPAREN expr=expr RPAREN COLON statements=innerStatementList -> newelseifListItem ;; ELSE COLON statements=innerStatementList | 0 -> newElseSingle ;; --TODO --resolve STRING vs. staticMember conflict -- ?[: LA(2).kind != Token_PAAMAYIM_NEKUDOTAYIM :] identifier=namespacedIdentifier | staticIdentifier = STATIC | dynamicClassNameReference=dynamicClassNameReference -> classNameReference ;; baseVariable=baseVariable (OBJECT_OPERATOR objectProperty=objectProperty properties=dynamicClassNameVariableProperties | 0) -> dynamicClassNameReference ;; #properties=dynamicClassNameVariableProperty* -> dynamicClassNameVariableProperties ;; OBJECT_OPERATOR property=objectProperty -> dynamicClassNameVariableProperty ;; objectDimList=objectDimList | variableWithoutObjects=variableWithoutObjects -> objectProperty ;; variableName=variableName #offsetItems=dimListItem* -> objectDimList ;; variable=compoundVariableWithSimpleIndirectReference #offsetItems=dimListItem* -> variableWithoutObjects ;; arrayIndex=arrayIndexSpecifier | LBRACE expr=expr RBRACE -> dimListItem ;; name=identifier | LBRACE expr=expr RBRACE -> variableName ;; commonScalar=commonScalar | constantOrClassConst=constantOrClassConst | varname=STRING_VARNAME | DOUBLE_QUOTE encapsList=encapsList DOUBLE_QUOTE stringIndex=stringIndexSpecifier* | START_HEREDOC encapsList=encapsList END_HEREDOC -> scalar ;; constant=namespacedIdentifier ( PAAMAYIM_NEKUDOTAYIM classConstant=classConstant | 0 ) -> constantOrClassConst ;; CLASS | identifier -> classConstant ;; #encaps=encaps* -> encapsList ;; var=encapsVar | value=ENCAPSED_AND_WHITESPACE -> encaps ;; -- first/first conflict resolved by LA(2) --(expr allows STRING_VARNAME too - but without [expr]) DOLLAR_OPEN_CURLY_BRACES ( ?[: LA(2).kind == Token_LBRACKET:] STRING_VARNAME arrayIndex=arrayIndexSpecifier RBRACE | expr=expr RBRACE ) | variable=variableIdentifier (OBJECT_OPERATOR propertyIdentifier=identifier | LBRACKET offset=encapsVarOffset RBRACKET | 0) | CURLY_OPEN expr=expr RBRACE -> encapsVar ;; STRING | NUM_STRING | variableIdentifier -> encapsVarOffset ;; LNUMBER [: (*yynode)->scalarType = ScalarTypeInt; :] | DNUMBER [: (*yynode)->scalarType = ScalarTypeFloat; :] | string=CONSTANT_ENCAPSED_STRING [: (*yynode)->scalarType = ScalarTypeString; :] stringIndex=stringIndexSpecifier* | LINE [: (*yynode)->scalarType = ScalarTypeInt; :] | FILE [: (*yynode)->scalarType = ScalarTypeString; :] | CLASS_C [: (*yynode)->scalarType = ScalarTypeString; :] | METHOD_C [: (*yynode)->scalarType = ScalarTypeString; :] | FUNC_C [: (*yynode)->scalarType = ScalarTypeString; :] | NAMESPACE_C [: (*yynode)->scalarType = ScalarTypeString; :] | START_NOWDOC STRING END_NOWDOC [: (*yynode)->scalarType = ScalarTypeString; :] -> commonScalar [ member variable scalarType: ScalarTypes; ] ;; FUNCTION (BIT_AND | 0) functionName=identifier - LPAREN parameters=parameterList RPAREN LBRACE try/recover(functionBody=innerStatementList) RBRACE + LPAREN parameters=parameterList RPAREN (COLON returnType=returnType | 0) + LBRACE try/recover(functionBody=innerStatementList) RBRACE -> functionDeclarationStatement ;; (#parameters=parameter @ COMMA) | 0 -> parameterList ;; (parameterType=parameterType | 0) (isRef=BIT_AND | 0) (isVariadic=ELLIPSIS | 0) variable=variableIdentifier (ASSIGN defaultValue=staticScalar | 0) -> parameter ;; (isNullable=QUESTION | 0) ( objectType=namespacedIdentifier | arrayType=ARRAY | callableType=CALLABLE | iterableType=ITERABLE | boolType=BOOL | floatType=FLOAT | intType=INT | stringType=STRING_TYPE ) -> parameterType ;; + (isNullable=QUESTION | 0) ( + objectType=namespacedIdentifier + | arrayType=ARRAY + | callableType=CALLABLE + | iterableType=ITERABLE + | boolType=BOOL + | floatType=FLOAT + | intType=INT + | stringType=STRING_TYPE + | voidType=VOID + ) +-> returnType ;; + value=commonScalar | constantOrClassConst=constantOrClassConst | PLUS plusValue=staticScalar | MINUS minusValue=staticScalar | array=ARRAY LPAREN (#arrayValues=staticArrayPairValue -- break because array(1,) is allowed @ (COMMA [: if (yytoken == Token_RPAREN) { break; } :] ) | 0) RPAREN | array=LBRACKET (#arrayValues=staticArrayPairValue -- break because [1,] is allowed @ (COMMA [: if (yytoken == Token_RBRACKET) { break; } :] ) | 0) RBRACKET -> staticScalar ;; #val1=staticScalar (DOUBLE_ARROW #val2=staticScalar | 0) -> staticArrayPairValue ;; (isGlobal=BACKSLASH | 0) #namespaceName=identifier+ @ BACKSLASH -> namespacedIdentifier ;; string=STRING -> identifier ;; variable=VARIABLE -> variableIdentifier ;; NAMESPACE #namespaceName=identifier* @ BACKSLASH ( -- the semicolon case needs at least one namespace identifier, the {...} case not... SEMICOLON [: if (!(*yynode)->namespaceNameSequence) { reportProblem(Error, QStringLiteral("Missing namespace identifier."), -2); } :] | LBRACE try/recover(body=innerStatementList) RBRACE ) -> namespaceDeclarationStatement ;; INTERFACE interfaceName=identifier (EXTENDS extends=classImplements | 0) LBRACE try/recover(body=classBody) RBRACE -> interfaceDeclarationStatement ;; TRAIT traitName=identifier LBRACE body=classBody RBRACE -> traitDeclarationStatement ;; modifier=optionalClassModifier CLASS className=identifier (EXTENDS extends=classExtends | 0) (IMPLEMENTS implements=classImplements | 0) LBRACE body=classBody RBRACE -> classDeclarationStatement ;; identifier=namespacedIdentifier -> classExtends ;; #implements=namespacedIdentifier @ COMMA -> classImplements ;; -- error recovery, to understand it you probably have to look at the generated code ;-) [: bool reported = false; while ( true ) { :] try/recover(#classStatements=classStatement)* [: if (yytoken != Token_RBRACE && yytoken != Token_EOF && yytoken != Token_CLOSE_TAG) { if (!reported) { reportProblem(Error, QStringLiteral("Unexpected token in class context.")); reported = true; } yylex(); } else { break; } } :] RBRACE [: rewind(tokenStream->index() - 2); :] -> classBody ;; CONST #consts=constantDeclaration @ COMMA SEMICOLON | VAR variable=classVariableDeclaration SEMICOLON | modifiers=optionalModifiers ( variable=classVariableDeclaration SEMICOLON | FUNCTION (BIT_AND | 0) methodName=identifier LPAREN parameters=parameterList RPAREN + ( COLON returnType=returnType | 0) methodBody=methodBody ) | USE #traits=namespacedIdentifier @ COMMA (imports=traitAliasDeclaration|SEMICOLON) -> classStatement ;; LBRACE #statements=traitAliasStatement @ (SEMICOLON [: if (yytoken == Token_RBRACE) { break; } :]) RBRACE -> traitAliasDeclaration ;; importIdentifier=traitAliasIdentifier (AS (modifiers=optionalModifiers | 0) aliasIdentifier=identifier|INSTEADOF #conflictIdentifier=namespacedIdentifier @ COMMA) -> traitAliasStatement ;; identifier=namespacedIdentifier PAAMAYIM_NEKUDOTAYIM methodIdentifier=identifier -> traitAliasIdentifier ;; SEMICOLON -- abstract method | LBRACE try/recover(statements=innerStatementList) RBRACE -> methodBody ;; #vars=classVariable @ COMMA -> classVariableDeclaration ;; variable=variableIdentifier (ASSIGN value=staticScalar | 0) -> classVariable ;; ( PUBLIC [: (*yynode)->modifiers |= ModifierPublic; :] | PROTECTED [: (*yynode)->modifiers |= ModifierProtected; :] | PRIVATE [: (*yynode)->modifiers |= ModifierPrivate; :] | STATIC [: (*yynode)->modifiers |= ModifierStatic; :] | ABSTRACT [: (*yynode)->modifiers |= ModifierAbstract; :] | FINAL [: (*yynode)->modifiers |= ModifierFinal; :] | 0 )* -> optionalModifiers[ member variable modifiers: unsigned int; ] ;; ( ABSTRACT [: (*yynode)->modifier = AbstractClass; :] | FINAL [: (*yynode)->modifier = FinalClass; :] | 0 ) -> optionalClassModifier[ member variable modifier: ClassModifier; ] ;; ----------------------------------------------------------------- -- Code segments copied to the implementation (.cpp) file. -- If existent, kdevelop-pg's current syntax requires this block -- to occur at the end of the file. ----------------------------------------------------------------- [: #include #include namespace Php { void Parser::tokenize(const QString& contents, int initialState) { m_contents = contents; Lexer lexer(tokenStream, contents, initialState); int kind = Parser::Token_EOF; int lastDocCommentBegin; int lastDocCommentEnd; do { lastDocCommentBegin = 0; lastDocCommentEnd = 0; kind = lexer.nextTokenKind(); while (kind == Parser::Token_WHITESPACE || kind == Parser::Token_COMMENT || kind == Parser::Token_DOC_COMMENT) { if (kind == Parser::Token_COMMENT || kind == Parser::Token_DOC_COMMENT) { extractTodosFromComment(tokenText(lexer.tokenBegin(), lexer.tokenEnd()), lexer.tokenBegin()); } if (kind == Parser::Token_DOC_COMMENT) { lastDocCommentBegin = lexer.tokenBegin(); lastDocCommentEnd = lexer.tokenEnd(); } kind = lexer.nextTokenKind(); } if ( !kind ) // when the lexer returns 0, the end of file is reached { kind = Parser::Token_EOF; } Parser::Token &t = tokenStream->push(); t.begin = lexer.tokenBegin(); t.end = lexer.tokenEnd(); t.kind = kind; t.docCommentBegin = lastDocCommentBegin; t.docCommentEnd = lastDocCommentEnd; //if ( m_debug ) qDebug() << kind << tokenText(t.begin,t.end) << t.begin << t.end; } while ( kind != Parser::Token_EOF ); yylex(); // produce the look ahead token } void Parser::extractTodosFromComment(const QString& comment, qint64 startPosition) { auto i = m_todoMarkers.globalMatch(comment); while (i.hasNext()) { auto match = i.next(); auto p = reportProblem(Todo, match.captured(1), 0); if (!p) { continue; } qint64 line = 0; qint64 column = 0; tokenStream->locationTable()->positionAt(startPosition, &line, &column); auto location = p->finalLocation(); location.setStart(KTextEditor::Cursor(line, column + match.capturedStart(1))); location.setEnd(KTextEditor::Cursor(line, column + match.capturedEnd(1))); p->setFinalLocation(location); }; } void Parser::setTodoMarkers(const QStringList& markers) { QString pattern = QStringLiteral("^(?:[/\\*\\s]*)(.*(?:"); bool first = true; foreach(const QString& marker, markers) { if (!first) { pattern += '|'; } pattern += QRegularExpression::escape(marker); first = false; } pattern += QStringLiteral(").*?)(?:[/\\*\\s]*)$"); m_todoMarkers.setPatternOptions(QRegularExpression::MultilineOption); m_todoMarkers.setPattern(pattern); } QString Parser::tokenText(qint64 begin, qint64 end) { return m_contents.mid(begin,end-begin+1); } KDevelop::ProblemPointer Parser::reportProblem( Parser::ProblemType type, const QString& message, int offset ) { qint64 sLine; qint64 sCol; qint64 index = tokenStream->index() + offset; if (index >= tokenStream->size()) { return {}; } tokenStream->startPosition(index, &sLine, &sCol); qint64 eLine; qint64 eCol; tokenStream->endPosition(index, &eLine, &eCol); auto p = KDevelop::ProblemPointer(new KDevelop::Problem()); p->setSource(KDevelop::IProblem::Parser); switch ( type ) { case Error: p->setSeverity(KDevelop::IProblem::Error); break; case Warning: p->setSeverity(KDevelop::IProblem::Warning); break; case Info: p->setSeverity(KDevelop::IProblem::Hint); break; case Todo: p->setSeverity(KDevelop::IProblem::Hint); p->setSource(KDevelop::IProblem::ToDo); break; } p->setDescription(message); KTextEditor::Range range(sLine, sCol, eLine, eCol + 1); p->setFinalLocation(KDevelop::DocumentRange(m_currentDocument, range)); m_problems << p; return p; } // custom error recovery void Parser::expectedToken(int /*expected*/, qint64 /*where*/, const QString& name) { reportProblem( Parser::Error, QStringLiteral("Expected token \"%1\"").arg(name)); } void Parser::expectedSymbol(int /*expectedSymbol*/, const QString& name) { qint64 line; qint64 col; qint64 index = tokenStream->index()-1; Token &token = tokenStream->at(index); qCDebug(PARSER) << "token starts at:" << token.begin; qCDebug(PARSER) << "index is:" << index; tokenStream->startPosition(index, &line, &col); QString tokenValue = tokenText(token.begin, token.end); qint64 eLine; qint64 eCol; tokenStream->endPosition(index, &eLine, &eCol); reportProblem( Parser::Error, QStringLiteral("Expected symbol \"%1\" (current token: \"%2\" [%3] at %4:%5 - %6:%7)") .arg(name, token.kind != 0 ? tokenValue : QStringLiteral("EOF")) .arg(token.kind) .arg(line) .arg(col) .arg(eLine) .arg(eCol)); } void Parser::setDebug( bool debug ) { m_debug = debug; } void Parser::setCurrentDocument(KDevelop::IndexedString url) { m_currentDocument = url; } Parser::ParserState *Parser::copyCurrentState() { ParserState *state = new ParserState(); state->varExpressionState = m_state.varExpressionState; state->varExpressionIsVariable = m_state.varExpressionIsVariable; return state; } void Parser::restoreState( Parser::ParserState* state) { m_state.varExpressionState = state->varExpressionState; m_state.varExpressionIsVariable = state->varExpressionIsVariable; } } // end of namespace Php :] -- kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on; mode KDevelop-PG[-Qt] diff --git a/parser/phplexer.cpp b/parser/phplexer.cpp index a810963..f4a6172 100644 --- a/parser/phplexer.cpp +++ b/parser/phplexer.cpp @@ -1,1048 +1,1050 @@ /*************************************************************************** * This file is part of KDevelop * * Copyright 2008 Niko Sams * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "phplexer.h" #include "phpparser.h" #include "tokenstream.h" #include #include #include #include #include "parserdebug.h" namespace Php { Lexer::Lexer(TokenStream* tokenStream, const QString& content, int initialState): m_content(content), m_tokenStream(tokenStream), m_curpos(0), m_contentSize(m_content.size()), m_tokenBegin(0), m_tokenEnd(0), m_haltCompiler(0) { pushState(ErrorState); if (initialState == DefaultState) { pushState(HtmlState); } pushState(initialState); } int Lexer::state(int deepness) const { return m_state.at(m_state.size() - deepness - 1); } void Lexer::printState() { int s = state(); if (s == ErrorState) qDebug() << "ErrorState"; else if (s == HtmlState) qDebug() << "HtmlState"; else if (s == DefaultState) qDebug() << "DefaultState"; else if (s == String) qDebug() << "String"; else if (s == StringVariable) qDebug() << "StringVariable"; else if (s == StringVariableBracket) qDebug() << "StringVariableBracket"; else if (s == StringVariableObjectOperator) qDebug() << "StringVariableObjectOperator"; else if (s == StringVariableCurly) qDebug() << "StringVariableCurly"; else if (s == StringVarname) qDebug() << "StringVarname"; else if (s == StringHeredoc) qDebug() << "StringHeredoc"; else if (s == StringBacktick) qDebug() << "StringBacktick"; } void Lexer::pushState(int state) { m_state.push(state); } void Lexer::popState() { m_state.pop(); } int Lexer::nextTokenKind() { int token = Parser::Token_INVALID; if (m_curpos >= m_contentSize) { m_tokenBegin = -1; m_tokenEnd = -1; createNewline(m_curpos); return 0; } const QChar* it = m_content.constData(); it += m_curpos; m_tokenBegin = m_curpos; switch (state()) { case HtmlState: if (it->unicode() == '<' && (it + 1)->unicode() == '?' ///TODO: per-project configuration to set whether we use shortags /// or not. In the former case we'd need to rise an error here && !( (it + 2)->toLower().unicode() == 'x' && (it + 3)->toLower().unicode() == 'm' && (it + 4)->toLower().unicode() == 'l' ) ) { token = Parser::Token_OPEN_TAG; if ((it + 2)->unicode() == '=') { token = Parser::Token_OPEN_TAG_WITH_ECHO; m_curpos++; it++; } else if ((it + 2)->toLower().unicode() == 'p' && (it + 3)->toLower().unicode() == 'h' && (it + 4)->toLower().unicode() == 'p' && (it + 5)->isSpace()) { m_curpos += 4; if ((it + 5)->unicode() == '\n') createNewline(m_curpos + 1); } m_curpos++; pushState(DefaultState); } else { token = Parser::Token_INLINE_HTML; while (m_curpos < m_contentSize) { if (it->unicode() == '\n') createNewline(m_curpos); if ((it + 1)->unicode() == '<' && (it + 2)->unicode() == '?') { break; } it++; m_curpos++; } } break; case DefaultState: case StringVariableCurly: { if (it->isSpace()) { token = Parser::Token_WHITESPACE; while (m_curpos < m_contentSize && it->isSpace()) { if (it->unicode() == '\n') createNewline(m_curpos); it++; m_curpos++; } m_curpos--; } else if (it->isDigit() || (it->unicode() == '.' && (it + 1)->isDigit())) { QString num;bool hasPoint = false; bool hex = false; bool bin = false; if (it->unicode() == '0' && (it + 1)->toLower() == 'x') { it += 2; m_curpos += 2; hex = true; } if (it->unicode() == '0' && (it + 1)->toLower() == 'b') { it += 2; m_curpos += 2; bin = true; } while (m_curpos < m_contentSize && ( it->isDigit() || (!hex && !hasPoint && it->unicode() == '.') || (bin && (it->unicode() == '0' || it->unicode() == '1')) || (hex && (it->toLower() == 'a' || it->toLower() == 'b' || it->toLower() == 'c' || it->toLower() == 'd' || it->toLower() == 'e' || it->toLower() == 'f')))) { if (it->unicode() == '.') hasPoint = true; num.append(*it); it++; m_curpos++; } if (!hex && !bin && it->toLower() == 'e' && ((it + 1)->isDigit() || (((it + 1)->unicode() == '-' || (it + 1)->unicode() == '+') && (it + 2)->isDigit()))) { //exponential number token = Parser::Token_DNUMBER; m_curpos++; it++; if (it->unicode() == '-' || it->unicode() == '+') { it++; m_curpos++; } while (m_curpos < m_contentSize && (it->isDigit())) { it++; m_curpos++; } m_curpos--; } else { m_curpos--; if (hasPoint) { token = Parser::Token_DNUMBER; } else { bool ok; //check if string can be converted to long //if we get an overflow use double num.toLong(&ok, hex ? 16 : 10); if (ok) { token = Parser::Token_LNUMBER; } else { token = Parser::Token_DNUMBER; } } } } else if (processVariable(it)) { token = Parser::Token_VARIABLE; } else if (it->unicode() == '$') { //when it was not recognized as variable token = Parser::Token_DOLLAR; } else if (it->unicode() == '}') { token = Parser::Token_RBRACE; if (state() == StringVariableCurly) { popState(); } } else if (it->unicode() == '{') { token = Parser::Token_LBRACE; if (state() == StringVariableCurly) { pushState(StringVariableCurly); } } else if (it->unicode() == ')') { token = Parser::Token_RPAREN; } else if (it->unicode() == '(') { it++; int pos = m_curpos + 1; while (pos < m_contentSize && it->isSpace()) { it++; pos++; } const int nameStart = pos; while (pos < m_contentSize && it->isLetter()) { it++; pos++; } const auto name = m_content.midRef(nameStart, pos - nameStart); while (pos < m_contentSize && it->isSpace()) { it++; pos++; } if (it->unicode() == ')') { if (name.compare(QLatin1String("int"), Qt::CaseInsensitive) == 0 || name.compare(QLatin1String("integer"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INT_CAST; } else if (name.compare(QLatin1String("real"), Qt::CaseInsensitive) == 0 || name.compare(QLatin1String("double"), Qt::CaseInsensitive) == 0 || name.compare(QLatin1String("float"), Qt::CaseInsensitive) == 0) { token = Parser::Token_DOUBLE_CAST; } else if (name.compare(QLatin1String("string"), Qt::CaseInsensitive) == 0) { token = Parser::Token_STRING_CAST; } else if (name.compare(QLatin1String("binary"), Qt::CaseInsensitive) == 0) { //as in php token = Parser::Token_STRING_CAST; } else if (name.compare(QLatin1String("array"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ARRAY_CAST; } else if (name.compare(QLatin1String("object"), Qt::CaseInsensitive) == 0) { token = Parser::Token_OBJECT_CAST; } else if (name.compare(QLatin1String("bool"), Qt::CaseInsensitive) == 0 || name.compare(QLatin1String("boolean"), Qt::CaseInsensitive) == 0) { token = Parser::Token_BOOL_CAST; } else if (name.compare(QLatin1String("unset"), Qt::CaseInsensitive) == 0) { token = Parser::Token_UNSET_CAST; } else { token = Parser::Token_LPAREN; } if (token != Parser::Token_LPAREN) { m_curpos = pos; } } else { token = Parser::Token_LPAREN; } } else if (it->unicode() == ']') { token = Parser::Token_RBRACKET; } else if (it->unicode() == '[') { token = Parser::Token_LBRACKET; } else if (it->unicode() == ',') { token = Parser::Token_COMMA; } else if (it->unicode() == '@') { token = Parser::Token_AT; } else if (it->unicode() == '!') { if ((it + 1)->unicode() == '=') { m_curpos++; if ((it + 2)->unicode() == '=') { m_curpos++; token = Parser::Token_IS_NOT_IDENTICAL; } else { token = Parser::Token_IS_NOT_EQUAL; } } else { token = Parser::Token_BANG; } } else if (it->unicode() == '<') { if ((it + 1)->unicode() == '<') { m_curpos++; if ((it + 2)->unicode() == '<' && state() != StringVariableCurly) { //HEREDOC string (<<< EOD\nfoo\nEOD;\n) int pos = 3; while (m_curpos + pos < m_contentSize && ((it + pos)->unicode() == ' ' || (it + pos)->unicode() == '\t')) { pos++; } bool isNowdoc = (it + pos)->unicode() == '\''; bool foundQuote = isNowdoc || (it + pos)->unicode() == '"'; if (foundQuote) { ++pos; } if ((it + pos)->isLetter() || (it + pos)->unicode() == '_') { //identifier must start with a letter m_hereNowDocIdentifier.clear(); while (m_curpos + pos < m_contentSize && ((it + pos)->isDigit() || (it + pos)->isLetter() || (it + pos)->unicode() == '_')) { m_hereNowDocIdentifier.append(*(it + pos)); pos++; } if (foundQuote && (m_curpos + pos) < m_contentSize) { if (isNowdoc && (it+pos)->unicode() == '\'') { ++pos; } else if ((it+pos)->unicode() == '"') { ++pos; } } if (m_curpos + pos < m_contentSize && (it + pos)->unicode() == '\n') { //identifier must be followed by newline, newline is part of HEREDOC token if (isNowdoc) { token = Parser::Token_START_NOWDOC; pushState(StringNowdoc); } else { token = Parser::Token_START_HEREDOC; pushState(StringHeredoc); } m_curpos += pos - 1; createNewline(m_curpos); } } } if (token != Parser::Token_START_HEREDOC && token != Parser::Token_START_NOWDOC) { if ((it + 2)->unicode() == '=') { m_curpos++; token = Parser::Token_SL_ASSIGN; } else { token = Parser::Token_SL; } } } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_IS_SMALLER_OR_EQUAL; } else if ((it + 1)->unicode() == '>') { m_curpos++; token = Parser::Token_IS_NOT_EQUAL; } else { token = Parser::Token_IS_SMALLER; } } else if (it->unicode() == '>') { if ((it + 1)->unicode() == '>') { m_curpos++; if ((it + 2)->unicode() == '=') { m_curpos++; token = Parser::Token_SR_ASSIGN; } else { token = Parser::Token_SR; } } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_IS_GREATER_OR_EQUAL; } else { token = Parser::Token_IS_GREATER; } } else if (it->unicode() == '~') { token = Parser::Token_TILDE; } else if (it->unicode() == ':') { if ((it + 1)->unicode() == ':') { m_curpos++; token = Parser::Token_PAAMAYIM_NEKUDOTAYIM; } else { token = Parser::Token_COLON; } } else if (it->unicode() == '?') { if ((it + 1)->unicode() == '>') { //accept CLOSE_TAG inside StringVariableCurly too, as php does token = Parser::Token_CLOSE_TAG; m_curpos++; while (state() != HtmlState) popState(); } else { token = Parser::Token_QUESTION; } } else if (it->unicode() == '-' && (it + 1)->unicode() == '>') { m_curpos++; token = Parser::Token_OBJECT_OPERATOR; if (isValidVariableIdentifier(it + 2)) { pushState(StringVariableObjectOperator); } } else if (it->unicode() == '%') { if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_MOD_ASSIGN; } else { token = Parser::Token_MOD; } } else if (it->unicode() == '/') { if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_DIV_ASSIGN; } else if ((it + 1)->unicode() == '/') { //accept COMMENT inside StringVariableCurly too, as php does if ((it + 2)->unicode() == '/') { token = Parser::Token_DOC_COMMENT; } else { token = Parser::Token_COMMENT; } while (m_curpos < m_contentSize) { if (m_curpos + 1 < m_contentSize && it->unicode() == '?' && (it + 1)->unicode() == '>') { --it; --m_curpos; break; } if ( it->unicode() == '\n' ) { createNewline(m_curpos); if ( token == Parser::Token_COMMENT ) { break; } else { // lookahead to check whether this doc comment spans multiple lines const QChar* it2 = it + 1; int pos = m_curpos + 1; while ( pos < m_contentSize && (it2)->isSpace() && (it2)->unicode() != '\n' ) { ++it2; ++pos; } if ( it2->unicode() == '/' && (it2 + 1)->unicode() == '/' && (it2 + 2)->unicode() == '/' ) { // seems to be a multi-line doc-comment it = it2 + 2; m_curpos = pos + 2; continue; } else { // not a multi-line doc-comment break; } } } it++; m_curpos++; } } else if ((it + 1)->unicode() == '*') { //accept COMMENT inside StringVariableCurly too, as php does if ((it + 2)->unicode() == '*' && (it + 3)->isSpace()) { token = Parser::Token_DOC_COMMENT; } else { token = Parser::Token_COMMENT; } it += 2; m_curpos += 2; while (m_curpos < m_contentSize && !(it->unicode() == '*' && (it + 1)->unicode() == '/')) { if (it->unicode() == '\n') { createNewline(m_curpos); } it++; m_curpos++; } m_curpos++; } else { token = Parser::Token_DIV; } } else if (it->unicode() == '#') { //accept COMMENT inside StringVariableCurly too, as php does token = Parser::Token_COMMENT; while (m_curpos < m_contentSize) { if (m_curpos + 1 < m_contentSize && it->unicode() == '?' && (it + 1)->unicode() == '>') { --it; --m_curpos; break; } if (it->unicode() == '\n') { createNewline(m_curpos); break; } it++; m_curpos++; } } else if (it->unicode() == '^') { if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_XOR_ASSIGN; } else { token = Parser::Token_BIT_XOR; } } else if (it->unicode() == '*') { if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_MUL_ASSIGN; } else if ((it + 1)->unicode() == '*') { m_curpos++; if ((it + 2)->unicode() == '=') { m_curpos++; token = Parser::Token_EXP_ASSIGN; } else { token = Parser::Token_EXP; } } else { token = Parser::Token_MUL; } } else if (it->unicode() == '|') { if ((it + 1)->unicode() == '|') { m_curpos++; token = Parser::Token_BOOLEAN_OR; } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_OR_ASSIGN; } else { token = Parser::Token_BIT_OR; } } else if (it->unicode() == '&') { if ((it + 1)->unicode() == '&') { m_curpos++; token = Parser::Token_BOOLEAN_AND; } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_AND_ASSIGN; } else { token = Parser::Token_BIT_AND; } } else if (it->unicode() == '+') { if ((it + 1)->unicode() == '+') { m_curpos++; token = Parser::Token_INC; } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_PLUS_ASSIGN; } else { token = Parser::Token_PLUS; } } else if (it->unicode() == '-') { if ((it + 1)->unicode() == '-') { m_curpos++; token = Parser::Token_DEC; } else if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_MINUS_ASSIGN; } else { token = Parser::Token_MINUS; } } else if (it->unicode() == '.') { if ((it + 1)->unicode() == '=') { m_curpos++; token = Parser::Token_CONCAT_ASSIGN; } else if ((it + 1)->unicode() == '.' && (it + 2)->unicode() == '.') { m_curpos = m_curpos + 2; token = Parser::Token_ELLIPSIS; } else { token = Parser::Token_CONCAT; } } else if (it->unicode() == '\\') { token = Parser::Token_BACKSLASH; } else if (it->unicode() == ';') { token = Parser::Token_SEMICOLON; } else if (it->unicode() == '\'') { token = Parser::Token_CONSTANT_ENCAPSED_STRING; it++; m_curpos++; int startPos = m_curpos; while (m_curpos < m_contentSize && (it->unicode() != '\'' || isEscapedWithBackslash(it, m_curpos, startPos))) { if (it->unicode() == '\n') createNewline(m_curpos); it++; m_curpos++; } // if the string is never terminated, make sure we don't overflow the boundaries if ( m_curpos == m_contentSize ) { --m_curpos; } } else if (it->unicode() == '"') { it++; m_curpos++; int stringSize = 0; bool foundVar = false; while (m_curpos + stringSize < m_contentSize && (it->unicode() != '"' || isEscapedWithBackslash(it, m_curpos + stringSize, m_curpos))) { if (it->unicode() == '$' && !isEscapedWithBackslash(it, m_curpos + stringSize, m_curpos) && ((it + 1)->unicode() == '{' || (isValidVariableIdentifier(it + 1) && !(it + 1)->isDigit()))) { foundVar = true; break; } it++; stringSize++; } if (!foundVar) { // if the string is never terminated, make sure we don't overflow the boundaries if ( m_curpos + stringSize == m_contentSize ) { m_curpos--; } token = Parser::Token_CONSTANT_ENCAPSED_STRING; it -= stringSize; for (int j = 0; j < stringSize; j++) { if (it->unicode() == '\n') { createNewline(m_curpos + j); } it++; } m_curpos += stringSize; } else { // properly set the token pos to the starting double quote m_curpos--; token = Parser::Token_DOUBLE_QUOTE; pushState(String); } } else if (it->unicode() == '`') { token = Parser::Token_BACKTICK; pushState(StringBacktick); } else if (it->unicode() == '=') { if ((it + 1)->unicode() == '=') { m_curpos++; if ((it + 2)->unicode() == '=') { m_curpos++; token = Parser::Token_IS_IDENTICAL; } else { token = Parser::Token_IS_EQUAL; } } else if ((it + 1)->unicode() == '>') { m_curpos++; token = Parser::Token_DOUBLE_ARROW; } else { token = Parser::Token_ASSIGN; } } else if (isValidVariableIdentifier(it) && !it->isDigit()) { const int from = m_curpos; while (m_curpos < m_contentSize && (isValidVariableIdentifier(it))) { it++; m_curpos++; } const QStringRef name = m_content.midRef(from, m_curpos - from); m_curpos--; if (name.compare(QLatin1String("echo"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ECHO; } else if (name.compare(QLatin1String("include"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INCLUDE; } else if (name.compare(QLatin1String("include_once"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INCLUDE_ONCE; } else if (name.compare(QLatin1String("require"), Qt::CaseInsensitive) == 0) { token = Parser::Token_REQUIRE; } else if (name.compare(QLatin1String("require_once"), Qt::CaseInsensitive) == 0) { token = Parser::Token_REQUIRE_ONCE; } else if (name.compare(QLatin1String("eval"), Qt::CaseInsensitive) == 0) { token = Parser::Token_EVAL; } else if (name.compare(QLatin1String("print"), Qt::CaseInsensitive) == 0) { token = Parser::Token_PRINT; } else if (name.compare(QLatin1String("abstract"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ABSTRACT; } else if (name.compare(QLatin1String("break"), Qt::CaseInsensitive) == 0) { token = Parser::Token_BREAK; } else if (name.compare(QLatin1String("case"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CASE; } else if (name.compare(QLatin1String("catch"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CATCH; } else if (name.compare(QLatin1String("class"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CLASS; } else if (name.compare(QLatin1String("const"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CONST; } else if (name.compare(QLatin1String("continue"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CONTINUE; } else if (name.compare(QLatin1String("default"), Qt::CaseInsensitive) == 0) { token = Parser::Token_DEFAULT; } else if (name.compare(QLatin1String("do"), Qt::CaseInsensitive) == 0) { token = Parser::Token_DO; } else if (name.compare(QLatin1String("else"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ELSE; } else if (name.compare(QLatin1String("extends"), Qt::CaseInsensitive) == 0) { token = Parser::Token_EXTENDS; } else if (name.compare(QLatin1String("final"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FINAL; } else if (name.compare(QLatin1String("for"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FOR; } else if (name.compare(QLatin1String("if"), Qt::CaseInsensitive) == 0) { token = Parser::Token_IF; } else if (name.compare(QLatin1String("implements"), Qt::CaseInsensitive) == 0) { token = Parser::Token_IMPLEMENTS; } else if (name.compare(QLatin1String("instanceof"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INSTANCEOF; } else if (name.compare(QLatin1String("insteadof"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INSTEADOF; } else if (name.compare(QLatin1String("interface"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INTERFACE; } else if (name.compare(QLatin1String("trait"), Qt::CaseInsensitive) == 0) { token = Parser::Token_TRAIT; } else if (name.compare(QLatin1String("new"), Qt::CaseInsensitive) == 0) { token = Parser::Token_NEW; } else if (name.compare(QLatin1String("private"), Qt::CaseInsensitive) == 0) { token = Parser::Token_PRIVATE; } else if (name.compare(QLatin1String("protected"), Qt::CaseInsensitive) == 0) { token = Parser::Token_PROTECTED; } else if (name.compare(QLatin1String("public"), Qt::CaseInsensitive) == 0) { token = Parser::Token_PUBLIC; } else if (name.compare(QLatin1String("return"), Qt::CaseInsensitive) == 0) { token = Parser::Token_RETURN; } else if (name.compare(QLatin1String("static"), Qt::CaseInsensitive) == 0) { const QChar* lookAhead = it; int pos = m_curpos; while (pos < m_contentSize && lookAhead->isSpace()) { ++lookAhead; ++pos; } if (pos + 1 < m_contentSize && lookAhead->unicode() == ':' && (++lookAhead)->unicode() == ':') { // PHP 5.3 - late static token = Parser::Token_STRING; } else { token = Parser::Token_STATIC; } } else if (name.compare(QLatin1String("switch"), Qt::CaseInsensitive) == 0) { token = Parser::Token_SWITCH; } else if (name.compare(QLatin1String("throw"), Qt::CaseInsensitive) == 0) { token = Parser::Token_THROW; } else if (name.compare(QLatin1String("try"), Qt::CaseInsensitive) == 0) { token = Parser::Token_TRY; } else if (name.compare(QLatin1String("finally"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FINALLY; } else if (name.compare(QLatin1String("while"), Qt::CaseInsensitive) == 0) { token = Parser::Token_WHILE; } else if (name.compare(QLatin1String("clone"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CLONE; } else if (name.compare(QLatin1String("exit"), Qt::CaseInsensitive) == 0 || name.compare(QLatin1String("die"), Qt::CaseInsensitive) == 0) { token = Parser::Token_EXIT; } else if (name.compare(QLatin1String("elseif"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ELSEIF; } else if (name.compare(QLatin1String("endif"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDIF; } else if (name.compare(QLatin1String("endwhile"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDWHILE; } else if (name.compare(QLatin1String("endfor"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDFOR; } else if (name.compare(QLatin1String("foreach"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FOREACH; } else if (name.compare(QLatin1String("endforeach"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDFOREACH; } else if (name.compare(QLatin1String("declare"), Qt::CaseInsensitive) == 0) { token = Parser::Token_DECLARE; } else if (name.compare(QLatin1String("enddeclare"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDDECLARE; } else if (name.compare(QLatin1String("as"), Qt::CaseInsensitive) == 0) { token = Parser::Token_AS; } else if (name.compare(QLatin1String("endswitch"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ENDSWITCH; } else if (name.compare(QLatin1String("function"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FUNCTION; } else if (name.compare(QLatin1String("use"), Qt::CaseInsensitive) == 0) { token = Parser::Token_USE; } else if (name.compare(QLatin1String("goto"), Qt::CaseInsensitive) == 0) { token = Parser::Token_GOTO; } else if (name.compare(QLatin1String("global"), Qt::CaseInsensitive) == 0) { token = Parser::Token_GLOBAL; } else if (name.compare(QLatin1String("var"), Qt::CaseInsensitive) == 0) { token = Parser::Token_VAR; } else if (name.compare(QLatin1String("unset"), Qt::CaseInsensitive) == 0) { token = Parser::Token_UNSET; } else if (name.compare(QLatin1String("isset"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ISSET; } else if (name.compare(QLatin1String("empty"), Qt::CaseInsensitive) == 0) { token = Parser::Token_EMPTY; } else if (name.compare(QLatin1String("__halt_compiler"), Qt::CaseInsensitive) == 0) { token = Parser::Token_HALT_COMPILER; } else if (name.compare(QLatin1String("list"), Qt::CaseInsensitive) == 0) { token = Parser::Token_LIST; } else if (name.compare(QLatin1String("array"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ARRAY; } else if (name.compare(QLatin1String("__class__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CLASS_C; } else if (name.compare(QLatin1String("__method__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_METHOD_C; } else if (name.compare(QLatin1String("__function__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FUNC_C; } else if (name.compare(QLatin1String("__line__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_LINE; } else if (name.compare(QLatin1String("__file__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FILE; } else if (name.compare(QLatin1String("or"), Qt::CaseInsensitive) == 0) { token = Parser::Token_LOGICAL_OR; } else if (name.compare(QLatin1String("and"), Qt::CaseInsensitive) == 0) { token = Parser::Token_LOGICAL_AND; } else if (name.compare(QLatin1String("xor"), Qt::CaseInsensitive) == 0) { token = Parser::Token_LOGICAL_XOR; } else if (name.compare(QLatin1String("namespace"), Qt::CaseInsensitive) == 0) { token = Parser::Token_NAMESPACE; } else if (name.compare(QLatin1String("__namespace__"), Qt::CaseInsensitive) == 0) { token = Parser::Token_NAMESPACE_C; } else if (name.compare(QLatin1String("callable"), Qt::CaseInsensitive) == 0) { token = Parser::Token_CALLABLE; } else if (name.compare(QLatin1String("iterable"), Qt::CaseInsensitive) == 0) { token = Parser::Token_ITERABLE; } else if (name.compare(QLatin1String("bool"), Qt::CaseInsensitive) == 0) { token = Parser::Token_BOOL; } else if (name.compare(QLatin1String("float"), Qt::CaseInsensitive) == 0) { token = Parser::Token_FLOAT; } else if (name.compare(QLatin1String("int"), Qt::CaseInsensitive) == 0) { token = Parser::Token_INT; } else if (name.compare(QLatin1String("string"), Qt::CaseInsensitive) == 0) { token = Parser::Token_STRING_TYPE; + } else if (name.compare(QLatin1String("void"), Qt::CaseInsensitive) == 0) { + token = Parser::Token_VOID; } else { token = Parser::Token_STRING; } } break; } case StringVariable: case String: case StringHeredoc: case StringBacktick: if ((state() == String || state(1) == String) && it->unicode() == '"') { token = Parser::Token_DOUBLE_QUOTE; if (state() == StringVariable) popState(); popState(); } else if ((state() == StringBacktick || state(1) == StringBacktick) && it->unicode() == '`') { token = Parser::Token_BACKTICK; if (state() == StringVariable) popState(); popState(); } else if ((state() == StringHeredoc || state(1) == StringHeredoc) && isHereNowDocEnd(it)) { token = Parser::Token_END_HEREDOC; m_curpos += m_hereNowDocIdentifier.length() - 1; if (state() == StringVariable) popState(); popState(); } else if (processVariable(it)) { token = Parser::Token_VARIABLE; if (state() != StringVariable) pushState(StringVariable); } else if (state() != StringVariable && it->unicode() == '$' && (it + 1)->unicode() == '{') { token = Parser::Token_DOLLAR_OPEN_CURLY_BRACES; m_curpos++; it += 2; //check if a valid variable follows if ((isValidVariableIdentifier(it) && !it->isDigit())) { pushState(StringVarname); } } else if (state() == StringVariable && it->unicode() == '[') { token = Parser::Token_LBRACKET; pushState(StringVariableBracket); } else if (state() != StringVariable && it->unicode() == '{' && (it + 1)->unicode() == '$' && ((isValidVariableIdentifier(it + 2) && !(it + 2)->isDigit()) || (it + 2)->unicode() == '{')) { token = Parser::Token_CURLY_OPEN; pushState(StringVariableCurly); } else if (state() == StringVariable && it->unicode() == '-' && (it + 1)->unicode() == '>' && isValidVariableIdentifier(it + 2) && !(it + 2)->isDigit()) { token = Parser::Token_OBJECT_OPERATOR; m_curpos++; pushState(StringVariableObjectOperator); } else { if (state() == StringVariable) popState(); token = Parser::Token_ENCAPSED_AND_WHITESPACE; int startPos = m_curpos; while (m_curpos < m_contentSize) { if (!isEscapedWithBackslash(it, m_curpos, startPos) && ((it->unicode() == '$' && (it + 1)->unicode() == '{') || (it->unicode() == '{' && (it + 1)->unicode() == '$' && isValidVariableIdentifier(it + 2)) || (it->unicode() == '$' && isValidVariableIdentifier(it + 1) && !(it + 1)->isDigit()))) { //variable is next ${var} or {$var} break; } if (state() == String && it->unicode() == '"' && !isEscapedWithBackslash(it, m_curpos, startPos)) { //end of string break; } if (state() == StringBacktick && it->unicode() == '`' && !isEscapedWithBackslash(it, m_curpos, startPos)) { //end of string break; } if (it->unicode() == '\n') createNewline(m_curpos); m_curpos++; it++; if (state() == StringHeredoc && (it - 1)->unicode() == '\n') { //check for end of heredoc (\nEOD;\n) if (state() == StringHeredoc && isHereNowDocEnd(it)) { break; } } } m_curpos--; } break; case StringNowdoc: if (isHereNowDocEnd(it)) { token = Parser::Token_END_NOWDOC; m_curpos += m_hereNowDocIdentifier.length() - 1; popState(); } else { token = Parser::Token_STRING; while (m_curpos < m_contentSize) { if (it->unicode() == '\n') createNewline(m_curpos); m_curpos++; it++; if ((it - 1)->unicode() == '\n' && isHereNowDocEnd(it)) { //check for end of nowdoc (\nEOD;\n) break; } } m_curpos--; } break; case StringVariableBracket: if (it->unicode() == ']') { token = Parser::Token_RBRACKET; popState(); popState(); } else if (it->isDigit()) { token = Parser::Token_NUM_STRING; while (m_curpos < m_contentSize && it->isDigit()) { it++; m_curpos++; } m_curpos--; } else { token = Parser::Token_STRING; while (m_curpos < m_contentSize && (it->unicode() != ']')) { if (it->unicode() == '\n') createNewline(m_curpos); it++; m_curpos++; } m_curpos--; } break; case StringVariableObjectOperator: token = Parser::Token_STRING; while (m_curpos < m_contentSize && isValidVariableIdentifier(it)) { it++; m_curpos++; } m_curpos--; popState(); if (state() == StringVariable) popState(); break; case StringVarname: popState(); pushState(StringVariableCurly); token = Parser::Token_STRING_VARNAME; while (m_curpos < m_contentSize && isValidVariableIdentifier(it)) { it++; m_curpos++; } m_curpos--; break; default: token = Parser::Token_INVALID; break; } if (m_curpos > m_contentSize) { m_tokenBegin = -1; m_tokenEnd = -1; return 0; } m_tokenEnd = m_curpos; m_curpos++; if (m_haltCompiler) { //look for __halt_compiler(); and stop lexer there if (m_haltCompiler == 4) { token = 0; //EOF } else if (token == Parser::Token_WHITESPACE || token == Parser::Token_COMMENT || token == Parser::Token_DOC_COMMENT) { //ignore } else if (m_haltCompiler == 1 && token == Parser::Token_LPAREN) { m_haltCompiler++; } else if (m_haltCompiler == 2 && token == Parser::Token_RPAREN) { m_haltCompiler++; } else if (m_haltCompiler == 3 && token == Parser::Token_SEMICOLON) { m_haltCompiler++; } else { m_haltCompiler = 0; } } if (token == Parser::Token_HALT_COMPILER && !m_haltCompiler) { m_haltCompiler = 1; } return token; } qint64 Lexer::tokenBegin() const { return m_tokenBegin; } qint64 Lexer::tokenEnd() const { return m_tokenEnd; } bool Lexer::isHereNowDocEnd(const QChar* it) { int identiferLen = m_hereNowDocIdentifier.length(); QString lineStart; for (int i = 0; i < identiferLen; i++) { if (m_curpos + i >= m_contentSize) break; lineStart.append(*(it + i)); } if (lineStart == m_hereNowDocIdentifier && ((it + identiferLen)->unicode() == '\n' || ((it + identiferLen)->unicode() == ';' && (it + identiferLen + 1)->unicode() == '\n'))) { return true; } return false; } //used for strings, to check if " is escaped (\" is, \\" not) bool Lexer::isEscapedWithBackslash(const QChar* it, int curPos, int startPos) { int cnt = 0; it--; while (curPos > startPos && it->unicode() == '\\') { cnt++; it--; } return (cnt % 2) == 1; } bool Lexer::processVariable(const QChar* it) { const QChar* c2 = it + 1; if (it->unicode() == '$' && (isValidVariableIdentifier(c2) && !c2->isDigit())) { it++; m_curpos++; while (m_curpos < m_contentSize && (isValidVariableIdentifier(it))) { it++; m_curpos++; } m_curpos--; return true; } else { return false; } } bool Lexer::isValidVariableIdentifier(const QChar* it) { return it->isLetter() || it->isDigit() || it->unicode() == '_' || it->unicode() > 0x7f; } void Lexer::createNewline(int pos) { if (m_tokenStream) m_tokenStream->locationTable()->newline(pos); } } diff --git a/parser/test/lexertest.cpp b/parser/test/lexertest.cpp index 7d38df8..e7da1ac 100644 --- a/parser/test/lexertest.cpp +++ b/parser/test/lexertest.cpp @@ -1,562 +1,581 @@ /* This file is part of KDevelop Copyright 2006 Hamish Rodda Copyright 2008 Niko Sams This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "lexertest.h" #include #include "parsesession.h" #include "phplexer.h" #include "phptokentext.h" QTEST_MAIN(Php::LexerTest) namespace Php { #define COMPARE_TOKEN(tokenStream, index, tokenKind, startLine, startColumn, endLine, endColumn) \ { \ QVERIFY(tokenStream->at(index).kind == tokenKind); \ qint64 line; qint64 column; \ tokenStream->startPosition(index, &line, &column); \ QCOMPARE(line, (qint64) startLine); \ QCOMPARE(column, (qint64) startColumn); \ tokenStream->endPosition(index, &line, &column); \ QCOMPARE(line, (qint64) endLine); \ QCOMPARE(column, (qint64) endColumn); \ } LexerTest::LexerTest() { } void LexerTest::testOpenTagWithNewline() { TokenStream* ts = tokenize(QStringLiteral("size() == 3); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 1, 0, 1, 2); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 3, 1, 3); delete ts; } void LexerTest::testOpenTagWithSpace() { TokenStream* ts = tokenize(QStringLiteral("size() == 3); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_STRING, 0, 6, 0, 8); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 0, 9, 0, 9); delete ts; } void LexerTest::testCommentOneLine() { TokenStream* ts = tokenize(QStringLiteral("size() == 4); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 9); COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2); COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3); delete ts; } void LexerTest::testCommentOneLine2() { TokenStream* ts = tokenize(QStringLiteral("size() == 4); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 8); COMPARE_TOKEN(ts, 2, Parser::Token_STRING, 2, 0, 2, 2); COMPARE_TOKEN(ts, 3, Parser::Token_SEMICOLON, 2, 3, 2, 3); delete ts; } void LexerTest::testCommentMultiLine() { TokenStream* ts = tokenize(QStringLiteral("size() == 5); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6); COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2); COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3); delete ts; } void LexerTest::testCommentMultiLine2() { TokenStream* ts = tokenize(QStringLiteral("size() == 5); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 2, 5); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 2, 6, 2, 6); COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 3, 0, 3, 2); COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 3, 3, 3, 3); delete ts; } void LexerTest::testEndTag() { TokenStream* ts = tokenize(QStringLiteral("\n>"), true, Lexer::DefaultState); //don't crash and we are fine delete ts; } void LexerTest::testNewlineInString() { //0 1 //012345 6 7 890123456789 TokenStream* ts = tokenize(QStringLiteral("size() == 3); COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1); delete ts; } void LexerTest::testNewlineInString2() { //0 //0123 4567 TokenStream* ts = tokenize(QStringLiteral("size(), 3); COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 1, 0); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 1, 1, 1); delete ts; } void LexerTest::testNewlineInStringWithVar() { TokenStream* ts = tokenize(QStringLiteral("size(), 6); COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6); COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8); COMPARE_TOKEN(ts, 3, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 9, 0, 9); COMPARE_TOKEN(ts, 4, Parser::Token_DOUBLE_QUOTE, 1, 0, 1, 0); COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 1, 1, 1, 1); delete ts; } void LexerTest::testNewlineInStringWithVar2() { //0 1 //012345 6 789 0123456789 TokenStream* ts = tokenize(QStringLiteral("size(), 7); COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6); COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 7); COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 1, 0, 1, 1); COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 1, 2, 1, 2); COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 2, 0, 2, 0); COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 2, 1, 2, 1); delete ts; } void LexerTest::testNewlineInStringWithVar3() { //0 1 //012345 6 789 0123456789 TokenStream* ts = tokenize(QStringLiteral("size(), 7); COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6); COMPARE_TOKEN(ts, 2, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 7, 0, 8); COMPARE_TOKEN(ts, 3, Parser::Token_VARIABLE, 0, 9, 0, 10); COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 0, 11, 0, 11); COMPARE_TOKEN(ts, 5, Parser::Token_DOUBLE_QUOTE, 0, 12, 0, 12); COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 0, 13, 0, 13); delete ts; } void LexerTest::testMultiplePhpSections() { //0 1 //012345 6 789 0123456789 TokenStream* ts = tokenize(QStringLiteral("\n\n"), true); QCOMPARE((int)ts->size(), 9); qint64 index = 0; for (qint64 line = 0; line <= 2; ++line) { if (line == 1) { // the html stuff in the middle COMPARE_TOKEN(ts, index, Parser::Token_INLINE_HTML, 0, 11, 1, 6); ++index; } else { // the php stuff (symmetric) at the start and end COMPARE_TOKEN(ts, index, Parser::Token_OPEN_TAG, line, 0, line, 5); ++index; COMPARE_TOKEN(ts, index, Parser::Token_VARIABLE, line, 6, line, 7); ++index; COMPARE_TOKEN(ts, index, Parser::Token_SEMICOLON, line, 8, line, 8); ++index; COMPARE_TOKEN(ts, index, Parser::Token_CLOSE_TAG, line, 9, line, 10); ++index; } } delete ts; } void LexerTest::testHereDoc() { TokenStream* ts = tokenize(QStringLiteral("size(), 12); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3); COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 12); COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5); COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10); COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3); COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3); COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4); COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6); COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7); delete ts; } void LexerTest::testHereDocQuoted() { TokenStream* ts = tokenize(QStringLiteral("size(), 12); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3); COMPARE_TOKEN(ts, 3, Parser::Token_START_HEREDOC, 1, 5, 1, 14); COMPARE_TOKEN(ts, 4, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 0, 2, 5); COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 2, 6, 2, 10); COMPARE_TOKEN(ts, 6, Parser::Token_ENCAPSED_AND_WHITESPACE, 2, 11, 3, 3); COMPARE_TOKEN(ts, 7, Parser::Token_END_HEREDOC, 4, 0, 4, 3); COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 4, 4, 4, 4); COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 5, 0, 5, 6); COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 5, 7, 5, 7); delete ts; } void LexerTest::testNowdoc() { TokenStream* ts = tokenize(QStringLiteral("size(), 10); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_ECHO, 1, 0, 1, 3); COMPARE_TOKEN(ts, 3, Parser::Token_START_NOWDOC, 1, 5, 1, 14); COMPARE_TOKEN(ts, 4, Parser::Token_STRING, 2, 0, 3, 3); COMPARE_TOKEN(ts, 5, Parser::Token_END_NOWDOC, 4, 0, 4, 3); COMPARE_TOKEN(ts, 6, Parser::Token_SEMICOLON, 4, 4, 4, 4); COMPARE_TOKEN(ts, 8, Parser::Token_VARIABLE, 5, 0, 5, 6); COMPARE_TOKEN(ts, 9, Parser::Token_SEMICOLON, 5, 7, 5, 7); delete ts; } void LexerTest::testCommonStringTokens() { // all these should have open_tag followed by constant encapsed string foreach ( const QString& code, QStringList() << "size(), 2); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_CONSTANT_ENCAPSED_STRING, 0, 6, 0, code.size() - 1); delete ts; } } void LexerTest::testNonTerminatedStringWithVar() { TokenStream* ts = tokenize(QStringLiteral("size(), 3); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_DOUBLE_QUOTE, 0, 6, 0, 6); COMPARE_TOKEN(ts, 2, Parser::Token_VARIABLE, 0, 7, 0, 8); delete ts; } void LexerTest::testPhpBlockWithComment() { TokenStream* ts = tokenize( QStringLiteral("\n" "size(), 5); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 1, 0, 1, 6); COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 2, 0, 2, 1); COMPARE_TOKEN(ts, 3, Parser::Token_INLINE_HTML, 2, 2, 2, 2); COMPARE_TOKEN(ts, 4, Parser::Token_OPEN_TAG, 3, 0, 3, 5); delete ts; } void LexerTest::testNamespaces() { TokenStream* ts = tokenize( QStringLiteral("size(), 25); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_NAMESPACE, 1, 0, 1, 8); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 9, 1, 9); COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 10, 1, 12); COMPARE_TOKEN(ts, 4, Parser::Token_SEMICOLON, 1, 13, 1, 13); COMPARE_TOKEN(ts, 6, Parser::Token_NAMESPACE, 2, 0, 2, 8); COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 2, 9, 2, 9); COMPARE_TOKEN(ts, 8, Parser::Token_STRING, 2, 10, 2, 12); COMPARE_TOKEN(ts, 9, Parser::Token_BACKSLASH, 2, 13, 2, 13); COMPARE_TOKEN(ts, 10, Parser::Token_STRING, 2, 14, 2, 16); COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 2, 17, 2, 17); COMPARE_TOKEN(ts, 13, Parser::Token_NAMESPACE, 3, 0, 3, 8); COMPARE_TOKEN(ts, 14, Parser::Token_WHITESPACE, 3, 9, 3, 9); COMPARE_TOKEN(ts, 15, Parser::Token_STRING, 3, 10, 3, 12); COMPARE_TOKEN(ts, 16, Parser::Token_BACKSLASH, 3, 13, 3, 13); COMPARE_TOKEN(ts, 17, Parser::Token_STRING, 3, 14, 3, 16); COMPARE_TOKEN(ts, 18, Parser::Token_BACKSLASH, 3, 17, 3, 17); COMPARE_TOKEN(ts, 19, Parser::Token_STRING, 3, 18, 3, 20); COMPARE_TOKEN(ts, 20, Parser::Token_WHITESPACE, 3, 21, 3, 21); COMPARE_TOKEN(ts, 21, Parser::Token_LBRACE, 3, 22, 3, 22); COMPARE_TOKEN(ts, 23, Parser::Token_RBRACE, 4, 0, 4, 0); delete ts; } void LexerTest::testCloseTagInComment() { { TokenStream* ts = tokenize( QStringLiteral("") , true); QCOMPARE((int)ts->size(), 3); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13); COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15); delete ts; } { TokenStream* ts = tokenize( QStringLiteral("") , true); QCOMPARE((int)ts->size(), 3); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_COMMENT, 0, 6, 0, 13); COMPARE_TOKEN(ts, 2, Parser::Token_CLOSE_TAG, 0, 14, 0, 15); delete ts; } } void LexerTest::testBinaryNumber() { TokenStream* ts = tokenize(QStringLiteral("size(), 6); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4); COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5); COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3); COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4); delete ts; } void LexerTest::testHexadecimalNumber() { TokenStream* ts = tokenize(QStringLiteral("size(), 12); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_LNUMBER, 1, 0, 1, 3); COMPARE_TOKEN(ts, 2, Parser::Token_SEMICOLON, 1, 4, 1, 4); COMPARE_TOKEN(ts, 3, Parser::Token_WHITESPACE, 1, 5, 1, 5); COMPARE_TOKEN(ts, 4, Parser::Token_LNUMBER, 2, 0, 2, 3); COMPARE_TOKEN(ts, 5, Parser::Token_SEMICOLON, 2, 4, 2, 4); COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 2, 5, 2, 5); COMPARE_TOKEN(ts, 7, Parser::Token_LNUMBER, 3, 0, 3, 6); COMPARE_TOKEN(ts, 8, Parser::Token_SEMICOLON, 3, 7, 3, 7); COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 3, 8, 3, 8); COMPARE_TOKEN(ts, 10, Parser::Token_LNUMBER, 4, 0, 4, 6); COMPARE_TOKEN(ts, 11, Parser::Token_SEMICOLON, 4, 7, 4, 7); delete ts; } void LexerTest::testTypeHintsOnFunction() { QScopedPointer ts(tokenize(QStringLiteral("size(), 25); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_FUNCTION, 1, 0, 1, 7); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 8, 1, 8); COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 9, 1, 9); COMPARE_TOKEN(ts, 4, Parser::Token_LPAREN, 1, 10, 1, 10); COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 1, 11, 1, 12); COMPARE_TOKEN(ts, 6, Parser::Token_COMMA, 1, 13, 1, 13); COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 1, 14, 1, 14); COMPARE_TOKEN(ts, 8, Parser::Token_ARRAY, 1, 15, 1, 19); COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 1, 20, 1, 20); COMPARE_TOKEN(ts, 10, Parser::Token_VARIABLE, 1, 21, 1, 22); COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 23, 1, 23); COMPARE_TOKEN(ts, 12, Parser::Token_ASSIGN, 1, 24, 1, 24); COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 25, 1, 25); COMPARE_TOKEN(ts, 14, Parser::Token_LBRACKET, 1, 26, 1, 26); COMPARE_TOKEN(ts, 15, Parser::Token_RBRACKET, 1, 27, 1, 27); COMPARE_TOKEN(ts, 16, Parser::Token_COMMA, 1, 28, 1, 28); COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 29, 1, 29); COMPARE_TOKEN(ts, 18, Parser::Token_CALLABLE, 1, 30, 1, 37); COMPARE_TOKEN(ts, 19, Parser::Token_WHITESPACE, 1, 38, 1, 38); COMPARE_TOKEN(ts, 20, Parser::Token_VARIABLE, 1, 39, 1, 40); COMPARE_TOKEN(ts, 21, Parser::Token_RPAREN, 1, 41, 1, 41); COMPARE_TOKEN(ts, 22, Parser::Token_WHITESPACE, 1, 42, 1, 42); COMPARE_TOKEN(ts, 23, Parser::Token_LBRACE, 1, 43, 1, 43); COMPARE_TOKEN(ts, 24, Parser::Token_RBRACE, 1, 44, 1, 44); } +void LexerTest::testReturnTypeHints() +{ + QScopedPointer ts(tokenize(QStringLiteral("size(), 12); + + COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); + COMPARE_TOKEN(ts, 1, Parser::Token_FUNCTION, 1, 0, 1, 7); + COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 8, 1, 8); + COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 9, 1, 9); + COMPARE_TOKEN(ts, 4, Parser::Token_LPAREN, 1, 10, 1, 10); + COMPARE_TOKEN(ts, 5, Parser::Token_RPAREN, 1, 11, 1, 11); + COMPARE_TOKEN(ts, 6, Parser::Token_COLON, 1, 12, 1, 12); + COMPARE_TOKEN(ts, 7, Parser::Token_WHITESPACE, 1, 13, 1, 13); + COMPARE_TOKEN(ts, 8, Parser::Token_STRING_TYPE, 1, 14, 1, 19); + COMPARE_TOKEN(ts, 9, Parser::Token_WHITESPACE, 1, 20, 1, 20); + COMPARE_TOKEN(ts, 10, Parser::Token_LBRACE, 1, 21, 1, 21); + COMPARE_TOKEN(ts, 11, Parser::Token_RBRACE, 1, 22, 1, 22); +} + void LexerTest::testExponentiation() { QScopedPointer ts(tokenize(QStringLiteral("size(), 18); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_VARIABLE, 1, 0, 1, 1); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 2, 1, 2); COMPARE_TOKEN(ts, 3, Parser::Token_ASSIGN, 1, 3, 1, 3); COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 4, 1, 4); COMPARE_TOKEN(ts, 5, Parser::Token_LNUMBER, 1, 5, 1, 5); COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 6, 1, 6); COMPARE_TOKEN(ts, 7, Parser::Token_EXP, 1, 7, 1, 8); COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 9, 1, 9); COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 10, 1, 10); COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 11, 1, 11); COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 12, 1, 12); COMPARE_TOKEN(ts, 12, Parser::Token_VARIABLE, 1, 13, 1, 14); COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15); COMPARE_TOKEN(ts, 14, Parser::Token_EXP_ASSIGN, 1, 16, 1, 18); COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 19, 1, 19); COMPARE_TOKEN(ts, 16, Parser::Token_LNUMBER, 1, 20, 1, 20); COMPARE_TOKEN(ts, 17, Parser::Token_SEMICOLON, 1, 21, 1, 21); } void LexerTest::testExceptionFinally() { QScopedPointer ts(tokenize(QStringLiteral("size(), 19); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_TRY, 1, 0, 1, 2); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 3, 1, 3); COMPARE_TOKEN(ts, 3, Parser::Token_LBRACE, 1, 4, 1, 4); COMPARE_TOKEN(ts, 4, Parser::Token_WHITESPACE, 1, 5, 1, 5); COMPARE_TOKEN(ts, 5, Parser::Token_VARIABLE, 1, 6, 1, 7); COMPARE_TOKEN(ts, 6, Parser::Token_WHITESPACE, 1, 8, 1, 8); COMPARE_TOKEN(ts, 7, Parser::Token_ASSIGN, 1, 9, 1, 9); COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 10, 1, 10); COMPARE_TOKEN(ts, 9, Parser::Token_LNUMBER, 1, 11, 1, 11); COMPARE_TOKEN(ts, 10, Parser::Token_SEMICOLON, 1, 12, 1, 12); COMPARE_TOKEN(ts, 11, Parser::Token_WHITESPACE, 1, 13, 1, 13); COMPARE_TOKEN(ts, 12, Parser::Token_RBRACE, 1, 14, 1, 14); COMPARE_TOKEN(ts, 13, Parser::Token_WHITESPACE, 1, 15, 1, 15); COMPARE_TOKEN(ts, 14, Parser::Token_FINALLY, 1, 16, 1, 22); COMPARE_TOKEN(ts, 15, Parser::Token_WHITESPACE, 1, 23, 1, 23); COMPARE_TOKEN(ts, 16, Parser::Token_LBRACE, 1, 24, 1, 24); COMPARE_TOKEN(ts, 17, Parser::Token_WHITESPACE, 1, 25, 1, 25); COMPARE_TOKEN(ts, 18, Parser::Token_RBRACE, 1, 26, 1, 26); } void LexerTest::testEllipsis() { QScopedPointer ts(tokenize(QStringLiteral("size(), 11); COMPARE_TOKEN(ts, 0, Parser::Token_OPEN_TAG, 0, 0, 0, 5); COMPARE_TOKEN(ts, 1, Parser::Token_FUNCTION, 1, 0, 1, 7); COMPARE_TOKEN(ts, 2, Parser::Token_WHITESPACE, 1, 8, 1, 8); COMPARE_TOKEN(ts, 3, Parser::Token_STRING, 1, 9, 1, 11); COMPARE_TOKEN(ts, 4, Parser::Token_LPAREN, 1, 12, 1, 12); COMPARE_TOKEN(ts, 5, Parser::Token_ELLIPSIS, 1, 13, 1, 15); COMPARE_TOKEN(ts, 6, Parser::Token_VARIABLE, 1, 16, 1, 20); COMPARE_TOKEN(ts, 7, Parser::Token_RPAREN, 1, 21, 1, 21); COMPARE_TOKEN(ts, 8, Parser::Token_WHITESPACE, 1, 22, 1, 22); COMPARE_TOKEN(ts, 9, Parser::Token_LBRACE, 1, 23, 1, 23); COMPARE_TOKEN(ts, 10, Parser::Token_RBRACE, 1, 24, 1, 24); } TokenStream* LexerTest::tokenize(const QString& unit, bool debug, int initialState) { TokenStream* tokenStream = new TokenStream; Lexer lexer(tokenStream, unit, initialState); int token; int i = 0; QList tokens; while ((token = lexer.nextTokenKind())) { Parser::Token &t = tokenStream->push(); t.begin = lexer.tokenBegin(); t.end = lexer.tokenEnd(); t.kind = token; tokens << t; } if (debug) { foreach(const Parser::Token &t, tokens) { qint64 beginLine; qint64 beginColumn; tokenStream->startPosition(i, &beginLine, &beginColumn); qint64 endLine; qint64 endColumn; tokenStream->endPosition(i, &endLine, &endColumn); qDebug() << tokenText(t.kind) << unit.mid(t.begin, t.end - t.begin + 1).replace('\n', QLatin1String("\\n")) << QStringLiteral("[%0-%1] - [%2-%3]").arg(beginLine).arg(beginColumn).arg(endLine).arg(endColumn); ++i; } } return tokenStream; } } diff --git a/parser/test/lexertest.h b/parser/test/lexertest.h index db73d22..980b08b 100644 --- a/parser/test/lexertest.h +++ b/parser/test/lexertest.h @@ -1,79 +1,80 @@ /* This file is part of KDevelop Copyright 2006 Hamish Rodda Copyright 2008 Niko Sams This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef LEXERTEST_H #define LEXERTEST_H #include #include #include #include "phplexer.h" namespace KDevelop { class TopDUContext; } namespace Php { class TokenStream; class LexerTest : public QObject { Q_OBJECT public: LexerTest(); private slots: void testOpenTagWithNewline(); void testOpenTagWithSpace(); void testCommentOneLine(); void testCommentOneLine2(); void testCommentMultiLine(); void testCommentMultiLine2(); void testEndTag(); void testNewlineInString(); void testNewlineInString2(); void testNewlineInStringWithVar(); void testNewlineInStringWithVar2(); void testNewlineInStringWithVar3(); void testMultiplePhpSections(); void testHereDoc(); void testHereDocQuoted(); void testNowdoc(); void testCommonStringTokens(); void testNonTerminatedStringWithVar(); void testPhpBlockWithComment(); void testNamespaces(); void testCloseTagInComment(); void testBinaryNumber(); void testHexadecimalNumber(); void testTypeHintsOnFunction(); + void testReturnTypeHints(); void testExponentiation(); void testExceptionFinally(); void testEllipsis(); protected: TokenStream* tokenize(const QString& unit, bool debug = false, int initialState = Lexer::HtmlState); }; } #endif // LEXERTEST_H