diff --git a/golangparsejob.cpp b/golangparsejob.cpp index 48061b8..d796b46 100644 --- a/golangparsejob.cpp +++ b/golangparsejob.cpp @@ -1,232 +1,233 @@ /************************************************************************************* * Copyright (C) 2014 by Pavel Petrushkov * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * * as published by the Free Software Foundation; either version 2 * * of the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * *************************************************************************************/ #include "golangparsejob.h" #include #include #include #include #include #include #include #include #include #include #include #include "parsesession.h" #include "duchain/builders/declarationbuilder.h" #include "duchain/builders/usebuilder.h" #include "duchain/helper.h" #include "godebug.h" using namespace KDevelop; QHash GoParseJob::canonicalImports; GoParseJob::GoParseJob(const KDevelop::IndexedString& url, KDevelop::ILanguageSupport* languageSupport): ParseJob(url, languageSupport) { } void GoParseJob::run(ThreadWeaver::JobPointer self, ThreadWeaver::Thread *thread) { qCDebug(Go) << "GoParseJob succesfully created for document " << document(); UrlParseLock urlLock(document()); if (abortRequested() || !isUpdateRequired(ParseSession::languageString())) { return; } ProblemPointer p = readContents(); if(p) { return; } QByteArray code = contents().contents; while(code.endsWith('\0')) code.chop(1); //ParseSession session(QString(contents().contents).toUtf8(), priority()); ParseSession session(code, parsePriority()); session.setCurrentDocument(document()); session.setFeatures(minimumFeatures()); if(abortRequested()) return; ReferencedTopDUContext context; { DUChainReadLocker lock; context = DUChainUtils::standardContextForUrl(document().toUrl()); } //ParsingEnvironmentFilePointer filep = context->parsingEnvironmentFile(); if (context) { translateDUChainToRevision(context); context->setRange(RangeInRevision(0, 0, INT_MAX, INT_MAX)); } qCDebug(Go) << "Job features: " << minimumFeatures(); qCDebug(Go) << "Job priority: " << parsePriority(); qCDebug(Go) << document(); bool result = session.startParsing(); //this is useful for testing parser, comment it for actual working //Q_ASSERT(result); //When switching between files(even if they are not modified) KDevelop decides they need to be updated //and calls parseJob with VisibleDeclarations feature //so for now feature, identifying import will be AllDeclarationsAndContexts, without Uses bool forExport = false; if((minimumFeatures() & TopDUContext::AllDeclarationsContextsAndUses) == TopDUContext::AllDeclarationsAndContexts) forExport = true; //qCDebug(Go) << contents().contents; if(!forExport) session.setIncludePaths(go::Helper::getSearchPaths(document().toUrl())); else session.setIncludePaths(go::Helper::getSearchPaths()); if(canonicalImports.empty()) parseCanonicalImports(); session.setCanonicalImports(&canonicalImports); if(result) { QReadLocker parseLock(languageSupport()->parseLock()); if(abortRequested()) return abortJob(); //qCDebug(Go) << QString(contents().contents); DeclarationBuilder builder(&session, forExport); context = builder.build(document(), session.ast(), context); if(!forExport) { go::UseBuilder useBuilder(&session); useBuilder.buildUses(session.ast()); } //this notifies other opened files of changes //session.reparseImporters(context); } if(!context){ DUChainWriteLocker lock; ParsingEnvironmentFile* file = new ParsingEnvironmentFile(document()); file->setLanguage(ParseSession::languageString()); context = new TopDUContext(document(), RangeInRevision(0, 0, INT_MAX, INT_MAX), file); DUChain::self()->addDocumentChain(context); } setDuChain(context); { DUChainWriteLocker lock; + context->setProblems(session.problems()); context->setFeatures(minimumFeatures()); ParsingEnvironmentFilePointer file = context->parsingEnvironmentFile(); Q_ASSERT(file); DUChain::self()->updateContextEnvironment(context->topContext(), file.data()); } highlightDUChain(); DUChain::self()->emitUpdateReady(document(), duChain()); if(result) qCDebug(Go) << "===Success===" << document().str(); else qCDebug(Go) << "===Failed===" << document().str(); } void GoParseJob::parseCanonicalImports() { QList importPaths = go::Helper::getSearchPaths(); for(const QString& path : importPaths) { QDirIterator iterator(path, QDir::Dirs | QDir::NoDotAndDotDot, QDirIterator::Subdirectories); while(iterator.hasNext()) { iterator.next(); QDir dir(iterator.filePath()); for(const QString& file : dir.entryList(QStringList("*.go"), QDir::Files | QDir::NoSymLinks)) { QFile f(dir.filePath(file)); f.open(QIODevice::ReadOnly); QByteArray contents = f.readAll(); f.close(); QString canonicalImport = extractCanonicalImport(QString(contents)); if(canonicalImport.length() != 0) { qCDebug(Go) << "Found canonical import for package " << iterator.filePath() << " import: " << canonicalImport; canonicalImports[canonicalImport] = iterator.filePath(); break; } } } } //if no canonical imports were found add stab value to map //so we won't search for them again if(canonicalImports.empty()) canonicalImports[""] = QString("none"); } QString GoParseJob::extractCanonicalImport(QString string) { int i = 0; while(i < string.length()) { if(string[i].isSpace()) i++; else if(string[i] == '/') { if(i + 1 < string.length() && string[i+1] == '/') { i += 2; while(i -- This program is free software; you can redistribute it and/or modify -- it under the terms of the GNU Library General Public License as -- published by the Free Software Foundation; either version 2 of the -- License, or (at your option) any later version. -- -- This program is distributed in the hope that it will be useful, -- but WITHOUT ANY WARRANTY; without even the implied warranty of -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- GNU General Public License for more details. -- -- You should have received a copy of the GNU Library General Public -- License along with this program; if not, write to the -- Free Software Foundation, Inc., -- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ----------------------------------------------------------------------------- --This grammar is based on official Go language specification http://golang.org/ref/spec --All first/first and first/follow conflicts are hidden behind try/rollbacks and LA's --I was trying to minimize usage of try/rollbacks, so some rules are very ugly for this reason --Before changing grammar consider contacting me first at onehundredof@gmail.com, or at least --run parser tests and test it manually on Go standart library --@Warning because this grammar uses some manual flags(inIfClause and inSwitchTypeClause) be careful with changing try/rollback order, --because when parser fails and rollbacks to catch statement flags may not be reset [: namespace KDevelop { class DUContext; } :] %token_stream Lexer; %input_encoding "UTF-8" %input_stream "KDevPG::QByteArrayIterator" %lexer_bits_header "QDebug" %parser_bits_header "QDebug" %parser_declaration_header "language/duchain/duchain.h" +%parser_declaration_header "language/duchain/problem.h" %export_macro "KDEVGOPARSER_EXPORT" %export_macro_header "kdevgoparser_export.h" %ast_extra_members [: KDevelop::DUContext* ducontext; :] +%parserclass (public declaration) +[: + enum ProblemType { + Error, + Warning, + Info, + Todo + }; + void setCurrentDocument(KDevelop::IndexedString url); + KDevelop::ProblemPointer reportProblem( Parser::ProblemType type, const QString& message, int tokenOffset = -1 ); + QList problems() { + return m_problems; + } +:] + %parserclass (private declaration) [: struct ParserState { }; ParserState m_state; + KDevelop::IndexedString m_currentDocument; + QList m_problems; qint64 lparenCount=0; bool inIfClause = false; bool inSwitchTypeClause = false; :] %token IDENT, INTEGER, FLOAT, COMPLEX, RUNE, STRING, TEST;; %token BREAK ("break"), CASE ("case"), CHAN ("chan"), CONST ("const"), CONTINUE ("continue"), DEFAULT ("default"), DEFER ("defer"), ELSE ("else"), FALLTHROUGH ("fallthrough"), FOR ("for"), FUNC ("func"), GO ("go"), GOTO ("goto"), IF ("if"), IMPORT ("import"), INTERFACE ("interface"), MAP ("map"), PACKAGE ("package"), RANGE ("range"), RETURN ("return"), SELECT ("select"), STRUCT ("struct"), SWITCH("switch"), TYPE ("type"), VAR ("var");; %token PLUS ("+"), AMPERSAND ("&"), PLUSEQUAL("+="), AMPEREQUAL ("&="), LOGICALAND ("&&"), ISEQUAL ("=="), ISNOTEQUAL ("!="), LPAREN ("("), RPAREN (")"), MINUS ("-"), BITWISEOR ("|"), MINUSEQUAL ("-="), OREQUAL ("|="), LOGICALOR ("||"), LESS ("<"), LESSOREQUAL ("<="), LBRACKET ("["), RBRACKET ("]"), STAR ("*"), HAT ("^"), MULTIPLYEQUAL ("*="), XOREQUAL ("^="), LEFTCHAN ("<-"), GREATER (">"), GREATEROREQUAL (">="), LBRACE ("{"), RBRACE ("}"), DIVIDE ("/"), LEFTSHIFT ("<<"), DIVIDEEQUAL ("/="), LEFTSHIFTEQUAL ("<<="), PLUSPLUS ("++"), ASSIGN ("="), AUTOASSIGN (":="), COMMA (","), SEMICOLON (";"), MOD ("%"), RIGHTSHIFT (">>"), MODEQUAL ("%="), RIGHTSHIFTEQUAL (">>="), MINUSMINUS ("--"), BANG("!"), TRIPLEDOT ("..."), DOT ("."), COLON (":"), AMPERXOR ("&^"), AMPERXOREQUAL ("&^=");; %lexer -> ------------------------------------------------------------ --comments -- one liner "//"[.^\n]* [: /*locationTable()->newline(lxCURR_IDX);*/ :] ; -- multi line "/*" [: /*std::cout << "entering comment\n";*/ lxSET_RULE_SET(incomment); :] ; --broken -- "/*"[.^"/*"]*"*/" [: std::cout << "multiline comment\n"; :] ; -- /\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+/ [: /*std::cout << "multiline comment\n";*/ :] ; ------------------------------------------------------------ --keywords "break" BREAK; "case" CASE; "chan" CHAN; "const" CONST; "continue" CONTINUE; "default" DEFAULT; "defer" DEFER; "else" ELSE; "fallthrough" FALLTHROUGH; "for" FOR; "func" FUNC; "go" GO; "goto" GOTO; "if" IF; "import" IMPORT; "interface" INTERFACE; "map" MAP; "package" PACKAGE; "range" RANGE; "return" RETURN; "select" SELECT; "struct" STRUCT; "switch" SWITCH; "type" TYPE; "var" VAR; ------------------------------------------------------------------ -- operators "+" PLUS; "&" AMPERSAND; "+=" PLUSEQUAL; "&=" AMPEREQUAL; "&&" LOGICALAND; "==" ISEQUAL; "!=" ISNOTEQUAL; "(" LPAREN; ")" RPAREN; "-" MINUS; "|" BITWISEOR; "-=" MINUSEQUAL; "|=" OREQUAL; "||" LOGICALOR; "<" LESS; "<=" LESSOREQUAL; "[" LBRACKET; "]" RBRACKET; "*" STAR; "^" HAT; "*=" MULTIPLYEQUAL; "^=" XOREQUAL; "<-" LEFTCHAN; ">" GREATER; ">=" GREATEROREQUAL; "{" LBRACE; --@TODO Go language specification allows omitting semicolons before closing ")" or "}" --so expressions like struct{ x, y float64 } are legal --I'm not sure how exactly these rules should be applied so for now I just --insert semicolon in the same conditions as newline --another example is { for {} } or {return "", 0 } --even though "}" cannot be first token in correct Go program --it can occur in completion snippet, so we check if size() > 0. --see labeledStmt parser rule if you wonder why 'prevkind == Token_COLON' is here. "}" [: if(size() > 0) { int prevkind = at(size()-1).kind; if(prevkind == Token_IDENT || prevkind == Token_INTEGER || prevkind == Token_FLOAT || prevkind == Token_COMPLEX || prevkind == Token_RUNE || prevkind == Token_STRING || prevkind == Token_BREAK || prevkind == Token_CONTINUE || prevkind == Token_FALLTHROUGH || prevkind == Token_RETURN || prevkind == Token_PLUSPLUS || prevkind == Token_MINUSMINUS || prevkind == Token_RPAREN || prevkind == Token_RBRACKET || prevkind == Token_RBRACE || prevkind == Token_COLON) lxTOKEN(SEMICOLON); } :] RBRACE; "/" DIVIDE; "<<" LEFTSHIFT; "/=" DIVIDEEQUAL; "<<=" LEFTSHIFTEQUAL; "++" PLUSPLUS; "=" ASSIGN; ":=" AUTOASSIGN; "," COMMA; ";" SEMICOLON; "%" MOD; ">>" RIGHTSHIFT; "%=" MODEQUAL; ">>=" RIGHTSHIFTEQUAL; "--" MINUSMINUS; "!" BANG; "..." TRIPLEDOT; "." DOT; ":" COLON; "&^" AMPERXOR; "&^=" AMPERXOREQUAL; --identifier--------------------------------------------------- ({alphabetic}|"_")({alphabetic}|[0-9]|"_")* IDENT; --integers----------------------------------------------------- ([1-9][0-9]*)|("0"[0-7]*)|("0"("x"|"X")[0-9a-fA-F]+) INTEGER; --floats------------------------------------------------------- (("e"|"E")("+"|"-")[0-9]+)|(("e"|"E")[0-9]+) -> exponent; ([0-9]+"."[0-9]*) | ([0-9]+"."[0-9]*{exponent}) | ([0-9]+{exponent}) | ("." [0-9]+) | ("."[0-9]+{exponent}) -> float_literal; {float_literal} FLOAT; --complex numbers------------------------------------------------------- ([0-9]+ | {float_literal})"i" COMPLEX; --rune literals--------------------------------------------------------- --apparently '\\' should be parsed as rune which we will have to write as "\\\\" --for some reason these runes don't work without this rules ("\'\\\\\'") | ("\'\\\'\'") | ("\'\\\"\'") RUNE; ("\\"[0-7][0-7][0-7]) | ("\\x"[0-9a-fA-F][0-9a-fA-F]) -> bbyte_value; ("\\u"[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]) | ("\\U"[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F] [0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]) | ("\\a")|("\\b")|("\\f")|("\\n")|("\\r")|("\\t")|("\\v")|("\\\\")|("\\\'")|("\\\"") | ({alphabetic} | {num} | [.^\'] ) -> unicode_value; --because of problems with utf-8 add this rule for now --disregard that, utf8 works fine [.^\']* -> utf8helper; "\'"({bbyte_value}|{unicode_value})"\'" RUNE; -- "\'"({utf8helper})"\'" RUNE; --strings---------------------------------------------------------------- --This works well ( [.^(\" | \\ | \n)] | \\.) ->string_internal; -- ( [.^(\` | \\)] | \\.) ->string_internal2; --I'm not sure if you can escape backticks ("\"" ({string_internal})* "\"") STRING; "\`" [: lxSET_RULE_SET(instring); lxCONTINUE; :]; --broken -- ("\""[.^\"]*"\"") | ("\`"[.^\`]*"\`") STRING; -- ("\"" ([.^\"] | "\\\"")* "\"") | ("\`" ([.^\`] | "\\\`" ) *"\`") STRING; -- @TODO I'm not really sure but in src/cmd/go/main.go there are `\` and `/` expressions -- Does this mean you can't escape backtick? --This regexp below failed to recognize strings like "\\" -- ("\"" ([.^\"] | "\\\"")* "\"") | ("\`" [.^\`] *"\`") STRING; --blanks, tabs and newlines---------------------------------------------- [\ \t]+ [: /* blank */ :] ; -- Go language specification requires inserting semicolons when met this conditions: \n [: locationTable()->newline(lxCURR_IDX); if(size() >= 1) { int prevkind = at(size()-1).kind; /*qint64 prevline, prevcol; endPosition(size()-1,&prevline, &prevcol); */ /*std::cout << "new line detected:" << at(size()-1).kind << prevline << std::endl;*/ if(prevkind == Token_IDENT || prevkind == Token_INTEGER || prevkind == Token_FLOAT || prevkind == Token_COMPLEX || prevkind == Token_RUNE || prevkind == Token_STRING || prevkind == Token_BREAK || prevkind == Token_CONTINUE || prevkind == Token_FALLTHROUGH || prevkind == Token_RETURN || prevkind == Token_PLUSPLUS || prevkind == Token_MINUSMINUS || prevkind == Token_RPAREN || prevkind == Token_RBRACKET || prevkind == Token_RBRACE) lxRETURN(SEMICOLON); } :] ; --output TEST on an unknown symbol so Lexer wouldn't crash . TEST; ; --special rule set for multilined comments %lexer "incomment" -> "*/" [: /*std::cout << "leaving comment\n";*/ lxSET_RULE_SET(start); :] ; \n [: locationTable()->newline(lxCURR_IDX); :] ; . ; ; %lexer "instring" -> \n [: locationTable()->newline(lxCURR_IDX); lxCONTINUE; :] ; "\`" [: lxSET_RULE_SET(start); :] STRING; . [: lxCONTINUE; :]; ; ------------------------------------------------------------------------- ------------------------------------------------------------------------- ------------------------------------------------------------------------- ----------------------GRAMMAR-------------------------------------------- ------------------------------------------------------------------------- ------------------------------------------------------------------------- sourceFile=sourceFile -> start;; --Types----------------------------------------------------------------- ------------------------------------------------------------------------ ------------------------------------------------------------------------ --resolving FIRST/FIRST conflict in typeName: IDENT and qualifiedIdent (e.g. math.complex) intersect --type_resolve is part of qualifiedIdent DOT fullName=identifier | 0 ->type_resolve;; name=identifier type_resolve=type_resolve -> typeName;; typeName=typeName | parenType=parenType | complexType=complexType -> type;; LPAREN type=type RPAREN -> parenType;; --separation of typeName, parenType and complexType is desired for parameterList ( arrayOrSliceType=arrayOrSliceType | structType=structType | pointerType=pointerType | functionType=functionType | interfaceType=interfaceType | mapType=mapType | chanType=chanType ) -> complexType;; --Array and Slice------------------------------------------------------------------- --resolving FIRST/FIRST conflict in arrayType/sliceType (both start with "[") LBRACKET arrayOrSliceResolve=arraySliceResolve -> arrayOrSliceType;; RBRACKET slice=type -- <- this is slice | expression=expression RBRACKET array=type -- <- this is array -> arraySliceResolve;; --Struct--------------------------------------------------------------------------- STRUCT LBRACE (#fieldDecl=fieldDecl SEMICOLON)* RBRACE -> structType;; --to resolve first/first between field declaration and anonymous field: STAR anonFieldStar=anonymousField (tag=tag | 0) | varid=identifier ( (idList=idList | 0) type=type | DOT fullname=identifier | 0) (tag=tag | 0) -> fieldDecl;; typeName=typeName -> anonymousField;; --anonymous field starting with "*" STRING -> tag;; (COMMA #id=identifier)+ -> idList;; STAR type=type ->pointerType;; --Function Type----------------------------------------------------- --parsing parameter list is a bit weird, because Go allows putting a comma after all parameters, --like so: func(a int, b float64,) --so writing just "(identifier type) @ COMMA" won't work --another problem is anonymous parameters like: func(int, float64) and combined parameter names: func(a, b int) --when we encounter first identifier it is unclear whether it would be a parameter name --or parameter type, we will have to decide that later FUNC signature=signature -> functionType;; parameters=parameters (result=result | 0) -> signature;; parameters=parameters | typeName=typeName | complexType=complexType -> result;; LPAREN (parameter=parameter (COMMA (#parameterList=parameter | 0))* | 0 ) RPAREN ->parameters;; complexType=complexType | parenType=parenType | TRIPLEDOT unnamedvartype=type | idOrType=identifier ( type=type | TRIPLEDOT vartype=type | DOT fulltype=identifier | 0 ) ->parameter;; --Interfaces------------------------------------------------------------ INTERFACE LBRACE (#methodSpec=methodSpec SEMICOLON)* RBRACE -> interfaceType;; -- Read(a) fmt.Reader Read <-interface name methodName=identifier (signature=signature | DOT fullName=identifier | 0) ->methodSpec;; --Map Type-------------------------------------------------------------- MAP LBRACKET keyType=type RBRACKET elemType=type -> mapType;; --Channel Type---------------------------------------------------------- --Actually grammar here is ambiguous --e.g. chan <- chan. Is that channel to sending channel or receiving channel to channel? --language associates leftmost <- hence the order of tokens in the next rule: CHAN (send=LEFTCHAN | 0 ) rtype=type | LEFTCHAN CHAN stype=type -> chanType;; --Declarations---------------------------------------------------------- ------------------------------------------------------------------------ ------------------------------------------------------------------------ ( declaration=declaration | FUNC (funcDecl=funcDeclaration | methodDecl=methodDeclaration) ) -> topLevelDeclaration;; ( constDecl=constDecl | typeDecl=typeDecl | varDecl=varDecl ) -> declaration;; --Const Declarations---------------------------------------------------- CONST ( constSpec=constSpec | LPAREN (#constSpecList=constSpec SEMICOLON)* RPAREN ) -> constDecl;; id=identifier (idList=idList | 0) ((type=type | 0) ASSIGN expression=expression (expressionList=expressionList | 0) | 0) -> constSpec;; --Type Declarations----------------------------------------------------- TYPE ( typeSpec=typeSpec | LPAREN (#typeSpecList=typeSpec SEMICOLON)* RPAREN ) -> typeDecl;; name=identifier type=type -> typeSpec;; --Var Declarations------------------------------------------------------ VAR ( varSpec=varSpec | LPAREN (#varSpecList=varSpec SEMICOLON)* RPAREN) -> varDecl;; id=identifier (idList=idList | 0) (type=type (ASSIGN expression=expression ( expressionList=expressionList | 0 ) | 0) | ASSIGN expression=expression ( expressionList=expressionList | 0 ) ) -> varSpec;; --Func Declaration------------------------------------------------------- funcName=identifier signature=signature (body=block | 0) -> funcDeclaration;; --Method Declaration----------------------------------------------------- methodRecv=methodRecv methodName=identifier signature=signature (body=block | 0) ->methodDeclaration;; LPAREN ( nameOrType=identifier (star=STAR | 0) (type=identifier | 0) | star=STAR ptype=identifier ) RPAREN ->methodRecv;; --Expressions--------------------------------------------------------------------------- ---------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------- --Operand------------------------------------------------------------------------------- --@Warning Original Go Language specification contains operand rules(see http://golang.org/ref/spec#Operands), --but I decided to go with potentially horrible decision: --eliminate operand rules and do one huge rule in Primary Expression --Idea here is that operand only used by primary expression and it has --many overlaps(first/first) with primary expression, so it's better --to look at everything in primary expression itself rather than trying LA and rollbacks --Basic literals(integers, strings...) integer=INTEGER | flt=FLOAT | cpx=COMPLEX | rune=RUNE | string=STRING -> basicLit;; --what is this? --id=IDENT type_resolve=type_resolve --->operandName;; --Composite Literals --@Warning @TODO semicolon is not in language specification --I added it because I insert semicolon before closing } --because language specification allows omitting it(see SEMICOLON lexer rule) LBRACE (element=element | 0) (COMMA (#elements=element | 0))* (SEMICOLON | 0) RBRACE -> literalValue;; --if you look at language specification this may be confusing --point here is that when parser sees IDENT(or Expression) --it cannot decide without LA if it's key or value --so we just save whatever there is and when traversing AST later --we decide what is what by looking at value node keyOrValue=keyOrValue (COLON value=value | 0 ) -> element;; indexOrValue=expression | literalValue=literalValue --Go 1.5 feature - literal values in map key -> keyOrValue;; expValue=expression | literalValue=literalValue -> value;; --Primary Expressions------------------------------------------------------- --if after looking for some time at next rule you want to ask why couldn't I just write type instead of all type redeclarations, --that's because literals of some types are impossible(like interface) --first/first with parenthesized expression and type conversion resolved with try/rollback --about inIfClause and lparenCount see ifStmt rule --unmatched lparenCount are there becase of try/rollback --TODO should I add lparenCount++ after LBRACKET in array/slice conversion/literal? --that may mess up things like: if [mystruct{3}]int{10} { } --(parenType convArgument) can accidentaly parse things like (funcname)(single_argument) where funcname is function name instead of type id=identifier ( ?[: !inIfClause || (inIfClause && lparenCount > 0) :] literalValue=literalValue | LPAREN [: if(inIfClause) lparenCount++; :] callOrBuiltinParam=callOrBuiltinParam [: if(inIfClause) lparenCount--; :] RPAREN | 0) ( primaryExprResolve=primaryExprResolve | 0 ) --named literal | basicLit=basicLit ( primaryExprResolve=primaryExprResolve | 0 ) --(primaryExprResolve=primaryExprResolve | 0) not sure why I wrote this twice --basic literal | structType=structType ( literalValue=literalValue | convArg=conversionArgument ) (primaryExprResolve=primaryExprResolve | 0) --struct conversion/literal | array=LBRACKET ( arrayOrSliceResolve=arraySliceResolve (literalValue=literalValue | convArg=conversionArgument) | tripledot=TRIPLEDOT RBRACKET element=type literalValue=literalValue )(primaryExprResolve=primaryExprResolve | 0) --array/slice conversion/literal | mapType=mapType ( literalValue=literalValue | convArg=conversionArgument )(primaryExprResolve=primaryExprResolve | 0) --map type conversion/literal | FUNC signature=signature (body=block | convArg=conversionArgument )(primaryExprResolve=primaryExprResolve | 0) --func type conversion/literal | pointerType=pointerType convArg=conversionArgument (primaryExprResolve=primaryExprResolve | 0) --pointer type conversion | interfaceType=interfaceType convArg=conversionArgument (primaryExprResolve=primaryExprResolve | 0) --interface type conversion | chanType=chanType convArg=conversionArgument (primaryExprResolve=primaryExprResolve | 0) --chan type conversion | try/rollback( parenType=parenType convArg=conversionArgument ) --parenthesized type conversion catch(LPAREN [: if(inIfClause) lparenCount++; :] expression=expression [: if(inIfClause) lparenCount--; :] RPAREN ) (primaryExprResolve=primaryExprResolve | 0) -> primaryExpr;; LPAREN [: if(inIfClause) lparenCount++; :] expression=expression (COMMA | 0) [: if(inIfClause) lparenCount--; :] RPAREN ->conversionArgument;; --allow first argument to be 'type' so builtin functions could work --TripleDot is to allow special form of append call: append(s1, s2...) try/rollback(expression=expression) catch(type=type) (COMMA (#expressions=expression | 0))* (tripleDot=TRIPLEDOT | 0) | 0 ->callOrBuiltinParam;; --@TODO apparently not only in append there can be such construction --e.g. go/cmd/go/test.go:926 exec.Command(args[0], args[1:]...) expression=expression (COMMA (#expressions=expression | 0))* (tripleDot=TRIPLEDOT | 0) | 0 ->callParam;; --first is selector/type typeAssertion --second is array/slice index --inSwitchTypeClause is true if we are looking for type switch statement ( switch x.(type) {} ) --in which case we must prevent primaryExpression from eating up extra DOT (which will fail later anyway) --another bug is complex-named literals like package.typename{} --the way we deal with that is add literalValue to this primaryExprResolve rule --inIfClause makes sure rule isn't invoked when parsing if clauses ( [: if(inSwitchTypeClause && LA(1).kind==Token_DOT && LA(2).kind==Token_LPAREN && LA(3).kind==Token_TYPE) { (*yynode)->endToken = tokenStream->index() - 2; return true;} :] DOT (selector=identifier | LPAREN typeAssertion=type RPAREN ) (primaryExprResolve=primaryExprResolve | 0 ) ) --increase lParenCounter within brackets to allow literals within brackets in if clauses | index=LBRACKET [: if(inIfClause) lparenCount++; :] ( ( low=expression (colon=COLON | 0) | colon=COLON ) ( high=expression (COLON max=expression | 0) | 0 )) [: if(inIfClause) lparenCount--; :] RBRACKET (primaryExprResolve=primaryExprResolve | 0) | LPAREN [: if(inIfClause) lparenCount++; :] callParam=callParam [: if(inIfClause) lparenCount--; :] RPAREN (primaryExprResolve=primaryExprResolve | 0) | [: if(inIfClause && lparenCount <= 0) { (*yynode)->endToken = tokenStream->index() - 2; return true;} :] literalValue=literalValue (primaryExprResolve=primaryExprResolve | 0) --@TODO this isn't in language specification. It's a way to deal with complex-named literals e.g. "a.b{1}" -> primaryExprResolve;; --Expression------------------------------------------------------------------------ ------------------------------------------------------------------------------------ unaryExpression=unaryExpression ( binary_op=binary_op expression=expression | 0) -> expression;; -- * and <- should be parsed first as operators try/rollback (unsafe_unary_op=unsafe_unary_op unaryExpression=unaryExpression ) catch( primaryExpr=primaryExpr | unary_op=unary_op unaryExpression=unaryExpression ) -> unaryExpression;; plus=PLUS | minus=MINUS | bang=BANG | hat=HAT | ampersand=AMPERSAND -> unary_op;; --unsafe here means it's conflicting with primaryExpression and may trigger rollback star=STAR | leftchan=LEFTCHAN -> unsafe_unary_op;; logicalor=LOGICALOR | logicaland=LOGICALAND | rel_op=rel_op | add_op=add_op | mul_op=mul_op -> binary_op;; isequal=ISEQUAL | isnotequal=ISNOTEQUAL | less=LESS | lessorequal=LESSOREQUAL | greater=GREATER | greaterorequal=GREATEROREQUAL -> rel_op;; plus=PLUS | minus=MINUS | bitwiseor=BITWISEOR | hat=HAT -> add_op;; star=STAR | divide=DIVIDE | mod=MOD | leftshift=LEFTSHIFT | rightshift=RIGHTSHIFT | ampersand=AMPERSAND | amperxor=AMPERXOR -> mul_op;; --Statements----------------------------------------------------------------- ------------------------------------------------------------------------ ------------------------------------------------------------------------ LBRACE (statements=statements | 0) RBRACE -> block;; (#statement=statement SEMICOLON)* -> statements;; --conflicts: --FIRST/FIRST between simpleStmt.expression and labeledStmt (IDENT) - resolved by try/rollback --@TODO consider resolving ^ this conflict with LA for better performance --FIRST/FIRST between simpleStmt.expression and shortVarDecl (IDENT) - resolved by try/rollback --@Warning there is a dangerous bug with the try/rollback usage --if try/rollback expression succedes catch rule won't be evaluated even if parser fails later --this means if catch expression is sub-expression of try/rollback expression then --we are likely to hit this bug --example: a simple IDENT is a valid simpleStmt, so if we have a labeledStmt in the catch --we will parse a simpleStmt and then fail looking for ";" instead of ":" declaration=declaration | ifStmt=ifStmt | switchStmt=switchStmt | forStmt=forStmt | goStmt=goStmt | selectStmt=selectStmt | ( returnStmt=returnStmt | breakStmt=breakStmt | continueStmt=continueStmt | gotoStmt=gotoStmt) | ( fallthroughStmt=fallthroughStmt | deferStmt=deferStmt ) | block=block | try/rollback(labeledStmt=labeledStmt) catch(simpleStmt=simpleStmt) | 0 --Empty statement is part of language specification -> statement;; --apparently it is legal in go to write: " ... -- label: -- }" --which is a labeled statement with an empty statement, but since lexer does not insert semicolons when line ends on COLON --last statement is left without any semicolon. --Therefore (hopefully) temporary workaround is to insert semicolons before '}' and after ':'. See lexer rule for '}'. label=identifier COLON statement=statement -> labeledStmt;; try/rollback(shortVarDecl=shortVarDecl) catch( expression=expression ( simpleStmtResolve=simpleStmtResolve | 0 )) -> simpleStmt;; LEFTCHAN expression=expression | increment=PLUSPLUS | decrement=MINUSMINUS | ( assigneeList=expressionList | 0 ) assign_op=assign_op expression=expression ( assignmentList=expressionList | 0 ) ->simpleStmtResolve;; (COMMA #expressions=expression)+ ->expressionList;; --Language specification says (add_op | mul_op ) ASSIGN --which I think is incorrect since we specifically for this case have things like PLUSEQUAL and so on... --and obviously lexer will match these tokens rather than PLUS and EQUAL separately ( plusequal=PLUSEQUAL | minusequal=MINUSEQUAL | orequal=OREQUAL | xorequal=XOREQUAL ) | ( multiplyequal=MULTIPLYEQUAL | divideequal=DIVIDEEQUAL | modequal=MODEQUAL | leftshiftequal=LEFTSHIFTEQUAL ) | ( rightshigtequal=RIGHTSHIFTEQUAL | amperequal=AMPEREQUAL | amperxorequal=AMPERXOREQUAL | assign=ASSIGN ) -> assign_op;; --ShortVarDecl------------------------------------------------------------- id=identifier (idList=idList | 0) autoassign=AUTOASSIGN expression=expression (expressionList=expressionList | 0) -> shortVarDecl;; --If Statement------------------------------------------------------------ --in this and few next rules there should be a conflict between expecting condition(expression) or simple statement and then condition --it can't be resolved with LA because it's unknown where statement will start differing from expression --properly we should distinguish it with try/rollback or some smart rule --but I decided to parse whatever comes first as a simple statement since expression is also a simpleStmt --then if there is a ";" we parse actual condtion and parse block if not --that allows for succesful parsing of structures not supported by language --but I think this small inconsistency is worth preformance we gain from not using try/rollback(or is it not?) --About the thing with all this lparenCounts : --When parsing If (and some others) statements there appears parsing ambiguity with lbraces --when we see opening brace after identifier it is unclear if it's composite literal or start of a body block --therefore language specification recommends putting composite literals in if clauses in parenthesis --see details at http://golang.org/ref/spec#Composite_literals --old way --ifToken=IF try/rollback( [: lparenCount=0; inIfClause=true; :] simpleStmt=simpleStmt SEMICOLON expression=expression --[: lparenCount=0; inIfClause=false; :] ifblock=block) --catch([: lparenCount=0; inIfClause=true; :] expression=expression --[: lparenCount=0; inIfClause=false; :] ifblock=block) --( elseToken=ELSE (ifStmt=ifStmt | elseblock=block) | 0 ) ---> ifStmt;; ifToken=IF [: lparenCount=0; inIfClause=true; :] stmtOrCondition=simpleStmt ( SEMICOLON condition=expression | 0) [: lparenCount=0; inIfClause=false; :] ifblock=block ( elseToken=ELSE (ifStmt=ifStmt | elseblock=block) | 0 ) -> ifStmt;; --Switch Statement------------------------------------------------------------- try/rollback(exprSwitchStatement=exprSwitchStatement) catch(typeSwitchStatement=typeSwitchStatement) -> switchStmt;; SWITCH [: lparenCount=0; inIfClause=true; :] ( stmtOrCondition=simpleStmt | 0) ( SEMICOLON (condition=expression | 0) | 0) [: lparenCount=0; inIfClause=false; :] LBRACE #exprCaseClause=exprCaseClause* RBRACE ->exprSwitchStatement;; ( caseToken=CASE expression=expression (expressionList=expressionList |0) | defaultToken=DEFAULT ) COLON statements=statements ->exprCaseClause;; SWITCH try/rollback( [: lparenCount=0; inIfClause=true; :] simpleStmt=simpleStmt SEMICOLON typeSwitchGuard=typeSwitchGuard [: lparenCount=0; inIfClause=false; :] LBRACE #typeCaseClause=typeCaseClause* RBRACE) catch( [: lparenCount=0; inIfClause=true; :] typeSwitchGuard=typeSwitchGuard [: lparenCount=0; inIfClause=false; :] LBRACE #typeCaseClause=typeCaseClause* RBRACE) ->typeSwitchStatement;; (?[: (LA(1).kind==Token_IDENT && LA(2).kind==Token_AUTOASSIGN) :] ident=identifier AUTOASSIGN | 0) [: inSwitchTypeClause=true; :] primaryExpr=primaryExpr [: inSwitchTypeClause=false; :] DOT LPAREN TYPE RPAREN ->typeSwitchGuard;; ( caseToken=CASE (#typelist=type @ COMMA) | defaultToken=DEFAULT) COLON statements=statements ->typeCaseClause;; --For Statement---------------------------------------------------------------------- --try/rollback is here because range clause incorrectly parses as statement FOR try/rollback( [: lparenCount=0; inIfClause=true; :] ( initStmtOrCondition=simpleStmt | 0) ( semicolon=SEMICOLON (condition=expression | 0) SEMICOLON ( postStmt=simpleStmt | 0) --c-style for statement | 0 ) [: lparenCount=0; inIfClause=false; :] block=block) catch( [: lparenCount=0; inIfClause=true; :] ( expression=expression (expressionList=expressionList | 0) (ASSIGN | autoassign=AUTOASSIGN) | 0 ) range=RANGE rangeExpression=expression [: lparenCount=0; inIfClause=false; :] block=block) --range for statement -- block=block -> forStmt;; --go statements------------------------------------------------------------------- GO expression=expression -> goStmt;; --select statements---------------------------------------------------------------- SELECT LBRACE #commClause=commClause* RBRACE -> selectStmt;; commCase=commCase COLON statements=statements -> commClause;; defaultToken=DEFAULT | CASE sendOrRecv=expression ( (expressionList=expressionList | 0) (ASSIGN | autoassign=AUTOASSIGN) recvExp=expression | LEFTCHAN sendExp=expression | 0 ) -> commCase;; --Return statement----------------------------------------------------------------- RETURN ( expression=expression (expressionList=expressionList | 0) | 0) -> returnStmt;; --Break, continue, goto, fallthrough, defer statement----------------------------------------------------------------- BREAK (label=identifier | 0) -> breakStmt;; CONTINUE (label=identifier | 0) -> continueStmt;; GOTO label=identifier -> gotoStmt;; fallthrough=FALLTHROUGH -> fallthroughStmt;; DEFER expression=expression -> deferStmt;; --Packages------------------------------------------------------------------ --------------------------------------------------------------------------- --------------------------------------------------------------------------- packageClause=packageClause SEMICOLON (#importDecl=importDecl SEMICOLON)* (#topDeclarations=topLevelDeclaration SEMICOLON)* -> sourceFile;; PACKAGE packageName=identifier -> packageClause;; IMPORT ( importSpec=importSpec | LPAREN (#importSpecs=importSpec SEMICOLON)* RPAREN) -> importDecl;; (dot=DOT | packageName=identifier | 0) importpath=importpath -> importSpec;; id=IDENT -> identifier;; --to find importpath range easier import=STRING ->importpath;; [: +#include + namespace go { +KDevelop::ProblemPointer Parser::reportProblem( Parser::ProblemType type, const QString& message, int offset ) +{ + qint64 sLine; + qint64 sCol; + qint64 index = tokenStream->index() + offset; + if (index >= tokenStream->size()) { + return {}; + } + tokenStream->startPosition(index, &sLine, &sCol); + qint64 eLine; + qint64 eCol; + tokenStream->endPosition(index, &eLine, &eCol); + auto p = KDevelop::ProblemPointer(new KDevelop::Problem()); + p->setSource(KDevelop::IProblem::Parser); + switch ( type ) { + case Error: + p->setSeverity(KDevelop::IProblem::Error); + break; + case Warning: + p->setSeverity(KDevelop::IProblem::Warning); + break; + case Info: + p->setSeverity(KDevelop::IProblem::Hint); + break; + case Todo: + p->setSeverity(KDevelop::IProblem::Hint); + p->setSource(KDevelop::IProblem::ToDo); + break; + } + p->setDescription(message); + KTextEditor::Range range(sLine, sCol + 1, eLine, eCol + 2); + p->setFinalLocation(KDevelop::DocumentRange(m_currentDocument, range)); + m_problems << p; + return p; +} + +void Parser::setCurrentDocument(KDevelop::IndexedString document) +{ + m_currentDocument = document; +} void Parser::expectedSymbol(int /*expectedSymbol*/, const QString& name) { - qDebug() << "Expected: " << name; - //qDebug() << "Expected symbol: " << expectedSymbol; + qint64 line; + qint64 col; + qint64 index = tokenStream->index()-1; + Token &token = tokenStream->at(index); + tokenStream->startPosition(index, &line, &col); + qint64 eLine; + qint64 eCol; + tokenStream->endPosition(index, &eLine, &eCol); + reportProblem( Parser::Error, + QStringLiteral("Expected symbol \"%1\" (current token [%2] at %3:%4 - %5:%6)") + .arg(name) + .arg(token.kind) + .arg(line) + .arg(col) + .arg(eLine) + .arg(eCol)); } void Parser::expectedToken(int /*expected*/, qint64 /*where*/, const QString& name) { - qDebug() << "Expected token: " << name; + reportProblem(Parser::Error, QStringLiteral("Expected token \"%1\"").arg(name)); } Parser::ParserState* Parser::copyCurrentState() { return new ParserState(); } void Parser::restoreState( Parser::ParserState* state) { } } :] \ No newline at end of file diff --git a/parser/parsesession.cpp b/parser/parsesession.cpp index f70ce9f..cd33743 100644 --- a/parser/parsesession.cpp +++ b/parser/parsesession.cpp @@ -1,415 +1,421 @@ /************************************************************************************* * Copyright (C) 2014 by Pavel Petrushkov * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * * as published by the Free Software Foundation; either version 2 * * of the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * *************************************************************************************/ #include "parsesession.h" #include "parser/golexer.h" #include "parser/goparser.h" #include "parser/goast.h" #include "parser/gotokentext.h" #include #include #include +#include #include #include #include #include "kdev-pg-memory-pool.h" #include "kdev-pg-token-stream.h" using namespace KDevelop; ParseSession::ParseSession(const QByteArray& contents, int priority, bool appendWithNewline) : m_pool(new KDevPG::MemoryPool()), m_parser(new go::Parser()), //converting to and back from QString eliminates all the \000 at the end of contents //m_contents(QString(contents).toUtf8()), m_contents(contents), m_priority(priority), m_features(TopDUContext::AllDeclarationsAndContexts) { //appending with new line helps lexer to set correct semicolons //(lexer sets semicolons on newlines if some conditions are met because // in Go user can omit semicolons after statements, but grammar still needs them) // See more in Go Language Specification http://golang.org/ref/spec#Semicolons if(appendWithNewline) m_contents.append("\n"); KDevPG::QByteArrayIterator iter(m_contents); m_lexer = new go::Lexer(iter); m_parser->setMemoryPool(m_pool); m_parser->setTokenStream(m_lexer); forExport=false; } ParseSession::~ParseSession() { delete m_pool; delete m_parser; //delete m_iter; } KDevelop::IndexedString ParseSession::languageString() { static const KDevelop::IndexedString langString("go"); return langString; } bool ParseSession::startParsing() { if(!lex()) return false; return m_parser->parseStart(&m_ast); } bool ParseSession::lex() { KDevPG::Token token; int kind = go::Parser::Token_EOF; while((kind = m_lexer->read().kind) != go::Parser::Token_EOF) { //qDebug() << go::tokenText(kind); if(kind == go::Parser::Token_TEST) { qint64 line, column; m_lexer->locationTable()->positionAt(m_lexer->index(), &line, &column); qDebug() << "Lexer error at: " << line << " : " << column; return false; } } m_parser->rewind(0); return true; } bool ParseSession::parseExpression(go::ExpressionAst** node) { if(!lex()) return false; return m_parser->parseExpression(node); } go::StartAst* ParseSession::ast() { return m_ast; } QString ParseSession::symbol(qint64 index) { if(index >= m_lexer->size()) return QString(); return QString(m_contents.mid(m_lexer->at(index).begin, m_lexer->at(index).end - m_lexer->at(index).begin+1)); } KDevelop::RangeInRevision ParseSession::findRange(go::AstNode* from, go::AstNode* to) { qint64 line, column, lineEnd, columnEnd; m_lexer->locationTable()->positionAt(m_lexer->at(from->startToken).begin, &line, &column); m_lexer->locationTable()->positionAt(m_lexer->at(to->endToken).end+1, &lineEnd, &columnEnd); //for some reason I need to shift columns when not on a first line if(line != 0) column++; if(lineEnd != 0) columnEnd++; return KDevelop::RangeInRevision(KDevelop::CursorInRevision(line, column), KDevelop::CursorInRevision(lineEnd, columnEnd)); } KDevelop::IndexedString ParseSession::currentDocument() { return m_document; } void ParseSession::setCurrentDocument(const KDevelop::IndexedString& document) { m_document = document; + m_parser->setCurrentDocument(m_document); } /** * Currently priority order works in this way * -1: Direct imports of opened file * 0: opened files * ... * 99... imports of imports of imports.... * 99998: Imports of direct imports(needed to resolve types of some function) * 99999: Reparse of direct imports, after its imports are finished * 100000: reparse of opened file, after all recursive imports * layers higher than 99998 are NOT parsed right now because its too slow */ QList ParseSession::contextForImport(QString package) { package = package.mid(1, package.length()-2); QStringList files; //try canonical paths first if(m_canonicalImports && m_canonicalImports->contains(package)) { QDir path((*m_canonicalImports)[package]); if(path.exists()) { for(const QString& file : path.entryList(QStringList("*.go"), QDir::Files | QDir::NoSymLinks)) files.append(path.filePath(file)); } } if(files.empty()) { for(const QString& pathname : m_includePaths) { QDir path(pathname); if(path.exists() && path.cd(package)) { for(const QString& file : path.entryList(QStringList("*.go"), QDir::Files | QDir::NoSymLinks)) files.append(path.filePath(file)); break; } } } QList contexts; bool shouldReparse=false; //reduce priority if it is recursive import //int priority = forExport ? m_priority + 2 : m_priority - 1; int priority = BackgroundParser::WorstPriority; if(!forExport) priority = -1; //parse direct imports as soon as possible else if(m_priority<=-1) priority = BackgroundParser::WorstPriority-2;//imports of direct imports to the stack bottom else priority = m_priority - 2;//currently parsejob does not get created in this cases to reduce recursion for(QString filename : files) { QFile file(filename); if(!file.exists()) continue; //test files are not part of binary package, so we can exclude them //we parse test files only if we open them in KDevelop if(filename.endsWith("_test.go")) continue; IndexedString url(filename); DUChainReadLocker lock; ReferencedTopDUContext context = DUChain::self()->chainForDocument(url); lock.unlock(); if(context) contexts.append(context); else { if(scheduleForParsing(url, priority, (TopDUContext::Features)(TopDUContext::ForceUpdate | TopDUContext::AllDeclarationsAndContexts))) shouldReparse = true; } } if(shouldReparse) //reparse this file after its imports are done scheduleForParsing(m_document, priority+1, (TopDUContext::Features)(m_features | TopDUContext::ForceUpdate)); if(!forExport && m_priority != BackgroundParser::WorstPriority)//always schedule last reparse after all recursive imports are done scheduleForParsing(m_document, BackgroundParser::WorstPriority, (TopDUContext::Features)(m_features | TopDUContext::ForceUpdate)); return contexts; } bool ParseSession::scheduleForParsing(const IndexedString& url, int priority, TopDUContext::Features features) { BackgroundParser* bgparser = KDevelop::ICore::self()->languageController()->backgroundParser(); //TopDUContext::Features features = (TopDUContext::Features)(TopDUContext::ForceUpdate | TopDUContext::VisibleDeclarationsAndContexts);//(TopDUContext::Features) //(TopDUContext::ForceUpdate | TopDUContext::AllDeclarationsContextsAndUses); //currently recursive imports work really slow, nor they usually needed //so disallow recursive imports int levels=1; //allowed levels of recursion if(forExport && priority >= BackgroundParser::InitialParsePriority && priority < BackgroundParser::WorstPriority - 2*levels) //if(forExport) return false; if (bgparser->isQueued(url)) { if (bgparser->priorityForDocument(url) > priority ) // Remove the document and re-queue it with a greater priority bgparser->removeDocument(url); else return true; } bgparser->addDocument(url, features, priority, 0, ParseJob::FullSequentialProcessing); return true; } void ParseSession::rescheduleThisFile() { BackgroundParser* bgparser = KDevelop::ICore::self()->languageController()->backgroundParser(); if(!bgparser->isQueued(m_document)) bgparser->addDocument(m_document, (TopDUContext::Features)(m_features | TopDUContext::ForceUpdate), m_priority, 0, ParseJob::FullSequentialProcessing); } /** * Reparse files that import current context. * Only works for opened files, so another opened files get notified of changed context */ void ParseSession::reparseImporters(DUContext* context) { DUChainReadLocker lock; if(forExport || m_priority!=0) return; for (DUContext* importer : context->importers()) { scheduleForParsing(importer->url(), BackgroundParser::WorstPriority, (TopDUContext::Features)(importer->topContext()->features() | TopDUContext::ForceUpdate)); } } QList< ReferencedTopDUContext > ParseSession::contextForThisPackage(IndexedString package) { QList contexts; QUrl url = package.toUrl(); QDir path(url.adjusted(QUrl::RemoveFilename).path()); if(path.exists()) { int priority = BackgroundParser::WorstPriority; if(!forExport) priority = -1; //import this package as soon as possible else if(m_priority<=-1) priority = BackgroundParser::WorstPriority-2;//all needed files should be scheduled already else priority = m_priority;//currently parsejob does not get created in this cases to reduce recursion QStringList files = path.entryList(QStringList("*.go"), QDir::Files | QDir::NoSymLinks); bool shouldReparse=false; for(QString filename : files) { filename = path.filePath(filename); QFile file(filename); if(!file.exists()) continue; if(forExport && filename.endsWith("_test.go")) continue; IndexedString url(filename); DUChainReadLocker lock; ReferencedTopDUContext context = DUChain::self()->chainForDocument(url); lock.unlock(); if(context) contexts.append(context); else { if(scheduleForParsing(url, priority, (TopDUContext::Features)(TopDUContext::ForceUpdate | TopDUContext::AllDeclarationsAndContexts))) shouldReparse=true; } } if(shouldReparse) scheduleForParsing(m_document, priority+1, (TopDUContext::Features)(m_features | TopDUContext::ForceUpdate)); } return contexts; } void ParseSession::setFeatures(TopDUContext::Features features) { m_features = features; if((m_features & TopDUContext::AllDeclarationsContextsAndUses) == TopDUContext::AllDeclarationsAndContexts) forExport = true; } QString ParseSession::textForNode(go::AstNode* node) { return QString(m_contents.mid(m_lexer->at(node->startToken).begin, m_lexer->at(node->endToken).end - m_lexer->at(node->startToken).begin+1)); } void ParseSession::setIncludePaths(const QList& paths) { m_includePaths = paths; } QByteArray ParseSession::commentBeforeToken(qint64 token) { int commentEnd = m_lexer->at(token).begin; int commentStart = 0; if(token - 1 >= 0) commentStart = m_lexer->at(token-1).end+1; QString comment = m_contents.mid(commentStart, commentEnd-commentStart); //in lexer, when we insert semicolons after newline //inserted token's end contains '\n' position //so in order not to lose this newline we prepend it if(commentStart > 0 && m_contents[commentStart-1] == '\n') comment.prepend('\n'); //any comment must have at least single '/' if(comment.indexOf('/') == -1) return QByteArray(); int i = 0; int start=-1, end=-1, lineStart=-1, lineEnd=-1; int currentLine = 0; //this flag is true when multiple single-lined comments have been encountered in a row bool contigiousComments = false; while(i < comment.length()) { if(comment[i] == '\n') { contigiousComments = false; currentLine++; i++; } else if(comment[i].isSpace()) { i++; }else if(comment[i] == '/') { if(i + 1 < comment.length() && comment[i+1] == '/') { if(!contigiousComments) { start = i+2; lineStart = currentLine; contigiousComments = true; } i += 2; while(i* imports) { m_canonicalImports = imports; } +QList ParseSession::problems() const +{ + return m_parser->problems(); +} diff --git a/parser/parsesession.h b/parser/parsesession.h index 876a65b..11507b1 100644 --- a/parser/parsesession.h +++ b/parser/parsesession.h @@ -1,136 +1,141 @@ /************************************************************************************* * Copyright (C) 2014 by Pavel Petrushkov * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * * as published by the Free Software Foundation; either version 2 * * of the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * *************************************************************************************/ #ifndef KDEVGOLANGPARSESESSION_H #define KDEVGOLANGPARSESESSION_H #include #include +#include #include +#include #include "kdevgoparser_export.h" #include "parser/goast.h" namespace KDevPG { class MemoryPool; } namespace go { class Lexer; class Parser; class StartAst; } typedef QPair SimpleUse; class KDEVGOPARSER_EXPORT ParseSession { public: ParseSession(const QByteArray& contents, int priority, bool appendWithNewline=true); virtual ~ParseSession(); static KDevelop::IndexedString languageString(); bool startParsing(); bool parseExpression(go::ExpressionAst **node); go::StartAst* ast(); QString symbol(qint64 index); KDevelop::RangeInRevision findRange(go::AstNode* from, go::AstNode* to); KDevelop::IndexedString currentDocument(); void setCurrentDocument(const KDevelop::IndexedString& document); KDevelop::IndexedString url(); QList contextForImport(QString package); QList contextForThisPackage(KDevelop::IndexedString package); + QList problems() const; + /** * Schedules for parsing with given priority and features. * NOTE this will not schedule recursive imports (imports of imports and so on) * to schedule any file use BackgroundParser instead **/ bool scheduleForParsing(const KDevelop::IndexedString& url, int priority, KDevelop::TopDUContext::Features features); /** * Reschedules this file with same features. **/ void rescheduleThisFile(); void reparseImporters(KDevelop::DUContext* context); void setFeatures(KDevelop::TopDUContext::Features features); QString textForNode(go::AstNode* node); void setIncludePaths(const QList &paths); void setCanonicalImports(QHash* imports); /** * Returns doc comment preceding given token. * GoDoc comments are multilined /*-style comments * or several consecutive single-lined //-style comments * with no empty line between them. * Comment must start on a new line and end a line before given declaration. **/ QByteArray commentBeforeToken(qint64 token); /** * Don't use this function! * Most of the times you don't need to access lexer of parseSession directly, * This only exists, because parser test application uses DebugVisitor, which needs a lexer */ friend go::Lexer* getLexer(const ParseSession& session) { return session.m_lexer; } void mapAstUse(go::AstNode* node, const SimpleUse& use) { Q_UNUSED(node); Q_UNUSED(use); } private: bool lex(); KDevPG::MemoryPool* m_pool; go::Lexer* m_lexer; go::Parser* m_parser; go::StartAst* m_ast; QByteArray m_contents; int m_priority; KDevelop::IndexedString m_document; KDevelop::TopDUContext::Features m_features; bool forExport; QList m_includePaths; QHash* m_canonicalImports; - + + QList m_problems; }; #endif \ No newline at end of file diff --git a/parser/test/CMakeLists.txt b/parser/test/CMakeLists.txt index 103ec31..516ab4d 100644 --- a/parser/test/CMakeLists.txt +++ b/parser/test/CMakeLists.txt @@ -1,16 +1,17 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ) include_directories( ${QT_INCLUDES} ${KDEVPGQT_INCLUDE_DIR} ) ecm_add_test(parsertest.cpp LINK_LIBRARIES kdevgoparser Qt5::Test Qt5::Core KDev::Language + KDev::Tests ) \ No newline at end of file diff --git a/parser/test/parsertest.cpp b/parser/test/parsertest.cpp index 789e985..7277d5c 100644 --- a/parser/test/parsertest.cpp +++ b/parser/test/parsertest.cpp @@ -1,529 +1,541 @@ /************************************************************************************* * Copyright (C) 2014 by Pavel Petrushkov * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * * as published by the Free Software Foundation; either version 2 * * of the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * *************************************************************************************/ #include "parsertest.h" #include +#include +#include #include "parser/golexer.h" #include "parser/goparser.h" #include "parser/godebugvisitor.h" #include "parser/gotokentext.h" #include "parsesession.h" QTEST_MAIN(go::ParserTest) namespace go { ParserTest::ParserTest() {} +void ParserTest::initTestCase() +{ + KDevelop::AutoTestShell::init(); + KDevelop::TestCore::initialize(KDevelop::Core::NoUi); +} + void ParserTest::testKeyWords() { QByteArray code = "break default func interface select case defer go map struct \ chan else goto package switch const fallthrough if range type continue for \ import return var"; KDevPG::QByteArrayIterator iter(code); Lexer lexer(iter); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_BREAK); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_DEFAULT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_FUNC); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_INTERFACE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_SELECT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_CASE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_DEFER); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_GO); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MAP); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_STRUCT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_CHAN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_ELSE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_GOTO); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_PACKAGE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_SWITCH); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_CONST); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_FALLTHROUGH); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_IF); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RANGE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_TYPE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_CONTINUE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_FOR); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_IMPORT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RETURN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_VAR); } void ParserTest::testOperators() { QByteArray code = "+ & += &= && == != ( ) - | -= |= || < <= [ ] * ^ *= ^= <- > >= { } \ / << /= <<= ++ = := , ; % >> %= >>= -- ! ... . : &^ &^="; KDevPG::QByteArrayIterator iter(code); Lexer lexer(iter); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_PLUS); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_AMPERSAND); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_PLUSEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_AMPEREQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LOGICALAND); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_ISEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_ISNOTEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LPAREN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RPAREN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MINUS); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_BITWISEOR); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MINUSEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_OREQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LOGICALOR); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LESS); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LESSOREQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LBRACKET); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RBRACKET); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_STAR); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_HAT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MULTIPLYEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_XOREQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LEFTCHAN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_GREATER); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_GREATEROREQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LBRACE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RBRACE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_DIVIDE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LEFTSHIFT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_DIVIDEEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_LEFTSHIFTEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_PLUSPLUS); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_ASSIGN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_AUTOASSIGN); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_COMMA); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_SEMICOLON); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MOD); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RIGHTSHIFT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MODEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RIGHTSHIFTEQUAL); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_MINUSMINUS); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_BANG); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_TRIPLEDOT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_DOT); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_COLON); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_AMPERXOR); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_AMPERXOREQUAL); } void ParserTest::testRunes() { QByteArray code = "'\\' '\"' '~' '_' '0' 'a' '\\'' 'щ' ''' "; KDevPG::QByteArrayIterator iter(code); Lexer lexer(iter); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_RUNE); QVERIFY( lexer.read().kind == TokenTypeWrapper::Token_TEST); } void ParserTest::testNumbers() { QFETCH(QByteArray, code); QFETCH(int, tokenType); KDevPG::QByteArrayIterator iter(code); Lexer lexer(iter); auto token = lexer.read(); QCOMPARE((TokenType)token.kind, (TokenType)tokenType); } void ParserTest::testNumbers_data() { QTest::addColumn("code"); QTest::addColumn("tokenType"); QTest::newRow("integer 42") << QByteArray("42") << (int)TokenTypeWrapper::Token_INTEGER; QTest::newRow("integer 0600") << QByteArray("0600") << (int)TokenTypeWrapper::Token_INTEGER; QTest::newRow("integer 0xDEADBEEF") << QByteArray("0xDEADBEEF") << (int)TokenTypeWrapper::Token_INTEGER; QTest::newRow("float 0.") << QByteArray("0.") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("float 1.12") << QByteArray("1.12") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("float 1.e+2") << QByteArray("1.e+2") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("float 1.e-2") << QByteArray("1.e-2") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("float 1E5") << QByteArray("1E5") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("float .123") << QByteArray(".123") << (int)TokenTypeWrapper::Token_FLOAT; QTest::newRow("complex number 0i") << QByteArray("0i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 011i") << QByteArray("011i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 0.i") << QByteArray("0.i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 2.3i") << QByteArray("2.3i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 1.e+0i") << QByteArray("1.e+0i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 5.5e-3i") << QByteArray("5.5e-3i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number 1E6i") << QByteArray("1E6i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number .23i") << QByteArray(".23i") << (int)TokenTypeWrapper::Token_COMPLEX; QTest::newRow("complex number .123E+4i") << QByteArray(".123E+4i") << (int)TokenTypeWrapper::Token_COMPLEX; } void prepareParser(const QByteArray& code, go::Parser** parser, go::Lexer** lexer) { KDevPG::TokenStream tokenStream; *parser = new go::Parser(); (*parser)->setMemoryPool(new KDevPG::MemoryPool()); KDevPG::QByteArrayIterator iter(code); *lexer = new go::Lexer(iter); (*parser)->setTokenStream(*lexer); (*parser)->rewind(0); } QByteArray ParserTest::getCodeFromNode(const QByteArray &code, go::Lexer *lexer, go::AstNode *node) { auto startToken = lexer->at(node->startToken); auto endToken = lexer->at(node->endToken); return code.mid(startToken.begin, endToken.end - startToken.begin+1); } void ParserTest::testCommentsAreIgnored() { QFETCH(QByteArray, code); go::Lexer *lexer; lexer = new go::Lexer(go::Lexer::ByteStringIterator(code)); QCOMPARE(lexer->read().kind, (int)TokenType::Token_EOF); } void ParserTest::testCommentsAreIgnored_data() { QTest::addColumn("code"); QTest::newRow("Single-lined comment") << QByteArray("//test\n"); QTest::newRow("Multiple single-lined comments") << QByteArray("//test\n//test\n"); QTest::newRow("Multi-lined comment") << QByteArray("/*\ntest\n*/\n"); } void ParserTest::testMultiLineStrings() { QFETCH(QByteArray, code); QFETCH(bool, isValid); QFETCH(QByteArray, expectedContent); go::Parser *parser; go::Lexer *lexer; go::StatementsAst* ast; prepareParser(code, &parser, &lexer); bool result = parser->parseStatements(&ast); QCOMPARE(result, isValid); if(isValid) { auto node = ast->statementSequence->front()->element->simpleStmt->shortVarDecl->expression->unaryExpression->primaryExpr->basicLit; auto stringValue = getCodeFromNode(code, lexer, node); QCOMPARE(stringValue, expectedContent); } + else + { + QVERIFY(parser->problems().size() > 0); + } } void ParserTest::testMultiLineStrings_data() { QTest::addColumn("code"); QTest::addColumn("isValid"); QTest::addColumn("expectedContent"); QTest::newRow("Multiline string") << QByteArray("a := `\\n\n\\n`\n") << true << QByteArray("`\\n\n\\n`"); QTest::newRow("Incorrect multiline string") << QByteArray("a := \"\n\"\n") << false << QByteArray(); } void ParserTest::testBasicTypes() { QByteArray code = "struct { array [5][2]string; slice []*int; \ x, y int \n A *[]int \n F func() \n T1 \n *T2 \n P.T3; \ *P.T4; afk func(a, b int, z float64, opt ...interface{}) (success bool); \ afk2 func(int, int) bool; afk3 func(n int) func(p *T); }"; //QByteArray code = "int"; go::Parser parser; KDevPG::MemoryPool pool; parser.setMemoryPool(&pool); KDevPG::QByteArrayIterator iter(code); go::Lexer lexer(iter); parser.setTokenStream(&lexer); //parser.rewind(0); //while((kind = lexer.read().kind) != TokenTypeWrapper::Token_EOF) //qDebug() << tokenText(kind); //} parser.rewind(0); go::TypeAst* ast; bool result=parser.parseType(&ast); QVERIFY(result); //go::DebugVisitor visitor(&lexer, code); //visitor.visitNode(ast); QString content = code; const KDevPG::ListNode *node = ast->complexType->structType->fieldDeclSequence->front(); //qDebug() << code.mid(lexer.at(node->element->startToken).begin, lexer.at(node->element->endToken).end - lexer.at(node->element->startToken).begin+1); auto actual = getCodeFromNode(code, &lexer, node->element); QByteArray expected = "array [5][2]string"; QCOMPARE(actual, expected); } void ParserTest::testIfClause() { QByteArray code = "if a:=(A{2}); a==(A{2}) { a += A{1}; }\n \ if ID(f()) { g(); }\n \ if ([]int)(A)==hello {} \n \ if (a)(A{3}) {} \n \ if ([]a)(A{3}) {} \n \ if x := f(); x < y { abc; def(); } \n \ if ([]a)(A{3}); (a)(A{f}) {} \n"; //QByteArray code = "int"; go::Parser parser; KDevPG::MemoryPool pool; parser.setMemoryPool(&pool); KDevPG::QByteArrayIterator iter(code); go::Lexer lexer(iter); parser.setTokenStream(&lexer); parser.rewind(0); go::StatementsAst* ast; bool result=parser.parseStatements(&ast); QVERIFY(result); //go::DebugVisitor visitor(&lexer, code); //visitor.visitNode(ast); } void ParserTest::testFuncTypes() { QByteArray code = "func () {} \n \ func(a) {} \n \ func(a,) {} \n \ func(a int,) {}\n \ func(a, b []int){} \n \ func(*char, [2]int, f chan float64){} \n \ func(nums ...int, ){} \n \ "; //QByteArray code = "int"; go::Parser parser; KDevPG::MemoryPool pool; parser.setMemoryPool(&pool); KDevPG::QByteArrayIterator iter(code); go::Lexer lexer(iter); parser.setTokenStream(&lexer); parser.rewind(0); go::StatementsAst* ast; bool result=parser.parseStatements(&ast); QVERIFY(result); //go::DebugVisitor visitor(&lexer, code); //visitor.visitNode(ast); } void ParserTest::testForRangeLoop() { QString code = "package main; func main() { str := \"string\"; for range str { c := 2 } } "; ParseSession session(code.toUtf8(), 0, true); QVERIFY(session.startParsing()); } void ParserTest::testForSingleConditionalLoop() { QByteArray code = "for i < 1 { i += 1 }\n"; QByteArray expectedConditionCode = "i < 1"; QByteArray expectedBlockCode = "{ i += 1 }"; go::Parser *parser; go::Lexer *lexer; go::StatementsAst* ast; prepareParser(code, &parser, &lexer); bool result = parser->parseStatements(&ast); QVERIFY(result); auto forStmt = ast->statementSequence->front()->element->forStmt; auto condition = forStmt->initStmtOrCondition; auto block = forStmt->block; bool foundCStyleCondition = forStmt->condition; bool foundPostStmt = forStmt->postStmt; QVERIFY(condition); QVERIFY(block); QVERIFY(!foundCStyleCondition); QVERIFY(!foundPostStmt); auto conditionCode = getCodeFromNode(code, lexer, condition); auto blockCode = getCodeFromNode(code, lexer, block); QCOMPARE(conditionCode, expectedConditionCode); QCOMPARE(blockCode, expectedBlockCode); } void ParserTest::testForWithForClauseLoop() { QByteArray code = "for i := 0; i < 10; i++ { f(i) }\n"; QByteArray expectedInitStmtCode = "i := 0"; QByteArray expectedConditionCode = "i < 10"; QByteArray expectedPostStmtCode = "i++"; QByteArray expectedBlockCode = "{ f(i) }"; go::Parser *parser; go::Lexer *lexer; go::StatementsAst* ast; prepareParser(code, &parser, &lexer); bool result = parser->parseStatements(&ast); QVERIFY(result); auto forStmt = ast->statementSequence->front()->element->forStmt; auto initStmt = forStmt->initStmtOrCondition; auto condition = forStmt->condition; auto postStmt = forStmt->postStmt; auto block = forStmt->block; QVERIFY(initStmt); QVERIFY(condition); QVERIFY(postStmt); QVERIFY(block); auto initStmtCode = getCodeFromNode(code, lexer, initStmt); auto conditionCode = getCodeFromNode(code, lexer, condition); auto postStmtCode = getCodeFromNode(code, lexer, postStmt); auto blockCode = getCodeFromNode(code, lexer, block); QCOMPARE(initStmtCode, expectedInitStmtCode); QCOMPARE(conditionCode, expectedConditionCode); QCOMPARE(postStmtCode, expectedPostStmtCode); QCOMPARE(blockCode, expectedBlockCode); } void ParserTest::testForWithEmptySingleConditionLoop() { QByteArray code = "for { i += 1 }\n"; QByteArray expectedBlockCode = "{ i += 1 }"; go::Parser *parser; go::Lexer *lexer; go::StatementsAst* ast; prepareParser(code, &parser, &lexer); bool result = parser->parseStatements(&ast); QVERIFY(result); auto forStmt = ast->statementSequence->front()->element->forStmt; auto conditionNode = forStmt->initStmtOrCondition; auto block = forStmt->block; bool foundCStyleCondition = forStmt->condition; bool foundPostStmt = forStmt->postStmt; QVERIFY(!conditionNode); QVERIFY(!foundCStyleCondition); QVERIFY(!foundPostStmt); QVERIFY(block); auto blockCode = getCodeFromNode(code, lexer, block); QCOMPARE(blockCode, expectedBlockCode); } void ParserTest::testShortVarDeclaration() { QFETCH(QByteArray, code); QFETCH(bool, idFound); QFETCH(bool, idListFound); QFETCH(bool, expressionFound); QFETCH(bool, expressionListFound); QFETCH(QByteArray, expectedId); QFETCH(QList, expectedIdList); QFETCH(QByteArray, expectedExpression); QFETCH(QList, expectedExpressionList); go::Parser *parser; go::Lexer *lexer; go::StatementsAst* ast; prepareParser(code, &parser, &lexer); bool result = parser->parseStatements(&ast); QVERIFY(result); auto shortVarDecl = ast->statementSequence->front()->element->simpleStmt->shortVarDecl; QCOMPARE(shortVarDecl->id != nullptr, idFound); QCOMPARE(shortVarDecl->idList != nullptr, idListFound); QCOMPARE(shortVarDecl->expression != nullptr, expressionFound); QCOMPARE(shortVarDecl->expressionList != nullptr, expressionListFound); if(idFound) { auto idCode = getCodeFromNode(code, lexer, shortVarDecl->id); QCOMPARE(idCode, expectedId); } if(idListFound) { auto idSequence = shortVarDecl->idList->idSequence; QList actualList; for(int i = 0; icount(); ++i) { actualList.append(getCodeFromNode(code, lexer, idSequence->at(i)->element)); } QCOMPARE(actualList, expectedIdList); } if(expressionFound) { auto expressionCode = getCodeFromNode(code, lexer, shortVarDecl->expression); QCOMPARE(expressionCode, expectedExpression); } if(expressionListFound) { auto expressionsSequence = shortVarDecl->expressionList->expressionsSequence; QList actualList; for(int i = 0; icount(); ++i) { actualList.append(getCodeFromNode(code, lexer, expressionsSequence->at(i)->element)); } QCOMPARE(actualList, expectedExpressionList); } } void ParserTest::testShortVarDeclaration_data() { QTest::addColumn("code"); QTest::addColumn("idFound"); QTest::addColumn("idListFound"); QTest::addColumn("expressionFound"); QTest::addColumn("expressionListFound"); QTest::addColumn("expectedId"); QTest::addColumn>("expectedIdList"); QTest::addColumn("expectedExpression"); QTest::addColumn>("expectedExpressionList"); QTest::newRow("One variable, one expression") << QByteArray("a := 10\n") << true << false << true << false << QByteArray("a") << QList() << QByteArray("10") << QList(); QTest::newRow("Few variables, one expression") << QByteArray("a, b, c := test()\n") << true << true << true << false << QByteArray("a") << (QList() << "b" << "c") << QByteArray("test()") << QList(); QTest::newRow("Few variables, few expressions") << QByteArray("a, b, c := 10, 3, 7\n") << true << true << true << true << QByteArray("a") << (QList() << "b" << "c") << QByteArray("10") << (QList() << "3" << "7"); } void ParserTest::testEmptyLabeledStmt() { QString code("package main; func main() { for i:=1; i<5; i++ { a := i; emptylabel: \n } }"); ParseSession session(code.toUtf8(), 0, true); QVERIFY(session.startParsing()); } void ParserTest::testMapKeyLiteralValue() { QString code("package main; type T struct { method string }; func main() { var x = map[T]int{{\"foo\"} : 3, }; \ var y = map[T]int { T{\"foo\"} : 3, }; var z = map[int]T{ 3 : {\"foo\"}, }; }"); ParseSession session(code.toUtf8(), 0, true); QVERIFY(session.startParsing()); } //add this tests: //go func(ch chan<- bool) {for {sleep(10); ch <- true; }}(c); //select { // case i1 = <-c1: // print("asdas", i3, "from c3\n"); // case c2 <- i2: // f(); g(); } diff --git a/parser/test/parsertest.h b/parser/test/parsertest.h index f99197c..7127fed 100644 --- a/parser/test/parsertest.h +++ b/parser/test/parsertest.h @@ -1,63 +1,64 @@ /************************************************************************************* * Copyright (C) 2014 by Pavel Petrushkov * * * * This program is free software; you can redistribute it and/or * * modify it under the terms of the GNU General Public License * * as published by the Free Software Foundation; either version 2 * * of the License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, write to the Free Software * * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA * *************************************************************************************/ #ifndef GOPARSERTEST #define GOPARSERTEST #include #include namespace go { class Lexer; class AstNode; class ParserTest : public QObject { Q_OBJECT public: ParserTest(); private slots: + void initTestCase(); void testKeyWords(); void testOperators(); void testRunes(); void testNumbers(); void testNumbers_data(); void testCommentsAreIgnored(); void testCommentsAreIgnored_data(); void testMultiLineStrings(); void testMultiLineStrings_data(); void testBasicTypes(); void testIfClause(); void testFuncTypes(); void testForRangeLoop(); void testForSingleConditionalLoop(); void testForWithForClauseLoop(); void testForWithEmptySingleConditionLoop(); void testShortVarDeclaration(); void testShortVarDeclaration_data(); void testEmptyLabeledStmt(); void testMapKeyLiteralValue(); //Go 1.5 feature private: QByteArray getCodeFromNode(const QByteArray &code, go::Lexer *lexer, go::AstNode *node); }; } #endif \ No newline at end of file