diff --git a/autotests/unit/lib/advancedqueryparsertest.cpp b/autotests/unit/lib/advancedqueryparsertest.cpp index 6b9f6ca2..265ba275 100644 --- a/autotests/unit/lib/advancedqueryparsertest.cpp +++ b/autotests/unit/lib/advancedqueryparsertest.cpp @@ -1,356 +1,356 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "advancedqueryparser.h" #include Q_DECLARE_METATYPE(Baloo::Term) using namespace Baloo; class AdvancedQueryParserTest : public QObject { Q_OBJECT private Q_SLOTS: void testSimpleProperty(); void testSimpleString(); void testStringAndProperty(); void testLogicalOps(); void testNesting(); void testDateTime(); void testOperators(); void testBinaryOperatorMissingFirstArg(); void testNestedParentheses(); void testNestedParentheses_data(); void testOptimizedLogic(); void testOptimizedLogic_data(); }; void AdvancedQueryParserTest::testSimpleProperty() { AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("artist:Coldplay")); Term expectedTerm(QStringLiteral("artist"), "Coldplay"); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testSimpleString() { AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("Coldplay")); Term expectedTerm(QLatin1String(""), "Coldplay"); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testStringAndProperty() { AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("stars artist:Coldplay fire")); Term expectedTerm(Term::And); expectedTerm.addSubTerm(Term(QLatin1String(""), "stars")); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QLatin1String(""), "fire")); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testLogicalOps() { // AND AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("artist:Coldplay AND type:song")); Term expectedTerm(Term::And); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QStringLiteral("type"), "song")); QCOMPARE(term, expectedTerm); // OR term = parser.parse(QStringLiteral("artist:Coldplay OR type:song")); expectedTerm = Term(Term::Or); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QStringLiteral("type"), "song")); QCOMPARE(term, expectedTerm); // AND then OR term = parser.parse(QStringLiteral("artist:Coldplay AND type:song OR stars")); expectedTerm = Term(Term::Or); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay") && Term(QStringLiteral("type"), "song")); expectedTerm.addSubTerm(Term(QLatin1String(""), "stars")); QCOMPARE(term, expectedTerm); // OR then AND term = parser.parse(QStringLiteral("artist:Coldplay OR type:song AND stars")); expectedTerm = Term(Term::And); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay") || Term(QStringLiteral("type"), "song")); expectedTerm.addSubTerm(Term(QLatin1String(""), "stars")); QCOMPARE(term, expectedTerm); // Multiple ANDs term = parser.parse(QStringLiteral("artist:Coldplay AND type:song AND stars")); expectedTerm = Term(Term::And); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QStringLiteral("type"), "song")); expectedTerm.addSubTerm(Term(QLatin1String(""), "stars")); QCOMPARE(term, expectedTerm); // Multiple ORs term = parser.parse(QStringLiteral("artist:Coldplay OR type:song OR stars")); expectedTerm = Term(Term::Or); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QStringLiteral("type"), "song")); expectedTerm.addSubTerm(Term(QLatin1String(""), "stars")); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testNesting() { AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("artist:Coldplay AND (type:song OR stars) fire")); Term expectedTerm(Term::And); expectedTerm.addSubTerm(Term(QStringLiteral("artist"), "Coldplay")); expectedTerm.addSubTerm(Term(QStringLiteral("type"), "song") || Term(QLatin1String(""), "stars")); expectedTerm.addSubTerm(Term(QLatin1String(""), "fire")); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testDateTime() { // Integers AdvancedQueryParser parser; Term term; Term expectedTerm; term = parser.parse(QStringLiteral("modified:2014-12-02")); expectedTerm = Term(QStringLiteral("modified"), QDate(2014, 12, 02)); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("modified:\"2014-12-02T23:22:1\"")); expectedTerm = Term(QStringLiteral("modified"), QDateTime(QDate(2014, 12, 02), QTime(23, 22, 1))); QEXPECT_FAIL("", "AQP cannot handle datetime", Abort); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testOperators() { AdvancedQueryParser parser; Term term; Term expectedTerm; term = parser.parse(QStringLiteral("width:500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::Equal); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("width=500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::Equal); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("width<500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::Less); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("width<=500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::LessEqual); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("width>500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::Greater); QCOMPARE(term, expectedTerm); term = parser.parse(QStringLiteral("width>=500")); expectedTerm = Term(QStringLiteral("width"), 500, Term::GreaterEqual); QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testBinaryOperatorMissingFirstArg() { AdvancedQueryParser parser; Term term = parser.parse(QStringLiteral("=:2")); Term expectedTerm; QCOMPARE(term, expectedTerm); } void AdvancedQueryParserTest::testNestedParentheses() { QFETCH(QString, searchInput); QFETCH(QString, failmessage); QFETCH(Term, expectedTerm); AdvancedQueryParser parser; const auto testTerm = parser.parse(searchInput); qDebug() << " result term" << testTerm; qDebug() << "expected term" << expectedTerm; if (!failmessage.isEmpty()) { QEXPECT_FAIL("", qPrintable(failmessage), Continue); } QCOMPARE(testTerm, expectedTerm); } void AdvancedQueryParserTest::testNestedParentheses_data() { QTest::addColumn("searchInput"); QTest::addColumn("expectedTerm"); QTest::addColumn("failmessage"); QTest::newRow("a AND b AND c AND d") << QStringLiteral("a AND b AND c AND d") << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{QString(), QStringLiteral("b"), Term::Contains}, Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains}, }} << QString() ; QTest::newRow("(a AND b) AND (c OR d)") << QStringLiteral("(a AND b) AND (c OR d)") << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{QString(), QStringLiteral("b"), Term::Contains}, Term{Term::Or, QList{ Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains}, }} }} << QString() ; QTest::newRow("(a AND (b AND (c AND d)))") << QStringLiteral("(a AND (b AND (c AND d)))") << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{QString(), QStringLiteral("b"), Term::Contains}, Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains}, }} << QStringLiteral("Fails to optimize for unknown reason, but output is semantically correct") ; // This test verifies that the above test is semantically correct QTest::newRow("(a AND (b AND (c AND d))) semantic") << QStringLiteral("(a AND (b AND (c AND d)))") << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{Term::And, QList{ Term{QString(), QStringLiteral("b"), Term::Contains}, Term{Term::And, QList{ Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains} }} }} }} << QString() ; // Test 1 for BUG: 392620 QTest::newRow("a OR ((b AND c) AND d)") << QStringLiteral("a OR ((b AND c) AND d)") << Term{Term::Or, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{Term::And, QList{ Term{QString(), QStringLiteral("b"), Term::Contains}, Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains} }} }} - << QStringLiteral("Opening '((' parsed incorrectly") + << QString() ; // Test 2 for BUG: 392620 QTest::newRow("a AND ((b OR c) OR d)") << QStringLiteral("a AND ((b OR c) OR d)") << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{Term::Or, QList{ Term{QString(), QStringLiteral("b"), Term::Contains}, Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains} }} }} << QStringLiteral("Opening '((' parsed incorrectly") ; } void AdvancedQueryParserTest::testOptimizedLogic() { QFETCH(Term, testTerm); QFETCH(Term, expectedTerm); qDebug() << " result term" << testTerm; qDebug() << "expected term" << expectedTerm; QCOMPARE(testTerm, expectedTerm); } void AdvancedQueryParserTest::testOptimizedLogic_data() { QTest::addColumn("testTerm"); QTest::addColumn("expectedTerm"); // a && b && c && d can be combined into one AND term with 4 subterms QTest::addRow("a && b && c && d") << (Term{QString(), QStringLiteral("a"), Term::Contains} && Term{QString(), QStringLiteral("b"), Term::Contains} && Term{QString(), QStringLiteral("c"), Term::Contains} && Term{QString(), QStringLiteral("d"), Term::Contains}) << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{QString(), QStringLiteral("b"), Term::Contains}, Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains}, }} ; // (a AND b) AND (c OR d) can be merged as (a AND b AND (c OR D) QTest::addRow("(a && b) && (c || d)") << ((Term{QString(), QStringLiteral("a"), Term::Contains} && Term{QString(), QStringLiteral("b"), Term::Contains}) && (Term{QString(), QStringLiteral("c"), Term::Contains} || Term{QString(), QStringLiteral("d"), Term::Contains} )) << Term{Term::And, QList{ Term{QString(), QStringLiteral("a"), Term::Contains}, Term{QString(), QStringLiteral("b"), Term::Contains}, Term{Term::Or, QList{ Term{QString(), QStringLiteral("c"), Term::Contains}, Term{QString(), QStringLiteral("d"), Term::Contains} }} }} ; } QTEST_MAIN(AdvancedQueryParserTest) #include "advancedqueryparsertest.moc" diff --git a/src/lib/advancedqueryparser.cpp b/src/lib/advancedqueryparser.cpp index 01abc28a..70bc0fd0 100644 --- a/src/lib/advancedqueryparser.cpp +++ b/src/lib/advancedqueryparser.cpp @@ -1,264 +1,253 @@ /* * Copyright (C) 2014-2015 Vishesh Handa * Copyright (C) 2014 Denis Steckelmacher * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "advancedqueryparser.h" #include #include #include using namespace Baloo; AdvancedQueryParser::AdvancedQueryParser() { } -static bool isOperator(const QChar& c) -{ - switch (c.toLatin1()) { - case ':': - case '=': - case '>': - case '<': - case '(': - case ')': - return true; - - default: - return false; - } -} - static QStringList lex(const QString& text) { - QStringList tokens; + QStringList tokenList; QString token; bool inQuotes = false; - for (int i=0, end=text.size(); i!=end; ++i) { + for (int i = 0, end = text.size(); i != end; ++i) { QChar c = text.at(i); if (c == QLatin1Char('"')) { // Quotes start or end string literals inQuotes = !inQuotes; } else if (inQuotes) { // Don't do any processing in strings token.append(c); - } else if (c.isSpace() || isOperator(c)) { - // Spaces and operators end tokens - if (token.size() > 0) { - tokens.append(token); + } else if (c.isSpace()) { + // Spaces end tokens + if (!token.isEmpty()) { + tokenList.append(token); token.clear(); } - - // Operators are tokens themselves - if (isOperator(c)) { - if (tokens.size() > 1) { - QString last = tokens.last(); - if (last.size() == 1 && isOperator(last[0])) { - last.append(c); - tokens[tokens.size() - 1] = last; - continue; - } - } - tokens.append(QString(c)); + } else if (c == '(' || c == ')') { + // Parentheses end tokens, and are tokens by themselves + if (!token.isEmpty()) { + tokenList.append(token); + token.clear(); + } + tokenList.append(c); + } else if (c == '>' || c == '<' || c == ':' || c == '=') { + // Operators end tokens + if (!token.isEmpty()) { + tokenList.append(token); + token.clear(); + } + // accept '=' after any of the above + if (text.at(i + 1) == '=') { + tokenList.append(text.mid(i, 2)); + i++; + } else { + tokenList.append(c); } - - continue; } else { // Simply extend the current token token.append(c); } } - if (token.size() > 0) { - tokens.append(token); + if (!token.isEmpty()) { + tokenList.append(token); } - return tokens; + return tokenList; } static void addTermToStack(QStack& stack, const Term& termInConstruction, Term::Operation op) { Term &tos = stack.top(); if (tos.isEmpty()) { // Empty top of stack, just assign termInConstruction to it tos = termInConstruction; return; } if (tos.subTerms().count() == 0 || tos.operation() != op) { // Top of stack is a "literal" term or a logical term of the wrong operation Term tmp = stack.pop(); stack.push(Term(op)); stack.top().addSubTerm(tmp); } stack.top().addSubTerm(termInConstruction); } static QVariant tokenToVariant(const QString& token) { bool okay = false; int intValue = token.toInt(&okay); if (okay) { return QVariant(intValue); } QDate date = QDate::fromString(token, Qt::ISODate); if (date.isValid() && !date.isNull()) { return date; } return token; } Term AdvancedQueryParser::parse(const QString& text) { // The parser does not do any look-ahead but has to store some state QStack stack; QStack ops; Term termInConstruction; bool valueExpected = false; Term::Operation nextOp = Term::And; stack.push(Term()); ops.push(Term::And); // Lex the input string QStringList tokens = lex(text); for (const QString &token : tokens) { // If a key and an operator have been parsed, now is time for a value if (valueExpected) { // When the parser encounters a literal, it puts it in the value of // termInConstruction so that "foo bar baz" is parsed as expected. auto property = termInConstruction.value().toString(); if (property.isEmpty()) { qDebug() << "Binary operator without first argument encountered:" << text; return Term(); } termInConstruction.setProperty(property); QVariant value = tokenToVariant(token); if (value.type() != QVariant::String) { if (termInConstruction.comparator() == Term::Contains) { termInConstruction.setComparator(Term::Equal); } } termInConstruction.setValue(value); valueExpected = false; continue; } // Handle the logic operators if (token == QStringLiteral("AND")) { nextOp = Term::And; continue; } else if (token == QStringLiteral("OR")) { nextOp = Term::Or; continue; } // Handle the different comparators (and braces) Term::Comparator comparator = Term::Auto; switch (token.at(0).toLatin1()) { case ':': comparator = Term::Contains; break; case '=': comparator = Term::Equal; break; case '<': { if (token.size() == 1) { comparator = Term::Less; } else if (token[1] == '=') { comparator = Term::LessEqual; } break; } case '>': { if (token.size() == 1) { comparator = Term::Greater; } else if (token[1] == '=') { comparator = Term::GreaterEqual; } break; } case '(': if (!termInConstruction.isEmpty()) { addTermToStack(stack, termInConstruction, ops.top()); ops.top() = nextOp; } stack.push(Term()); ops.push(Term::And); nextOp = Term::And; termInConstruction = Term(); continue; case ')': // Prevent a stack underflow if the user writes "a b ))))" if (stack.size() > 1) { // Don't forget the term just before the closing brace if (termInConstruction.value().isValid()) { addTermToStack(stack, termInConstruction, ops.top()); } // stack.pop() is the term that has just been closed. Append // it to the term just above it. ops.pop(); addTermToStack(stack, stack.pop(), ops.top()); nextOp = Term::And; termInConstruction = Term(); } continue; default: break; } if (comparator != Term::Auto) { // Set the comparator of the term in construction and expect a value termInConstruction.setComparator(comparator); valueExpected = true; } else { // A new term will be started, so termInConstruction has to be appended // to the top-level subterm list. if (!termInConstruction.isEmpty()) { addTermToStack(stack, termInConstruction, ops.top()); ops.top() = nextOp; nextOp = Term::And; } termInConstruction = Term(QString(), token); } } if (termInConstruction.value().isValid()) { addTermToStack(stack, termInConstruction, ops.top()); } return stack.top(); }