diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 9e61a3d0..b77dfc98 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -1,680 +1,688 @@ /* * Copyright (C) 2016 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "datastorequery.h" #include "log.h" #include "applicationdomaintype.h" using namespace Sink; using namespace Sink::Storage; static QByteArray operationName(const Sink::Operation op) { switch(op) { case Sink::Operation_Creation: return "Creation"; case Sink::Operation_Modification: return "Modification"; case Sink::Operation_Removal: return "Removal"; } return ""; } class Source : public FilterBase { public: typedef QSharedPointer Ptr; QVector mIds; QVector::ConstIterator mIt; QVector mIncrementalIds; QVector::ConstIterator mIncrementalIt; Source (const QVector &ids, DataStoreQuery *store) : FilterBase(store), mIds(ids), mIt(mIds.constBegin()) { } virtual ~Source(){} virtual void skip() Q_DECL_OVERRIDE { if (mIt != mIds.constEnd()) { mIt++; } }; void add(const QVector &ids) { mIncrementalIds = ids; mIncrementalIt = mIncrementalIds.constBegin(); } bool next(const std::function &callback) Q_DECL_OVERRIDE { if (!mIncrementalIds.isEmpty()) { if (mIncrementalIt == mIncrementalIds.constEnd()) { return false; } readEntity(*mIncrementalIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); callback({entity, operation}); }); mIncrementalIt++; if (mIncrementalIt == mIncrementalIds.constEnd()) { return false; } return true; } else { if (mIt == mIds.constEnd()) { return false; } readEntity(*mIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); callback({entity, operation}); }); mIt++; return mIt != mIds.constEnd(); } } }; class Collector : public FilterBase { public: typedef QSharedPointer Ptr; Collector(FilterBase::Ptr source, DataStoreQuery *store) : FilterBase(source, store) { } virtual ~Collector(){} bool next(const std::function &callback) Q_DECL_OVERRIDE { return mSource->next(callback); } }; class Filter : public FilterBase { public: typedef QSharedPointer Ptr; QHash propertyFilter; Filter(FilterBase::Ptr source, DataStoreQuery *store) : FilterBase(source, store) { } virtual ~Filter(){} virtual bool next(const std::function &callback) Q_DECL_OVERRIDE { bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { SinkTraceCtx(mDatastore->mLogCtx) << "Filter: " << result.entity.identifier() << operationName(result.operation); //Always accept removals. They can't match the filter since the data is gone. if (result.operation == Sink::Operation_Removal) { SinkTraceCtx(mDatastore->mLogCtx) << "Removal: " << result.entity.identifier() << operationName(result.operation); callback(result); foundValue = true; } else if (matchesFilter(result.entity)) { SinkTraceCtx(mDatastore->mLogCtx) << "Accepted: " << result.entity.identifier() << operationName(result.operation); callback(result); foundValue = true; //TODO if something did not match the filter so far but does now, turn into an add operation. } else { SinkTraceCtx(mDatastore->mLogCtx) << "Rejected: " << result.entity.identifier() << operationName(result.operation); //TODO emit a removal if we had the uid in the result set and this is a modification. //We don't know if this results in a removal from the dataset, so we emit a removal notification anyways callback({result.entity, Sink::Operation_Removal, result.aggregateValues}); } return false; })) {} return foundValue; } bool matchesFilter(const ApplicationDomain::ApplicationDomainType &entity) { for (const auto &filterProperty : propertyFilter.keys()) { const auto property = entity.getProperty(filterProperty); const auto comparator = propertyFilter.value(filterProperty); //We can't deal with a fulltext filter if (comparator.comparator == QueryBase::Comparator::Fulltext) { continue; } if (!comparator.matches(property)) { SinkTraceCtx(mDatastore->mLogCtx) << "Filtering entity due to property mismatch on filter: " << entity.identifier() << "Property: " << filterProperty << property << " Filter:" << comparator.value; return false; } } return true; } }; class Reduce : public Filter { public: typedef QSharedPointer Ptr; struct Aggregator { QueryBase::Reduce::Aggregator::Operation operation; QByteArray property; QByteArray resultProperty; Aggregator(QueryBase::Reduce::Aggregator::Operation o, const QByteArray &property_, const QByteArray &resultProperty_) : operation(o), property(property_), resultProperty(resultProperty_) { } void process(const QVariant &value) { if (operation == QueryBase::Reduce::Aggregator::Collect) { mResult = mResult.toList() << value; } else if (operation == QueryBase::Reduce::Aggregator::Count) { mResult = mResult.toInt() + 1; } else { Q_ASSERT(false); } } void reset() { mResult.clear(); } QVariant result() const { return mResult; } private: QVariant mResult; }; QSet mReducedValues; QSet mIncrementallyReducedValues; QHash mSelectedValues; QByteArray mReductionProperty; QByteArray mSelectionProperty; QueryBase::Reduce::Selector::Comparator mSelectionComparator; QList mAggregators; Reduce(const QByteArray &reductionProperty, const QByteArray &selectionProperty, QueryBase::Reduce::Selector::Comparator comparator, FilterBase::Ptr source, DataStoreQuery *store) : Filter(source, store), mReductionProperty(reductionProperty), mSelectionProperty(selectionProperty), mSelectionComparator(comparator) { } virtual ~Reduce(){} void updateComplete() Q_DECL_OVERRIDE { mIncrementallyReducedValues.clear(); } static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { return value.toDateTime().toString().toLatin1(); } if (value.isValid() && !value.toByteArray().isEmpty()) { return value.toByteArray(); } return QByteArray(); } static bool compare(const QVariant &left, const QVariant &right, QueryBase::Reduce::Selector::Comparator comparator) { if (comparator == QueryBase::Reduce::Selector::Max) { return left > right; } return false; } struct ReductionResult { QByteArray selection; QVector aggregateIds; QMap aggregateValues; }; ReductionResult reduceOnValue(const QVariant &reductionValue) { QMap aggregateValues; QVariant selectionResultValue; QByteArray selectionResult; const auto results = indexLookup(mReductionProperty, reductionValue); for (auto &aggregator : mAggregators) { aggregator.reset(); } - + QVector reducedAndFilteredResults; for (const auto &r : results) { readEntity(r, [&, this](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { //We need to apply all property filters that we have until the reduction, because the index lookup was unfiltered. if (!matchesFilter(entity)) { return; } - + reducedAndFilteredResults << r; Q_ASSERT(operation != Sink::Operation_Removal); for (auto &aggregator : mAggregators) { if (!aggregator.property.isEmpty()) { aggregator.process(entity.getProperty(aggregator.property)); } else { aggregator.process(QVariant{}); } } auto selectionValue = entity.getProperty(mSelectionProperty); if (!selectionResultValue.isValid() || compare(selectionValue, selectionResultValue, mSelectionComparator)) { selectionResultValue = selectionValue; selectionResult = entity.identifier(); } }); } for (auto &aggregator : mAggregators) { aggregateValues.insert(aggregator.resultProperty, aggregator.result()); } - return {selectionResult, results, aggregateValues}; + return {selectionResult, reducedAndFilteredResults, aggregateValues}; } bool next(const std::function &callback) Q_DECL_OVERRIDE { bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { const auto reductionValue = [&] { - if (result.operation == Sink::Operation_Removal) { + const auto v = result.entity.getProperty(mReductionProperty); + //Because we also get Operation_Removal for filtered entities. We use the fact that actually removed entites + //won't have the property to reduce on. + //TODO: Perhaps find a cleaner solutoin than abusing Operation::Removed for filtered properties. + if (v.isNull() && result.operation == Sink::Operation_Removal) { //For removals we have to read the last revision to get a value, and thus be able to find the correct thread. QVariant reductionValue; readPrevious(result.entity.identifier(), [&] (const ApplicationDomain::ApplicationDomainType &prev) { reductionValue = prev.getProperty(mReductionProperty); }); return reductionValue; } else { - return result.entity.getProperty(mReductionProperty); + return v; } }(); - const auto &reductionValueBa = getByteArray(reductionValue); + if (reductionValue.isNull()) { + //We failed to find a value to reduce on, so ignore this entity. + //Can happen if the entity was already removed and we have no previous revision. + return; + } + const auto reductionValueBa = getByteArray(reductionValue); if (!mReducedValues.contains(reductionValueBa)) { //Only reduce every value once. mReducedValues.insert(reductionValueBa); auto reductionResult = reduceOnValue(reductionValue); mSelectedValues.insert(reductionValueBa, reductionResult.selection); readEntity(reductionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { callback({entity, operation, reductionResult.aggregateValues, reductionResult.aggregateIds}); foundValue = true; }); } else { //During initial query, do nothing. The lookup above will take care of it. //During updates adjust the reduction according to the modification/addition or removal //We have to redo the reduction for every element, because of the aggregation values. if (mIncremental && !mIncrementallyReducedValues.contains(reductionValueBa)) { mIncrementallyReducedValues.insert(reductionValueBa); //Redo the reduction to find new aggregated values auto selectionResult = reduceOnValue(reductionValue); //TODO if old and new are the same a modification would be enough auto oldSelectionResult = mSelectedValues.take(reductionValueBa); if (oldSelectionResult == selectionResult.selection) { mSelectedValues.insert(reductionValueBa, selectionResult.selection); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Modification, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } else { //remove old result readEntity(oldSelectionResult, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Removal}); }); //If the last item has been removed, then there's nothing to add if (!selectionResult.selection.isEmpty()) { //add new result mSelectedValues.insert(reductionValueBa, selectionResult.selection); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Creation, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } } } } - return false; })) {} return foundValue; } }; class Bloom : public Filter { public: typedef QSharedPointer Ptr; QByteArray mBloomProperty; Bloom(const QByteArray &bloomProperty, FilterBase::Ptr source, DataStoreQuery *store) : Filter(source, store), mBloomProperty(bloomProperty) { } virtual ~Bloom(){} bool next(const std::function &callback) Q_DECL_OVERRIDE { if (!mBloomed) { //Initially we bloom on the first value that matches. //From there on we just filter. bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { mBloomValue = result.entity.getProperty(mBloomProperty); auto results = indexLookup(mBloomProperty, mBloomValue); for (const auto &r : results) { readEntity(r, [&, this](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { callback({entity, Sink::Operation_Creation}); SinkTraceCtx(mDatastore->mLogCtx) << "Bloom result: " << entity.identifier() << operationName(operation); foundValue = true; }); } return false; })) {} mBloomed = true; propertyFilter.insert(mBloomProperty, mBloomValue); return foundValue; } else { //Filter on bloom value return Filter::next(callback); } } QVariant mBloomValue; bool mBloomed = false; }; DataStoreQuery::DataStoreQuery(const Sink::QueryBase &query, const QByteArray &type, EntityStore &store) : mType(type), mStore(store), mLogCtx(store.logContext().subContext("datastorequery")) { //This is what we use during a new query setupQuery(query); } DataStoreQuery::DataStoreQuery(const DataStoreQuery::State &state, const QByteArray &type, Sink::Storage::EntityStore &store, bool incremental) : mType(type), mStore(store), mLogCtx(store.logContext().subContext("datastorequery")) { //This is what we use when fetching more data, without having a new revision with incremental=false //And this is what we use when the data changed and we want to update with incremental = true mCollector = state.mCollector; mSource = state.mSource; auto source = mCollector; while (source) { source->mDatastore = this; source->mIncremental = incremental; source = source->mSource; } } DataStoreQuery::~DataStoreQuery() { } DataStoreQuery::State::Ptr DataStoreQuery::getState() { auto state = State::Ptr::create(); state->mSource = mSource; state->mCollector = mCollector; return state; } void DataStoreQuery::readEntity(const QByteArray &key, const BufferCallback &resultCallback) { mStore.readLatest(mType, key, resultCallback); } void DataStoreQuery::readPrevious(const QByteArray &key, const std::function &callback) { mStore.readPrevious(mType, key, mStore.maxRevision(), callback); } QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) { return mStore.indexLookup(mType, property, value); } /* ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty) */ /* { */ /* const bool sortingRequired = !sortProperty.isEmpty(); */ /* if (mInitialQuery && sortingRequired) { */ /* SinkTrace() << "Sorting the resultset in memory according to property: " << sortProperty; */ /* // Sort the complete set by reading the sort property and filling into a sorted map */ /* auto sortedMap = QSharedPointer>::create(); */ /* while (resultSet.next()) { */ /* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */ /* readEntity(resultSet.id(), */ /* [this, filter, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* const auto operation = buffer.operation(); */ /* // We're not interested in removals during the initial query */ /* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */ /* if (!sortProperty.isEmpty()) { */ /* const auto sortValue = getProperty(buffer.entity(), sortProperty); */ /* if (sortValue.type() == QVariant::DateTime) { */ /* sortedMap->insert(QByteArray::number(std::numeric_limits::max() - sortValue.toDateTime().toTime_t()), uid); */ /* } else { */ /* sortedMap->insert(sortValue.toString().toLatin1(), uid); */ /* } */ /* } else { */ /* sortedMap->insert(uid, uid); */ /* } */ /* } */ /* }); */ /* } */ /* SinkTrace() << "Sorted " << sortedMap->size() << " values."; */ /* auto iterator = QSharedPointer>::create(*sortedMap); */ /* ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter]( */ /* std::function callback) -> bool { */ /* if (iterator->hasNext()) { */ /* readEntity(iterator->next().value(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* callback(uid, buffer, Sink::Operation_Creation); */ /* }); */ /* return true; */ /* } */ /* return false; */ /* }; */ /* auto skip = [iterator]() { */ /* if (iterator->hasNext()) { */ /* iterator->next(); */ /* } */ /* }; */ /* return ResultSet(generator, skip); */ /* } else { */ /* auto resultSetPtr = QSharedPointer::create(resultSet); */ /* ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { */ /* if (resultSetPtr->next()) { */ /* SinkTrace() << "Reading the next value: " << resultSetPtr->id(); */ /* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */ /* readEntity(resultSetPtr->id(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* const auto operation = buffer.operation(); */ /* if (mInitialQuery) { */ /* // We're not interested in removals during the initial query */ /* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */ /* // In the initial set every entity is new */ /* callback(uid, buffer, Sink::Operation_Creation); */ /* } */ /* } else { */ /* // Always remove removals, they probably don't match due to non-available properties */ /* if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) { */ /* // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results) */ /* callback(uid, buffer, operation); */ /* } */ /* } */ /* }); */ /* return true; */ /* } */ /* return false; */ /* }; */ /* auto skip = [resultSetPtr]() { resultSetPtr->skip(1); }; */ /* return ResultSet(generator, skip); */ /* } */ /* } */ QByteArrayList DataStoreQuery::executeSubquery(const QueryBase &subquery) { Q_ASSERT(!subquery.type().isEmpty()); auto sub = DataStoreQuery(subquery, subquery.type(), mStore); auto result = sub.execute(); QByteArrayList ids; while (result.next([&ids](const ResultSet::Result &result) { ids << result.entity.identifier(); })) {} return ids; } void DataStoreQuery::setupQuery(const Sink::QueryBase &query_) { auto query = query_; auto baseFilters = query.getBaseFilters(); //Resolve any subqueries we have for (const auto &k : baseFilters.keys()) { const auto comparator = baseFilters.value(k); if (comparator.value.canConvert()) { SinkTraceCtx(mLogCtx) << "Executing subquery for property: " << k; const auto result = executeSubquery(comparator.value.value()); baseFilters.insert(k, Query::Comparator(QVariant::fromValue(result), Query::Comparator::In)); } } query.setBaseFilters(baseFilters); QByteArray appliedSorting; //Determine initial set mSource = [&]() { if (!query.ids().isEmpty()) { //We have a set of ids as a starting point return Source::Ptr::create(query.ids().toVector(), this); } else { QSet appliedFilters; auto resultSet = mStore.indexLookup(mType, query, appliedFilters, appliedSorting); if (!appliedFilters.isEmpty()) { //We have an index lookup as starting point return Source::Ptr::create(resultSet, this); } // We do a full scan if there were no indexes available to create the initial set (this is going to be expensive for large sets). return Source::Ptr::create(mStore.fullScan(mType), this); } }(); FilterBase::Ptr baseSet = mSource; if (!query.getBaseFilters().isEmpty()) { auto filter = Filter::Ptr::create(baseSet, this); //For incremental queries the remaining filters are not sufficient for (const auto &f : query.getBaseFilters().keys()) { filter->propertyFilter.insert(f, query.getFilter(f)); } baseSet = filter; } /* if (appliedSorting.isEmpty() && !query.sortProperty.isEmpty()) { */ /* //Apply manual sorting */ /* baseSet = Sort::Ptr::create(baseSet, query.sortProperty); */ /* } */ //Setup the rest of the filter stages on top of the base set for (const auto &stage : query.getFilterStages()) { if (auto filter = stage.dynamicCast()) { auto f = Filter::Ptr::create(baseSet, this); f->propertyFilter = filter->propertyFilter; baseSet = f; } else if (auto filter = stage.dynamicCast()) { auto reduction = Reduce::Ptr::create(filter->property, filter->selector.property, filter->selector.comparator, baseSet, this); for (const auto &aggregator : filter->aggregators) { reduction->mAggregators << Reduce::Aggregator(aggregator.operation, aggregator.propertyToCollect, aggregator.resultProperty); } reduction->propertyFilter = query.getBaseFilters(); baseSet = reduction; } else if (auto filter = stage.dynamicCast()) { baseSet = Bloom::Ptr::create(filter->property, baseSet, this); } } mCollector = Collector::Ptr::create(baseSet, this); } QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) { QVector changedKeys; mStore.readRevisions(baseRevision, mType, [&](const QByteArray &key) { changedKeys << key; }); return changedKeys; } ResultSet DataStoreQuery::update(qint64 baseRevision) { SinkTraceCtx(mLogCtx) << "Executing query update from revision " << baseRevision; auto incrementalResultSet = loadIncrementalResultSet(baseRevision); SinkTraceCtx(mLogCtx) << "Incremental changes: " << incrementalResultSet; mSource->add(incrementalResultSet); ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { if (mCollector->next([this, callback](const ResultSet::Result &result) { SinkTraceCtx(mLogCtx) << "Got incremental result: " << result.entity.identifier() << operationName(result.operation); callback(result); })) { return true; } return false; }; return ResultSet(generator, [this]() { mCollector->skip(); }); } void DataStoreQuery::updateComplete() { mSource->mIncrementalIds.clear(); auto source = mCollector; while (source) { source->updateComplete(); source = source->mSource; } } ResultSet DataStoreQuery::execute() { SinkTraceCtx(mLogCtx) << "Executing query"; Q_ASSERT(mCollector); ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { if (mCollector->next([this, callback](const ResultSet::Result &result) { if (result.operation != Sink::Operation_Removal) { SinkTraceCtx(mLogCtx) << "Got initial result: " << result.entity.identifier() << result.operation; callback(ResultSet::Result{result.entity, Sink::Operation_Creation, result.aggregateValues, result.aggregateIds}); } })) { return true; } return false; }; return ResultSet(generator, [this]() { mCollector->skip(); }); } diff --git a/tests/querytest.cpp b/tests/querytest.cpp index f65d4770..52f00242 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1,1394 +1,1394 @@ #include #include #include #include "resource.h" #include "store.h" #include "resourcecontrol.h" #include "commands.h" #include "resourceconfig.h" #include "log.h" #include "modelresult.h" #include "test.h" #include "testutils.h" #include "applicationdomaintype.h" #include using namespace Sink; using namespace Sink::ApplicationDomain; /** * Test of the query system using the dummy resource. * * This test requires the dummy resource installed. */ class QueryTest : public QObject { Q_OBJECT private slots: void initTestCase() { Sink::Test::initTest(); auto factory = Sink::ResourceFactory::load("sink.dummy"); QVERIFY(factory); ResourceConfig::addResource("sink.dummy.instance1", "sink.dummy"); VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void cleanup() { VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void init() { qDebug(); qDebug() << "-----------------------------------------"; qDebug(); } void testSerialization() { auto type = QByteArray("type"); auto sort = QByteArray("sort"); Sink::QueryBase::Filter filter; filter.ids << "id"; filter.propertyFilter.insert("foo", QVariant::fromValue(QByteArray("bar"))); Sink::Query query; query.setFilter(filter); query.setType(type); query.setSortProperty(sort); QByteArray data; { QDataStream stream(&data, QIODevice::WriteOnly); stream << query; } Sink::Query deserializedQuery; { QDataStream stream(&data, QIODevice::ReadOnly); stream >> deserializedQuery; } QCOMPARE(deserializedQuery.type(), type); QCOMPARE(deserializedQuery.sortProperty(), sort); QCOMPARE(deserializedQuery.getFilter().ids, filter.ids); QCOMPARE(deserializedQuery.getFilter().propertyFilter.keys(), filter.propertyFilter.keys()); QCOMPARE(deserializedQuery.getFilter().propertyFilter, filter.propertyFilter); } void testNoResources() { // Test Sink::Query query; query.resourceFilter("foobar"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } void testSingle() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); } void testSingleWithDelay() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testFilter() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); VERIFYEXEC(Sink::Store::create(mail)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter("folder1"); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value(); { mail->setFolder("folder2"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 0); { mail->setFolder("folder1"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 1); } void testById() { QByteArray id; // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); mail.setExtractedMessageId("test2"); VERIFYEXEC(Sink::Store::create(mail)); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed Sink::Store::synchronize(query).exec().waitForFinished(); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QVERIFY(model->rowCount() >= 1); id = model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->identifier(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(id); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testFolder() { // Setup { Folder folder("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); } void testFolderTree() { // Setup { auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QCOMPARE(model->rowCount(model->index(0, 0)), 1); } void testIncrementalFolderTree() { // Setup auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); // Test Sink::Query query{Sink::Query::LiveQuery}; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Ensure the folder appears QTRY_COMPARE(model->rowCount(model->index(0, 0)), 1); //...and dissapears again after removal VERIFYEXEC(Sink::Store::remove(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(model->index(0, 0)), 0); } void testMailByMessageId() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } /* * Filter by two properties to make sure that we also use a non-index based filter. */ void testMailByMessageIdAndFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); Mail mail1("sink.dummy.instance1"); mail1.setExtractedMessageId("test1"); mail1.setFolder("foobar"); Sink::Store::create(mail1).exec().waitForFinished(); Mail mail2("sink.dummy.instance1"); mail2.setExtractedMessageId("test2"); mail2.setFolder(folderEntity->identifier()); Sink::Store::create(mail2).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolderSortedByDate() { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testSecond"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-1)); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLatest"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLast"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.limit(1); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); // The model is not sorted, but the limited set is sorted, so we can only test for the latest result. QCOMPARE(model->rowCount(), 1); QCOMPARE(model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testLatest")); model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); // We can't make any assumptions about the order of the indexes // QCOMPARE(model->index(1, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testSecond")); //New revisions always go through { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testInjected"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } QTRY_COMPARE(model->rowCount(), 3); //Ensure we can continue fetching after the incremental update model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); //Ensure we have fetched all model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); } void testReactToNewResource() { Sink::Query query; query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(QModelIndex()), 0); auto res = DummyResource::create(""); VERIFYEXEC(Sink::Store::create(res)); auto folder = Folder::create(res.identifier()); VERIFYEXEC(Sink::Store::create(folder)); QTRY_COMPARE(model->rowCount(QModelIndex()), 1); VERIFYEXEC(Sink::Store::remove(res)); } void testAccountFilter() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup QString accountName("name"); QString accountIcon("icon"); auto account1 = ApplicationDomainType::createEntity(); account1.setAccountType("maildir"); account1.setName(accountName); account1.setIcon(accountIcon); VERIFYEXEC(Store::create(account1)); auto account2 = ApplicationDomainType::createEntity(); account2.setAccountType("maildir"); account2.setName(accountName); account2.setIcon(accountIcon); VERIFYEXEC(Store::create(account2)); auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setAccount(account1); Store::create(resource1).exec().waitForFinished(); auto resource2 = ApplicationDomainType::createEntity(); resource2.setResourceType("sink.dummy"); resource2.setAccount(account2); Store::create(resource2).exec().waitForFinished(); { Folder folder1(resource1.identifier()); VERIFYEXEC(Sink::Store::create(folder1)); Folder folder2(resource2.identifier()); VERIFYEXEC(Sink::Store::create(folder2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); // Test Sink::Query query; query.resourceFilter(account1); auto folders = Sink::Store::read(query); QCOMPARE(folders.size(), 1); } void testSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); auto mails = Sink::Store::read(query); QCOMPARE(mails.size(), 1); QCOMPARE(mails.first().getMessageId(), QByteArray("mail1")); } void testLiveSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //This folder should not make it through the query { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } //But this one should { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } QTRY_COMPARE(model->rowCount(), 2); } void testResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Sink::Store::create(Folder{resource1.identifier()})); VERIFYEXEC(Sink::Store::create(Folder{resource2.identifier()})); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource2.identifier())); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto folders = Sink::Store::read(Sink::Query{}.resourceContainsFilter("cap1")); QCOMPARE(folders.size(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testFilteredLiveResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); VERIFYEXEC(Store::create(Folder{resource1.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource1.identifier())); auto model = Sink::Store::loadModel(Query{Query::LiveQuery}.resourceContainsFilter("cap1")); QTRY_COMPARE(model->rowCount(), 1); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Store::create(Folder{resource2.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource2.identifier())); //The new resource should be filtered and thus not make it in here QCOMPARE(model->rowCount(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testLivequeryUnmatchInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setFolder(folder2); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testLivequeryRemoveOneInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); QCOMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 2); //After the removal, the thread size should be reduced by one { VERIFYEXEC(Sink::Store::remove(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); QTRY_COMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 1); //After the second removal, the thread should be gone { VERIFYEXEC(Sink::Store::remove(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testDontUpdateNonLiveQuery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; //Not a live query query.setFlags(Query::Flags{}); query.setId("testNoLiveQuery"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.request(); QVERIFY(!query.liveQuery()); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTest::qWait(100); QCOMPARE(mail->getUnread(), false); } void testLivequeryModifcationUpdateInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.setFlags(Query::LiveQuery); query.request(); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getUnread(), true); QCOMPARE(mail->getProperty("count").toInt(), 1); QCOMPARE(mail->getProperty("folders").toList().size(), 1); } void testReductionUpdate() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader should change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setUnread(false); mail2.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail2"}); QCOMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 0); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testBloom() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilterCreationInThread"); query.filter(mail1.identifier()); query.bloom(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); } void testLivequeryFilterCreationInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilterCreationInThread"); query.filter(mail1.identifier()); query.bloom(); query.sort(); query.setFlags(Query::LiveQuery); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //This modification should make it through { //This should not trigger an entity already in model warning mail1.setUnread(false); VERIFYEXEC(Sink::Store::modify(mail1)); } //This mail should make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } //This mail shouldn't make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); QTest::qWait(100); QCOMPARE(model->rowCount(), 2); //From mail2 QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 0); //From the modification QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testLivequeryThreadleaderChange() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader shouldn't change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setExtractedDate(earlier); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail1"}); QTRY_COMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); } QCOMPARE(insertedSpy.size(), 0); QCOMPARE(removedSpy.size(), 0); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //The leader should change to mail3 after the modification { auto mail3 = Mail::createEntity("sink.dummy.instance1"); mail3.setExtractedMessageId("mail3"); mail3.setFolder(folder1); mail3.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail3)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail3"}); QCOMPARE(mail->getProperty("count").toInt(), 3); QCOMPARE(mail->getProperty("folders").toList().size(), 3); } //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //Nothing should change on third mail in separate folder { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder2); mail.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 2); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } /* * Ensure that we handle the situation properly if the thread-leader doesn't match a property filter. */ void testFilteredThreadLeader() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId(messageid); mail.setFolder(folder); mail.setExtractedDate(date); mail.setImportant(important); return mail; }; VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, earlier, false))); VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, later, true))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); - query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); + query.reduce(Query::Reduce::Selector::max()).count().collect(); query.sort(); query.request(); query.filter(false); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QCOMPARE(mail->getMessageId(), QByteArray{"mail1"}); - QCOMPARE(mail->getProperty("count").toInt(), 2); - QCOMPARE(mail->getProperty("folders").toList().size(), 2); + QCOMPARE(mail->count(), 2); + QCOMPARE(mail->getCollectedProperty().size(), 2); } } /* * This test is here to ensure we don't crash if we call removeFromDisk with a running query. */ void testRemoveFromDiskWithRunningQuery() { { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testSecond"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-1)); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLatest"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLast"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.limit(1); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); } //FIXME: this will result in a crash in the above still running query. VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void testMailFulltextSubject() { // Setup { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Subject To Search"); msg->setBody("This is the searchable body."); msg->assemble(); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setExtractedSubject("Subject To Search"); mail.setFolder("folder1"); mail.setMimeMessage(msg->encodedContent()); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); mail.setExtractedSubject("Stuff"); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); } // Test { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject To Search"), QueryBase::Comparator::Fulltext)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Search"), QueryBase::Comparator::Fulltext)); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("search"), QueryBase::Comparator::Fulltext)); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("sear*"), QueryBase::Comparator::Fulltext)); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("searchable"), QueryBase::Comparator::Fulltext)); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder1"); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder2"); auto result = Sink::Store::read(query); QCOMPARE(result.size(), 0); } } }; QTEST_MAIN(QueryTest) #include "querytest.moc"