diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 08164db5..fce5089a 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -1,520 +1,528 @@ /* Copyright (c) 2015 Christian Mollekopf This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "typeindex.h" #include "log.h" #include "index.h" #include "fulltextindex.h" #include #include using namespace Sink; using Storage::Identifier; static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { QByteArray result; QDataStream ds(&result, QIODevice::WriteOnly); ds << value.toDateTime(); return result; } if (value.type() == QVariant::Bool) { return value.toBool() ? "t" : "f"; } if (value.canConvert()) { const auto ba = value.value().value; if (!ba.isEmpty()) { return ba; } } if (value.isValid() && !value.toByteArray().isEmpty()) { return value.toByteArray(); } // LMDB can't handle empty keys, so use something different return "toplevel"; } static QByteArray toSortableByteArrayImpl(const QDateTime &date) { // Sort invalid last if (!date.isValid()) { return QByteArray::number(std::numeric_limits::max()); } return padNumber(std::numeric_limits::max() - date.toTime_t()); } static QByteArray toSortableByteArray(const QVariant &value) { if (!value.isValid()) { // FIXME: we don't know the type, so we don't know what to return // This mean we're fixing every sorted index keys to use unsigned int return QByteArray::number(std::numeric_limits::max()); } if (value.canConvert()) { return toSortableByteArrayImpl(value.toDateTime()); } SinkWarning() << "Not knowing how to convert a" << value.typeName() << "to a sortable key, falling back to default conversion"; return getByteArray(value); } TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) { } QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &sortProperty) const { if (sortProperty.isEmpty()) { return mType + ".index." + property; } return mType + ".index." + property + ".sort." + sortProperty; } QByteArray TypeIndex::sortedIndexName(const QByteArray &property) const { return mType + ".index." + property + ".sorted"; } QByteArray TypeIndex::sampledPeriodIndexName(const QByteArray &rangeBeginProperty, const QByteArray &rangeEndProperty) const { return mType + ".index." + rangeBeginProperty + ".range." + rangeEndProperty; } static unsigned int bucketOf(const QVariant &value) { if (value.canConvert()) { return value.value().date().toJulianDay() / 7; } SinkError() << "Not knowing how to get the bucket of a" << value.typeName(); return {}; } static void update(TypeIndex::Action action, const QByteArray &indexName, const QByteArray &key, const QByteArray &value, Sink::Storage::DataStore::Transaction &transaction) { Index index(indexName, transaction); switch (action) { case TypeIndex::Add: index.add(key, value); break; case TypeIndex::Remove: index.remove(key, value); break; } } void TypeIndex::addProperty(const QByteArray &property) { auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { update(action, indexName(property), getByteArray(value), identifier.toInternalByteArray(), transaction); }; mIndexer.insert(property, indexer); mProperties << property; } template <> void TypeIndex::addSortedProperty(const QByteArray &property) { auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { update(action, sortedIndexName(property), toSortableByteArray(value), identifier.toInternalByteArray(), transaction); }; mSortIndexer.insert(property, indexer); mSortedProperties << property; } template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { const auto date = sortValue.toDateTime(); const auto propertyValue = getByteArray(value); update(action, indexName(property, sortProperty), propertyValue + toSortableByteArray(date), identifier.toInternalByteArray(), transaction); }; mGroupedSortIndexer.insert(property + sortProperty, indexer); mGroupedSortedProperties.insert(property, sortProperty); } template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { addPropertyWithSorting(property, sortProperty); } template <> void TypeIndex::addSampledPeriodIndex( const QByteArray &beginProperty, const QByteArray &endProperty) { auto indexer = [=](Action action, const Identifier &identifier, const QVariant &begin, const QVariant &end, Sink::Storage::DataStore::Transaction &transaction) { const auto beginDate = begin.toDateTime(); const auto endDate = end.toDateTime(); auto beginBucket = bucketOf(beginDate); auto endBucket = bucketOf(endDate); if (beginBucket > endBucket) { SinkError() << "End bucket greater than begin bucket"; return; } Index index(sampledPeriodIndexName(beginProperty, endProperty), transaction); for (auto bucket = beginBucket; bucket <= endBucket; ++bucket) { QByteArray bucketKey = padNumber(bucket); switch (action) { case TypeIndex::Add: index.add(bucketKey, identifier.toInternalByteArray()); break; case TypeIndex::Remove: index.remove(bucketKey, identifier.toInternalByteArray(), true); break; } } }; mSampledPeriodProperties.insert({ beginProperty, endProperty }); mSampledPeriodIndexer.insert({ beginProperty, endProperty }, indexer); } void TypeIndex::updateIndex(Action action, const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { for (const auto &property : mProperties) { const auto value = entity.getProperty(property); auto indexer = mIndexer.value(property); indexer(action, identifier, value, transaction); } for (const auto &properties : mSampledPeriodProperties) { auto indexer = mSampledPeriodIndexer.value(properties); auto indexRanges = entity.getProperty("indexRanges"); if (indexRanges.isValid()) { //This is to override the indexed ranges from the evenpreprocessor const auto list = indexRanges.value>>(); for (const auto &period : list) { indexer(action, identifier, period.first, period.second, transaction); } } else { //This is the regular case //NOTE Since we don't generate the ranges for removal we just end up trying to remove all possible buckets here instead. const auto beginValue = entity.getProperty(properties.first); const auto endValue = entity.getProperty(properties.second); indexer(action, identifier, beginValue, endValue, transaction); } } for (const auto &property : mSortedProperties) { const auto value = entity.getProperty(property); auto indexer = mSortIndexer.value(property); indexer(action, identifier, value, transaction); } for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { const auto value = entity.getProperty(it.key()); const auto sortValue = entity.getProperty(it.value()); auto indexer = mGroupedSortIndexer.value(it.key() + it.value()); indexer(action, identifier, value, sortValue, transaction); } } void TypeIndex::commitTransaction() { for (const auto &indexer : mCustomIndexer) { indexer->commitTransaction(); } } void TypeIndex::abortTransaction() { for (const auto &indexer : mCustomIndexer) { indexer->abortTransaction(); } } void TypeIndex::add(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Add, identifier, entity, transaction, resourceInstanceId); for (const auto &indexer : mCustomIndexer) { indexer->setup(this, &transaction, resourceInstanceId); indexer->add(entity); } } void TypeIndex::modify(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Remove, identifier, oldEntity, transaction, resourceInstanceId); updateIndex(Add, identifier, newEntity, transaction, resourceInstanceId); for (const auto &indexer : mCustomIndexer) { indexer->setup(this, &transaction, resourceInstanceId); indexer->modify(oldEntity, newEntity); } } void TypeIndex::remove(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Remove, identifier, entity, transaction, resourceInstanceId); for (const auto &indexer : mCustomIndexer) { indexer->setup(this, &transaction, resourceInstanceId); indexer->remove(entity); } } static QVector indexLookup(Index &index, QueryBase::Comparator filter, std::function valueToKey = getByteArray) { QVector keys; QByteArrayList lookupKeys; if (filter.comparator == Query::Comparator::Equals) { lookupKeys << valueToKey(filter.value); } else if (filter.comparator == Query::Comparator::In) { for (const QVariant &value : filter.value.value()) { lookupKeys << valueToKey(value); } } else { Q_ASSERT(false); } for (const auto &lookupKey : lookupKeys) { index.lookup(lookupKey, [&](const QByteArray &value) { keys << Identifier::fromInternalByteArray(value); }, [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, true); } return keys; } static QVector sortedIndexLookup(Index &index, QueryBase::Comparator filter) { if (filter.comparator == Query::Comparator::In || filter.comparator == Query::Comparator::Contains) { SinkWarning() << "In and Contains comparison not supported on sorted indexes"; } if (filter.comparator != Query::Comparator::Within) { return indexLookup(index, filter, toSortableByteArray); } QVector keys; QByteArray lowerBound, upperBound; auto bounds = filter.value.value(); if (bounds[0].canConvert()) { // Inverse the bounds because dates are stored newest first upperBound = toSortableByteArray(bounds[0].toDateTime()); lowerBound = toSortableByteArray(bounds[1].toDateTime()); } else { lowerBound = bounds[0].toByteArray(); upperBound = bounds[1].toByteArray(); } index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { - keys << Identifier::fromInternalByteArray(value); + const auto id = Identifier::fromInternalByteArray(value); + //Deduplicate because an id could be in multiple buckets + if (!keys.contains(id)) { + keys << id; + } }, [bounds](const Index::Error &error) { SinkWarning() << "Lookup error in index:" << error.message << "with bounds:" << bounds[0] << bounds[1]; }); return keys; } static QVector sampledIndexLookup(Index &index, QueryBase::Comparator filter) { if (filter.comparator != Query::Comparator::Overlap) { SinkWarning() << "Comparisons other than Overlap not supported on sampled period indexes"; return {}; } QVector keys; auto bounds = filter.value.value(); QByteArray lowerBound = toSortableByteArray(bounds[0]); QByteArray upperBound = toSortableByteArray(bounds[1]); QByteArray lowerBucket = padNumber(bucketOf(bounds[0])); QByteArray upperBucket = padNumber(bucketOf(bounds[1])); SinkTrace() << "Looking up from bucket:" << lowerBucket << "to:" << upperBucket; index.rangeLookup(lowerBucket, upperBucket, [&](const QByteArray &value) { - keys << Identifier::fromInternalByteArray(value); + const auto id = Identifier::fromInternalByteArray(value); + //Deduplicate because an id could be in multiple buckets + if (!keys.contains(id)) { + keys << id; + } }, [bounds](const Index::Error &error) { SinkWarning() << "Lookup error in index:" << error.message << "with bounds:" << bounds[0] << bounds[1]; }); return keys; } QVector TypeIndex::query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { const auto baseFilters = query.getBaseFilters(); for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { if (it.value().comparator == QueryBase::Comparator::Fulltext) { FulltextIndex fulltextIndex{resourceInstanceId}; QVector keys; const auto ids = fulltextIndex.lookup(it.value().value.toString()); keys.reserve(ids.size()); for (const auto &id : ids) { keys.append(Identifier::fromDisplayByteArray(id)); } appliedFilters << it.key(); SinkTraceCtx(mLogCtx) << "Fulltext index lookup found " << keys.size() << " keys."; return keys; } } for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { if (it.value().comparator == QueryBase::Comparator::Overlap) { if (mSampledPeriodProperties.contains({it.key()[0], it.key()[1]})) { Index index(sampledPeriodIndexName(it.key()[0], it.key()[1]), transaction); const auto keys = sampledIndexLookup(index, query.getFilter(it.key())); appliedFilters << it.key(); SinkTraceCtx(mLogCtx) << "Sampled period index lookup on" << it.key() << "found" << keys.size() << "keys."; return keys; } else { SinkWarning() << "Overlap search without sampled period index"; } } } for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { Index index(indexName(it.key(), it.value()), transaction); const auto keys = indexLookup(index, query.getFilter(it.key())); appliedFilters.insert({it.key()}); appliedSorting = it.value(); SinkTraceCtx(mLogCtx) << "Grouped sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; return keys; } } for (const auto &property : mSortedProperties) { if (query.hasFilter(property)) { Index index(sortedIndexName(property), transaction); const auto keys = sortedIndexLookup(index, query.getFilter(property)); appliedFilters.insert({property}); SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << property << " found " << keys.size() << " keys."; return keys; } } for (const auto &property : mProperties) { if (query.hasFilter(property)) { Index index(indexName(property), transaction); const auto keys = indexLookup(index, query.getFilter(property)); appliedFilters.insert({property}); SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } } SinkTraceCtx(mLogCtx) << "No matching index"; return {}; } QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties; if (mProperties.contains(property)) { QVector keys; Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); index.lookup(lookupKey, [&](const QByteArray &value) { keys << Identifier::fromInternalByteArray(value); }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } else if (mSecondaryProperties.contains(property)) { // Lookups on secondary indexes first lookup the key, and then lookup the results again to // resolve to entity id's QVector keys; auto resultProperty = mSecondaryProperties.value(property); QVector secondaryKeys; Index index(indexName(property + resultProperty), transaction); const auto lookupKey = getByteArray(value); index.lookup(lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTraceCtx(mLogCtx) << "Looked up secondary keys for the following lookup key: " << lookupKey << " => " << secondaryKeys; for (const auto &secondary : secondaryKeys) { keys += lookup(resultProperty, secondary, transaction); } return keys; } else { SinkWarning() << "Tried to lookup " << property << " but couldn't find value"; } return {}; } template <> void TypeIndex::index(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).add(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::index(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).add(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); } template <> QVector TypeIndex::secondaryLookup(const QByteArray &leftName, const QByteArray &rightName, const QVariant &value) { QVector keys; Index index(indexName(leftName + rightName), *mTransaction); const auto lookupKey = getByteArray(value); index.lookup( lookupKey, [&](const QByteArray &value) { keys << QByteArray{value.constData(), value.size()}; }, [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message << value << lookupKey; }); return keys; } template <> QVector TypeIndex::secondaryLookup(const QByteArray &leftName, const QByteArray &rightName, const QVariant &value) { return secondaryLookup(leftName, rightName, value); } diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 9aea9090..8c25c3f1 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1,1954 +1,1959 @@ #include #include #include #include "resource.h" #include "store.h" #include "resourcecontrol.h" #include "commands.h" #include "resourceconfig.h" #include "log.h" #include "modelresult.h" #include "test.h" #include "testutils.h" #include "applicationdomaintype.h" #include "queryrunner.h" #include "adaptorfactoryregistry.h" #include "fulltextindex.h" #include #include #include using namespace Sink; using namespace Sink::ApplicationDomain; /** * Test of the query system using the dummy resource. * * This test requires the dummy resource installed. */ class QueryTest : public QObject { Q_OBJECT private slots: void initTestCase() { Sink::Test::initTest(); auto factory = Sink::ResourceFactory::load("sink.dummy"); QVERIFY(factory); ResourceConfig::addResource("sink.dummy.instance1", "sink.dummy"); VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void cleanup() { VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void init() { qDebug(); qDebug() << "-----------------------------------------"; qDebug(); } void testSerialization() { auto type = QByteArray("type"); auto sort = QByteArray("sort"); Sink::QueryBase::Filter filter; filter.ids << "id"; filter.propertyFilter.insert({"foo"}, QVariant::fromValue(QByteArray("bar"))); Sink::Query query; query.setFilter(filter); query.setType(type); query.setSortProperty(sort); QByteArray data; { QDataStream stream(&data, QIODevice::WriteOnly); stream << query; } Sink::Query deserializedQuery; { QDataStream stream(&data, QIODevice::ReadOnly); stream >> deserializedQuery; } QCOMPARE(deserializedQuery.type(), type); QCOMPARE(deserializedQuery.sortProperty(), sort); QCOMPARE(deserializedQuery.getFilter().ids, filter.ids); QCOMPARE(deserializedQuery.getFilter().propertyFilter.keys(), filter.propertyFilter.keys()); QCOMPARE(deserializedQuery.getFilter().propertyFilter, filter.propertyFilter); } void testNoResources() { // Test Sink::Query query; query.resourceFilter("foobar"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } void testSingle() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); } void testSingleWithDelay() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testFilter() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); VERIFYEXEC(Sink::Store::create(mail)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter("folder1"); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value(); { mail->setFolder("folder2"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 0); { mail->setFolder("folder1"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 1); } void testById() { QByteArray id; // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); mail.setExtractedMessageId("test2"); VERIFYEXEC(Sink::Store::create(mail)); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed Sink::Store::synchronize(query).exec().waitForFinished(); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QVERIFY(model->rowCount() >= 1); id = model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->identifier(); } // Test { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(id); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); //Try a non-existing id query.filter("{87fcea5e-8d2e-408e-bb8d-b27b9dcf5e92}"); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } } void testFolder() { // Setup { Folder folder("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); } void testFolderTree() { // Setup { auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QCOMPARE(model->rowCount(model->index(0, 0)), 1); } void testIncrementalFolderTree() { // Setup auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); // Test Sink::Query query{Sink::Query::LiveQuery}; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Ensure the folder appears QTRY_COMPARE(model->rowCount(model->index(0, 0)), 1); //...and dissapears again after removal VERIFYEXEC(Sink::Store::remove(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(model->index(0, 0)), 0); } void testMailByMessageId() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } /* * Filter by two properties to make sure that we also use a non-index based filter. */ void testMailByMessageIdAndFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); Mail mail1("sink.dummy.instance1"); mail1.setExtractedMessageId("test1"); mail1.setFolder("foobar"); Sink::Store::create(mail1).exec().waitForFinished(); Mail mail2("sink.dummy.instance1"); mail2.setExtractedMessageId("test2"); mail2.setFolder(folderEntity->identifier()); Sink::Store::create(mail2).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolderSortedByDate() { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testSecond"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-1)); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLatest"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLast"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.limit(1); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); // The model is not sorted, but the limited set is sorted, so we can only test for the latest result. QCOMPARE(model->rowCount(), 1); QCOMPARE(model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testLatest")); model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); // We can't make any assumptions about the order of the indexes // QCOMPARE(model->index(1, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testSecond")); //New revisions always go through { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testInjected"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } QTRY_COMPARE(model->rowCount(), 3); //Ensure we can continue fetching after the incremental update model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); //Ensure we have fetched all model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); } void testReactToNewResource() { Sink::Query query; query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(QModelIndex()), 0); auto res = DummyResource::create(""); VERIFYEXEC(Sink::Store::create(res)); auto folder = Folder::create(res.identifier()); VERIFYEXEC(Sink::Store::create(folder)); QTRY_COMPARE(model->rowCount(QModelIndex()), 1); VERIFYEXEC(Sink::Store::remove(res)); } void testAccountFilter() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup QString accountName("name"); QString accountIcon("icon"); auto account1 = ApplicationDomainType::createEntity(); account1.setAccountType("maildir"); account1.setName(accountName); account1.setIcon(accountIcon); VERIFYEXEC(Store::create(account1)); auto account2 = ApplicationDomainType::createEntity(); account2.setAccountType("maildir"); account2.setName(accountName); account2.setIcon(accountIcon); VERIFYEXEC(Store::create(account2)); auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setAccount(account1); Store::create(resource1).exec().waitForFinished(); auto resource2 = ApplicationDomainType::createEntity(); resource2.setResourceType("sink.dummy"); resource2.setAccount(account2); Store::create(resource2).exec().waitForFinished(); { Folder folder1(resource1.identifier()); VERIFYEXEC(Sink::Store::create(folder1)); Folder folder2(resource2.identifier()); VERIFYEXEC(Sink::Store::create(folder2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); // Test Sink::Query query; query.resourceFilter(account1); auto folders = Sink::Store::read(query); QCOMPARE(folders.size(), 1); } void testSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); auto mails = Sink::Store::read(query); QCOMPARE(mails.size(), 1); QCOMPARE(mails.first().getMessageId(), QByteArray("mail1")); } void testLiveSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //This folder should not make it through the query { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } //But this one should { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } QTRY_COMPARE(model->rowCount(), 2); } void testResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Sink::Store::create(Folder{resource1.identifier()})); VERIFYEXEC(Sink::Store::create(Folder{resource2.identifier()})); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource2.identifier())); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto folders = Sink::Store::read(Sink::Query{}.resourceContainsFilter("cap1")); QCOMPARE(folders.size(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testFilteredLiveResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); VERIFYEXEC(Store::create(Folder{resource1.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource1.identifier())); auto model = Sink::Store::loadModel(Query{Query::LiveQuery}.resourceContainsFilter("cap1")); QTRY_COMPARE(model->rowCount(), 1); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Store::create(Folder{resource2.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource2.identifier())); //The new resource should be filtered and thus not make it in here QCOMPARE(model->rowCount(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testLivequeryUnmatchInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setFolder(folder2); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testLivequeryRemoveOneInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); QCOMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 2); //After the removal, the thread size should be reduced by one { VERIFYEXEC(Sink::Store::remove(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); QTRY_COMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 1); //After the second removal, the thread should be gone { VERIFYEXEC(Sink::Store::remove(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testDontUpdateNonLiveQuery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; //Not a live query query.setFlags(Query::Flags{}); query.setId("testNoLiveQuery"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.request(); QVERIFY(!query.liveQuery()); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTest::qWait(100); QCOMPARE(mail->getUnread(), false); } void testLivequeryModifcationUpdateInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.setFlags(Query::LiveQuery); query.request(); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getUnread(), true); QCOMPARE(mail->getProperty("count").toInt(), 1); QCOMPARE(mail->getProperty("folders").toList().size(), 1); } void testReductionUpdate() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader should change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setUnread(false); mail2.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail2"}); QCOMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 0); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testFilteredReductionUpdate() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilteredReductionUpdate"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //Ensure we don't end up with a mail in the thread that was filtered //This tests the case of an otherwise emtpy thread on purpose. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("filtered"); mail.setFolder(folder2); mail.setExtractedDate(QDateTime{QDate{2017, 2, 3}, QTime{11, 0, 0}}); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QCOMPARE(model->rowCount(), 0); //Ensure the non-filtered still get through. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("not-filtered"); mail.setFolder(folder1); mail.setExtractedDate(QDateTime{QDate{2017, 2, 3}, QTime{11, 0, 0}}); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); } /* * Two messages in the same thread. The first get's filtered, the second one makes it. */ void testFilteredReductionUpdateInSameThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilteredReductionUpdate"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count"); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); //The first message will be filtered (but would be aggreagted together with the message that passes) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); //Ensure that we can deal with a modification to the filtered message mail.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QCOMPARE(model->rowCount(), 0); //Ensure the non-filtered still gets through. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); //Ensure that we can deal with a modification to the filtered message mail.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); QCOMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 1); //Ensure another entity still results in a modification { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 2); } void testBloom() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.resourceFilter("sink.dummy.instance1"); query.setId("testFilterCreationInThread"); query.filter(mail1.identifier()); query.bloom(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); } void testLivequeryFilterCreationInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilterCreationInThread"); query.resourceFilter("sink.dummy.instance1"); query.filter(mail1.identifier()); query.bloom(); query.sort(); query.setFlags(Query::LiveQuery); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //This modification should make it through { //This should not trigger an entity already in model warning mail1.setUnread(false); VERIFYEXEC(Sink::Store::modify(mail1)); } //This mail should make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } //This mail shouldn't make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); QTest::qWait(100); QCOMPARE(model->rowCount(), 2); //From mail2 QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 0); //From the modification QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testLivequeryThreadleaderChange() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader shouldn't change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setExtractedDate(earlier); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail1"}); QTRY_COMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); } QCOMPARE(insertedSpy.size(), 0); QCOMPARE(removedSpy.size(), 0); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //The leader should change to mail3 after the modification { auto mail3 = Mail::createEntity("sink.dummy.instance1"); mail3.setExtractedMessageId("mail3"); mail3.setFolder(folder1); mail3.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail3)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail3"}); QCOMPARE(mail->getProperty("count").toInt(), 3); QCOMPARE(mail->getProperty("folders").toList().size(), 3); } //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //Nothing should change on third mail in separate folder { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder2); mail.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 2); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } /* * Ensure that we handle the situation properly if the thread-leader doesn't match a property filter. */ void testFilteredThreadLeader() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId(messageid); mail.setFolder(folder); mail.setExtractedDate(date); mail.setImportant(important); return mail; }; VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, earlier, false))); VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, later, true))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()).count().collect(); query.sort(); query.request(); query.filter(false); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QCOMPARE(mail->getMessageId(), QByteArray{"mail1"}); QCOMPARE(mail->count(), 2); QCOMPARE(mail->getCollectedProperty().size(), 2); } } void testQueryRunnerDontMissUpdates() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId(messageid); mail.setFolder(folder); mail.setExtractedDate(date); mail.setImportant(important); return mail; }; VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setFlags(Query::LiveQuery); Sink::ResourceContext resourceContext{"sink.dummy.instance1", "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}; Sink::Log::Context logCtx; auto runner = new QueryRunner(query, resourceContext, ApplicationDomain::getTypeName(), logCtx); runner->delayNextQuery(); auto emitter = runner->emitter(); QList added; emitter->onAdded([&](Mail::Ptr mail) { added << mail; }); emitter->fetch(); VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, now, false))); QTRY_COMPARE(added.size(), 2); runner->delayNextQuery(); VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, now, false))); //The second revision update is supposed to come in while the initial revision update is still in the query. //So wait a bit to make sure the query is currently runnning. QTest::qWait(500); VERIFYEXEC(Sink::Store::create(createMail("mail4", folder1, now, false))); QTRY_COMPARE(added.size(), 4); } /* * This test excercises the scenario where a fetchMore is triggered after * the revision is already updated in storage, but the incremental query was not run yet. * This resulted in lost modification updates. */ void testQueryRunnerDontMissUpdatesWithFetchMore() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setName("name1"); VERIFYEXEC(Sink::Store::create(folder1)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setFlags(Query::LiveQuery); Sink::ResourceContext resourceContext{"sink.dummy.instance1", "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}; Sink::Log::Context logCtx; auto runner = new QueryRunner(query, resourceContext, ApplicationDomain::getTypeName(), logCtx); auto emitter = runner->emitter(); QList added; emitter->onAdded([&](Folder::Ptr folder) { added << folder; }); QList modified; emitter->onModified([&](Folder::Ptr folder) { modified << folder; }); emitter->fetch(); QTRY_COMPARE(added.size(), 1); QCOMPARE(modified.size(), 0); runner->ignoreRevisionChanges(true); folder1.setName("name2"); VERIFYEXEC(Sink::Store::modify(folder1)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); emitter->fetch(); runner->ignoreRevisionChanges(false); runner->triggerRevisionChange(); QTRY_COMPARE(added.size(), 1); QTRY_COMPARE(modified.size(), 1); } /* * This test is here to ensure we don't crash if we call removeFromDisk with a running query. */ void testRemoveFromDiskWithRunningQuery() { // FIXME: we currently crash QSKIP("Skipping because this produces a crash."); { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); //Add enough data so the query takes long enough that we remove the data from disk whlie the query is ongoing. for (int i = 0; i < 100; i++) { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test" + QByteArray::number(i)); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(i)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); } //FIXME: this will result in a crash in the above still running query. VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void testMailFulltext() { QByteArray id1; QByteArray id2; // Setup { { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Subject To Search"); msg->setBody("This is the searchable body bar. unique sender2"); msg->from()->from7BitString("\"The Sender\""); msg->to()->from7BitString("\"Foo Bar\""); msg->assemble(); auto mail = ApplicationDomainType::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); mail.setMimeMessage(msg->encodedContent()); VERIFYEXEC(Sink::Store::create(mail)); id1 = mail.identifier(); } { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Stuff to Search"); msg->setBody("Body foo bar"); msg->from()->from7BitString("\"Another Sender2\""); msg->assemble(); auto mail = ApplicationDomainType::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); mail.setMimeMessage(msg->encodedContent()); VERIFYEXEC(Sink::Store::create(mail)); id2 = mail.identifier(); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { FulltextIndex index("sink.dummy.instance1", Sink::Storage::DataStore::ReadOnly); qInfo() << QString("Found document 1 with terms: ") + index.getIndexContent(id1).terms.join(", "); qInfo() << QString("Found document 2 with terms: ") + index.getIndexContent(id2).terms.join(", "); } } // Test // Default search { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject To Search"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } // Phrase search { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("\"Subject To Search\""), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("\"Stuff to Search\""), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); } //Operators { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject AND search"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject OR search"), QueryBase::Comparator::Fulltext)); QCOMPARE(Sink::Store::read(query).size(), 2); } //Case-insensitive { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Case-insensitive { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Partial match { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subj"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by body { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("searchable"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by folder { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder1"); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by folder { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder2"); QCOMPARE(Sink::Store::read(query).size(), 0); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 2); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("Sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 2); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender@example"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("The Sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender2@unique.com"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id2); } //Filter by recipient { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("foo-bar@example.org"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by recipient { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("foo-bar@example.com"), Sink::QueryBase::Comparator::Fulltext)); QCOMPARE(Sink::Store::read(query).size(), 0); } } void mailsWithDates() { { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); mail.setExtractedMessageId("message1"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:50:00Z", Qt::ISODate)); mail.setExtractedMessageId("message2"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); mail.setExtractedMessageId("message3"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("message4"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2078-05-23T13:49:41Z", Qt::ISODate)); mail.setExtractedMessageId("message5"); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); } void testMailDate() { mailsWithDates(); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-27T13:49:41Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } } void testMailRange() { mailsWithDates(); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-25T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 3); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2118-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); } } void testOverlap() { auto createEvent = [] (const QString &start, const QString &end) { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString(start, Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString(end, Qt::ISODate)); Event event("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC(Sink::Store::create(event)); }; createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); createEvent("2018-05-23T13:00:00Z", "2018-05-23T14:00:00Z"); createEvent("2018-05-23T14:00:00Z", "2018-05-23T15:00:00Z"); createEvent("2018-05-24T12:00:00Z", "2018-05-24T14:00:00Z"); + //Long event that spans multiple buckets + createEvent("2018-05-30T22:00:00", "2019-04-25T03:00:00"); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); auto findInRange = [] (const QString &start, const QString &end) { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString(start, Qt::ISODate), QDateTime::fromString(end, Qt::ISODate) }, QueryBase::Comparator::Overlap)); return Sink::Store::read(query); }; //Find all QCOMPARE(findInRange("2018-05-22T12:00:00Z", "2018-05-30T13:00:00Z").size(), 4); //Find none on day without events QCOMPARE(findInRange("2018-05-22T12:00:00Z", "2018-05-22T13:00:00Z").size(), 0); //Find none on day with events QCOMPARE(findInRange("2018-05-24T10:00:00Z", "2018-05-24T11:00:00Z").size(), 0); //Find on same day QCOMPARE(findInRange("2018-05-23T12:30:00Z", "2018-05-23T12:31:00Z").size(), 1); //Find on different days QCOMPARE(findInRange("2018-05-22T12:30:00Z", "2018-05-23T12:00:00Z").size(), 1); QCOMPARE(findInRange("2018-05-23T14:30:00Z", "2018-05-23T16:00:00Z").size(), 1); + + //Find long range event + QCOMPARE(findInRange("2018-07-23T14:30:00Z", "2018-10-23T16:00:00Z").size(), 1); } void testOverlapLive() { auto createEvent = [] (const QString &start, const QString &end) { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString(start, Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString(end, Qt::ISODate)); Event event = Event::createEntity("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC_RET(Sink::Store::create(event), {}); return event; }; createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); createEvent("2018-05-23T13:00:00Z", "2018-05-23T14:00:00Z"); createEvent("2018-05-23T14:00:00Z", "2018-05-23T15:00:00Z"); createEvent("2018-05-24T12:00:00Z", "2018-05-24T14:00:00Z"); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString("2018-05-22T12:00:00Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:00:00Z", Qt::ISODate) }, QueryBase::Comparator::Overlap)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); auto event1 = createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); auto event2 = createEvent("2018-05-31T12:00:00Z", "2018-05-31T13:00:00Z"); QTRY_COMPARE(model->rowCount(), 5); VERIFYEXEC(Sink::Store::remove(event1)); VERIFYEXEC(Sink::Store::remove(event2)); QTRY_COMPARE(model->rowCount(), 4); } } void testRecurringEvents() { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString("2018-05-10T13:00:00Z", Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString("2018-05-10T14:00:00Z", Qt::ISODate)); icalEvent->recurrence()->setWeekly(3); Event event = Event::createEntity("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC(Sink::Store::create(event)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString("2018-05-15T12:00:00Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:00:00Z", Qt::ISODate) }, QueryBase::Comparator::Overlap)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); VERIFYEXEC(Sink::Store::remove(event)); QTRY_COMPARE(model->rowCount(), 0); } }; QTEST_MAIN(QueryTest) #include "querytest.moc"