diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp index 43b46603..0d1a87dd 100644 --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -1,721 +1,775 @@ /* * Copyright (C) 2016 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "datastorequery.h" #include "log.h" #include "applicationdomaintype.h" using namespace Sink; using namespace Sink::Storage; static QByteArray operationName(const Sink::Operation op) { switch(op) { case Sink::Operation_Creation: return "Creation"; case Sink::Operation_Modification: return "Modification"; case Sink::Operation_Removal: return "Removal"; } return ""; } +static bool compare(const QVariant &left, const QVariant &right, QueryBase::Reduce::Selector::Comparator comparator) { + if (comparator == QueryBase::Reduce::Selector::Max) { + return left > right; + } + if (comparator == QueryBase::Reduce::Selector::Min) { + return left < right; + } + return false; +} + + class Source : public FilterBase { public: typedef QSharedPointer Ptr; QVector mIds; QVector::ConstIterator mIt; QVector mIncrementalIds; QVector::ConstIterator mIncrementalIt; Source (const QVector &ids, DataStoreQuery *store) : FilterBase(store), mIds(ids) { mIt = mIds.constBegin(); } virtual ~Source(){} virtual void skip() Q_DECL_OVERRIDE { if (mIt != mIds.constEnd()) { mIt++; } }; void add(const QVector &keys) { mIncrementalIds.clear(); mIncrementalIds.reserve(keys.size()); for (const auto &key : keys) { mIncrementalIds.append(key.identifier()); } mIncrementalIt = mIncrementalIds.constBegin(); } bool next(const std::function &callback) Q_DECL_OVERRIDE { if (!mIncrementalIds.isEmpty()) { if (mIncrementalIt == mIncrementalIds.constEnd()) { return false; } readEntity(*mIncrementalIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); callback({entity, operation}); }); mIncrementalIt++; if (mIncrementalIt == mIncrementalIds.constEnd()) { return false; } return true; } else { if (mIt == mIds.constEnd()) { return false; } readEntity(*mIt, [this, callback](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { SinkTraceCtx(mDatastore->mLogCtx) << "Source: Read entity: " << entity.identifier() << operationName(operation); callback({entity, operation}); }); mIt++; return mIt != mIds.constEnd(); } } }; class Collector : public FilterBase { public: typedef QSharedPointer Ptr; Collector(FilterBase::Ptr source, DataStoreQuery *store) : FilterBase(source, store) { } virtual ~Collector(){} bool next(const std::function &callback) Q_DECL_OVERRIDE { return mSource->next(callback); } }; class Filter : public FilterBase { public: typedef QSharedPointer Ptr; QHash propertyFilter; Filter(FilterBase::Ptr source, DataStoreQuery *store) : FilterBase(source, store) { } virtual ~Filter(){} virtual bool next(const std::function &callback) Q_DECL_OVERRIDE { bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { SinkTraceCtx(mDatastore->mLogCtx) << "Filter: " << result.entity.identifier() << operationName(result.operation); //Always accept removals. They can't match the filter since the data is gone. if (result.operation == Sink::Operation_Removal) { SinkTraceCtx(mDatastore->mLogCtx) << "Removal: " << result.entity.identifier() << operationName(result.operation); callback(result); foundValue = true; } else if (matchesFilter(result.entity)) { SinkTraceCtx(mDatastore->mLogCtx) << "Accepted: " << result.entity.identifier() << operationName(result.operation); callback(result); foundValue = true; //TODO if something did not match the filter so far but does now, turn into an add operation. } else { SinkTraceCtx(mDatastore->mLogCtx) << "Rejected: " << result.entity.identifier() << operationName(result.operation); //TODO emit a removal if we had the uid in the result set and this is a modification. //We don't know if this results in a removal from the dataset, so we emit a removal notification anyways callback({result.entity, Sink::Operation_Removal, result.aggregateValues}); } return false; })) {} return foundValue; } bool matchesFilter(const ApplicationDomain::ApplicationDomainType &entity) { for (const auto &filterProperty : propertyFilter.keys()) { QVariant property; if (filterProperty.size() == 1) { property = entity.getProperty(filterProperty[0]); } else { QVariantList propList; for (const auto &propName : filterProperty) { propList.push_back(entity.getProperty(propName)); } property = propList; } const auto comparator = propertyFilter.value(filterProperty); //We can't deal with a fulltext filter if (comparator.comparator == QueryBase::Comparator::Fulltext) { continue; } if (!comparator.matches(property)) { SinkTraceCtx(mDatastore->mLogCtx) << "Filtering entity due to property mismatch on filter: " << entity.identifier() << "Property: " << filterProperty << property << " Filter:" << comparator.value; return false; } } return true; } }; class Reduce : public Filter { public: typedef QSharedPointer Ptr; struct Aggregator { QueryBase::Reduce::Aggregator::Operation operation; QByteArray property; QByteArray resultProperty; Aggregator(QueryBase::Reduce::Aggregator::Operation o, const QByteArray &property_, const QByteArray &resultProperty_) : operation(o), property(property_), resultProperty(resultProperty_) { } void process(const QVariant &value) { if (operation == QueryBase::Reduce::Aggregator::Collect) { mResult = mResult.toList() << value; } else if (operation == QueryBase::Reduce::Aggregator::Count) { mResult = mResult.toInt() + 1; } else { Q_ASSERT(false); } } void reset() { mResult.clear(); } QVariant result() const { return mResult; } private: QVariant mResult; }; + struct PropertySelector { + QueryBase::Reduce::Selector selector; + QByteArray resultProperty; + + PropertySelector(QueryBase::Reduce::Selector s, const QByteArray &resultProperty_) + : selector(s), resultProperty(resultProperty_) + { + + } + + void process(const QVariant &value, const QVariant &selectionValue) { + if (!selectionResultValue.isValid() || compare(selectionValue, selectionResultValue, selector.comparator)) { + selectionResultValue = selectionValue; + mResult = value; + } + } + + void reset() + { + selectionResultValue.clear(); + mResult.clear(); + } + + QVariant result() const + { + return mResult; + } + private: + + QVariant selectionResultValue; + QVariant mResult; + }; + QSet mReducedValues; QSet mIncrementallyReducedValues; QHash mSelectedValues; QByteArray mReductionProperty; QByteArray mSelectionProperty; QueryBase::Reduce::Selector::Comparator mSelectionComparator; QList mAggregators; + QList mSelectors; Reduce(const QByteArray &reductionProperty, const QByteArray &selectionProperty, QueryBase::Reduce::Selector::Comparator comparator, FilterBase::Ptr source, DataStoreQuery *store) : Filter(source, store), mReductionProperty(reductionProperty), mSelectionProperty(selectionProperty), mSelectionComparator(comparator) { } virtual ~Reduce(){} void updateComplete() Q_DECL_OVERRIDE { SinkTraceCtx(mDatastore->mLogCtx) << "Reduction update is complete."; mIncrementallyReducedValues.clear(); } static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { return value.toDateTime().toString().toLatin1(); } if (value.isValid() && !value.toByteArray().isEmpty()) { return value.toByteArray(); } return QByteArray(); } - static bool compare(const QVariant &left, const QVariant &right, QueryBase::Reduce::Selector::Comparator comparator) { - if (comparator == QueryBase::Reduce::Selector::Max) { - return left > right; - } - return false; - } - struct ReductionResult { Identifier selection; QVector aggregateIds; QMap aggregateValues; }; ReductionResult reduceOnValue(const QVariant &reductionValue) { QMap aggregateValues; QVariant selectionResultValue; Identifier selectionResult; const auto results = indexLookup(mReductionProperty, reductionValue); for (auto &aggregator : mAggregators) { aggregator.reset(); } + for (auto &selector : mSelectors) { + selector.reset(); + } QVector reducedAndFilteredResults; for (const auto &r : results) { readEntity(r, [&, this](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { //We need to apply all property filters that we have until the reduction, because the index lookup was unfiltered. if (!matchesFilter(entity)) { return; } reducedAndFilteredResults << r; Q_ASSERT(operation != Sink::Operation_Removal); for (auto &aggregator : mAggregators) { if (!aggregator.property.isEmpty()) { aggregator.process(entity.getProperty(aggregator.property)); } else { aggregator.process(QVariant{}); } } - auto selectionValue = entity.getProperty(mSelectionProperty); + + const auto selectionValue = entity.getProperty(mSelectionProperty); + + for (auto &selector : mSelectors) { + if (!selector.selector.property.isEmpty()) { + selector.process(entity.getProperty(selector.selector.property), selectionValue); + } + } if (!selectionResultValue.isValid() || compare(selectionValue, selectionResultValue, mSelectionComparator)) { selectionResultValue = selectionValue; selectionResult = Identifier::fromDisplayByteArray(entity.identifier()); } }); } - for (auto &aggregator : mAggregators) { + for (const auto &aggregator : mAggregators) { aggregateValues.insert(aggregator.resultProperty, aggregator.result()); } + for (const auto &selector : mSelectors) { + aggregateValues.insert(selector.resultProperty, selector.result()); + } return {selectionResult, reducedAndFilteredResults, aggregateValues}; } bool next(const std::function &callback) Q_DECL_OVERRIDE { bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { const auto reductionValue = [&] { const auto v = result.entity.getProperty(mReductionProperty); //Because we also get Operation_Removal for filtered entities. We use the fact that actually removed entites //won't have the property to reduce on. //TODO: Perhaps find a cleaner solutoin than abusing Operation::Removed for filtered properties. if (v.isNull() && result.operation == Sink::Operation_Removal) { //For removals we have to read the last revision to get a value, and thus be able to find the correct thread. QVariant reductionValue; const auto id = Identifier::fromDisplayByteArray(result.entity.identifier()); readPrevious(id, [&] (const ApplicationDomain::ApplicationDomainType &prev) { Q_ASSERT(result.entity.identifier() == prev.identifier()); reductionValue = prev.getProperty(mReductionProperty); }); return reductionValue; } else { return v; } }(); if (reductionValue.isNull()) { SinkTraceCtx(mDatastore->mLogCtx) << "No reduction value: " << result.entity.identifier(); //We failed to find a value to reduce on, so ignore this entity. //Can happen if the entity was already removed and we have no previous revision. return; } const auto reductionValueBa = getByteArray(reductionValue); if (!mReducedValues.contains(reductionValueBa)) { //Only reduce every value once. mReducedValues.insert(reductionValueBa); SinkTraceCtx(mDatastore->mLogCtx) << "Reducing new value: " << result.entity.identifier() << reductionValueBa; auto reductionResult = reduceOnValue(reductionValue); //This can happen if we get a removal message from a filtered entity and all entites of the reduction are filtered. if (reductionResult.selection.isNull()) { return; } mSelectedValues.insert(reductionValueBa, reductionResult.selection); readEntity(reductionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { callback({entity, operation, reductionResult.aggregateValues, reductionResult.aggregateIds}); foundValue = true; }); } else { //During initial query, do nothing. The lookup above will take care of it. //During updates adjust the reduction according to the modification/addition or removal //We have to redo the reduction for every element, because of the aggregation values. if (mIncremental && !mIncrementallyReducedValues.contains(reductionValueBa)) { SinkTraceCtx(mDatastore->mLogCtx) << "Incremental reduction update: " << result.entity.identifier() << reductionValueBa; mIncrementallyReducedValues.insert(reductionValueBa); //Redo the reduction to find new aggregated values auto selectionResult = reduceOnValue(reductionValue); //If mSelectedValues did not contain the value, oldSelectionResult will be empty.(Happens if entites have been filtered) auto oldSelectionResult = mSelectedValues.take(reductionValueBa); SinkTraceCtx(mDatastore->mLogCtx) << "Old selection result: " << oldSelectionResult << " New selection result: " << selectionResult.selection; if (selectionResult.selection.isNull() && oldSelectionResult.isNull()) { //Nothing to do, the item was filtered before, and still is. } else if (oldSelectionResult == selectionResult.selection) { mSelectedValues.insert(reductionValueBa, selectionResult.selection); Q_ASSERT(!selectionResult.selection.isNull()); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Modification, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } else { //remove old result if (!oldSelectionResult.isNull()) { readEntity(oldSelectionResult, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Removal}); }); } //If the last item has been removed, then there's nothing to add if (!selectionResult.selection.isNull()) { //add new result mSelectedValues.insert(reductionValueBa, selectionResult.selection); Q_ASSERT(!selectionResult.selection.isNull()); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Creation, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } } } } })) {} return foundValue; } }; class Bloom : public Filter { public: typedef QSharedPointer Ptr; QByteArray mBloomProperty; Bloom(const QByteArray &bloomProperty, FilterBase::Ptr source, DataStoreQuery *store) : Filter(source, store), mBloomProperty(bloomProperty) { } virtual ~Bloom(){} bool next(const std::function &callback) Q_DECL_OVERRIDE { if (!mBloomed) { //Initially we bloom on the first value that matches. //From there on we just filter. bool foundValue = false; while(!foundValue && mSource->next([this, callback, &foundValue](const ResultSet::Result &result) { mBloomValue = result.entity.getProperty(mBloomProperty); auto results = indexLookup(mBloomProperty, mBloomValue); for (const auto &r : results) { readEntity(r, [&, this](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { callback({entity, Sink::Operation_Creation}); SinkTraceCtx(mDatastore->mLogCtx) << "Bloom result: " << entity.identifier() << operationName(operation); foundValue = true; }); } return false; })) {} mBloomed = true; propertyFilter.insert({mBloomProperty}, mBloomValue); return foundValue; } else { //Filter on bloom value return Filter::next(callback); } } QVariant mBloomValue; bool mBloomed = false; }; DataStoreQuery::DataStoreQuery(const Sink::QueryBase &query, const QByteArray &type, EntityStore &store) : mType(type), mStore(store), mLogCtx(store.logContext().subContext("datastorequery")) { //This is what we use during a new query setupQuery(query); } DataStoreQuery::DataStoreQuery(const DataStoreQuery::State &state, const QByteArray &type, Sink::Storage::EntityStore &store, bool incremental) : mType(type), mStore(store), mLogCtx(store.logContext().subContext("datastorequery")) { //This is what we use when fetching more data, without having a new revision with incremental=false //And this is what we use when the data changed and we want to update with incremental = true mCollector = state.mCollector; mSource = state.mSource; auto source = mCollector; while (source) { source->mDatastore = this; source->mIncremental = incremental; source = source->mSource; } } DataStoreQuery::~DataStoreQuery() { } DataStoreQuery::State::Ptr DataStoreQuery::getState() { auto state = State::Ptr::create(); state->mSource = mSource; state->mCollector = mCollector; return state; } void DataStoreQuery::readEntity(const Identifier &id, const BufferCallback &resultCallback) { mStore.readLatest(mType, id, resultCallback); } void DataStoreQuery::readPrevious(const Identifier &id, const std::function &callback) { mStore.readPrevious(mType, id, mStore.maxRevision(), callback); } QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) { return mStore.indexLookup(mType, property, value); } /* ResultSet DataStoreQuery::filterAndSortSet(ResultSet &resultSet, const FilterFunction &filter, const QByteArray &sortProperty) */ /* { */ /* const bool sortingRequired = !sortProperty.isEmpty(); */ /* if (mInitialQuery && sortingRequired) { */ /* SinkTrace() << "Sorting the resultset in memory according to property: " << sortProperty; */ /* // Sort the complete set by reading the sort property and filling into a sorted map */ /* auto sortedMap = QSharedPointer>::create(); */ /* while (resultSet.next()) { */ /* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */ /* readEntity(resultSet.id(), */ /* [this, filter, sortedMap, sortProperty, &resultSet](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* const auto operation = buffer.operation(); */ /* // We're not interested in removals during the initial query */ /* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */ /* if (!sortProperty.isEmpty()) { */ /* const auto sortValue = getProperty(buffer.entity(), sortProperty); */ /* if (sortValue.type() == QVariant::DateTime) { */ /* sortedMap->insert(QByteArray::number(std::numeric_limits::max() - sortValue.toDateTime().toTime_t()), uid); */ /* } else { */ /* sortedMap->insert(sortValue.toString().toLatin1(), uid); */ /* } */ /* } else { */ /* sortedMap->insert(uid, uid); */ /* } */ /* } */ /* }); */ /* } */ /* SinkTrace() << "Sorted " << sortedMap->size() << " values."; */ /* auto iterator = QSharedPointer>::create(*sortedMap); */ /* ResultSet::ValueGenerator generator = [this, iterator, sortedMap, filter]( */ /* std::function callback) -> bool { */ /* if (iterator->hasNext()) { */ /* readEntity(iterator->next().value(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* callback(uid, buffer, Sink::Operation_Creation); */ /* }); */ /* return true; */ /* } */ /* return false; */ /* }; */ /* auto skip = [iterator]() { */ /* if (iterator->hasNext()) { */ /* iterator->next(); */ /* } */ /* }; */ /* return ResultSet(generator, skip); */ /* } else { */ /* auto resultSetPtr = QSharedPointer::create(resultSet); */ /* ResultSet::ValueGenerator generator = [this, resultSetPtr, filter](const ResultSet::Callback &callback) -> bool { */ /* if (resultSetPtr->next()) { */ /* SinkTrace() << "Reading the next value: " << resultSetPtr->id(); */ /* // readEntity is only necessary if we actually want to filter or know the operation type (but not a big deal if we do it always I guess) */ /* readEntity(resultSetPtr->id(), [this, filter, callback](const QByteArray &uid, const Sink::EntityBuffer &buffer) { */ /* const auto operation = buffer.operation(); */ /* if (mInitialQuery) { */ /* // We're not interested in removals during the initial query */ /* if ((operation != Sink::Operation_Removal) && filter(uid, buffer)) { */ /* // In the initial set every entity is new */ /* callback(uid, buffer, Sink::Operation_Creation); */ /* } */ /* } else { */ /* // Always remove removals, they probably don't match due to non-available properties */ /* if ((operation == Sink::Operation_Removal) || filter(uid, buffer)) { */ /* // TODO only replay if this is in the currently visible set (or just always replay, worst case we have a couple to many results) */ /* callback(uid, buffer, operation); */ /* } */ /* } */ /* }); */ /* return true; */ /* } */ /* return false; */ /* }; */ /* auto skip = [resultSetPtr]() { resultSetPtr->skip(1); }; */ /* return ResultSet(generator, skip); */ /* } */ /* } */ QByteArrayList DataStoreQuery::executeSubquery(const QueryBase &subquery) { Q_ASSERT(!subquery.type().isEmpty()); auto sub = DataStoreQuery(subquery, subquery.type(), mStore); auto result = sub.execute(); QByteArrayList ids; while (result.next([&ids](const ResultSet::Result &result) { ids << result.entity.identifier(); })) {} return ids; } void DataStoreQuery::setupQuery(const Sink::QueryBase &query_) { auto query = query_; auto baseFilters = query.getBaseFilters(); //Resolve any subqueries we have for (const auto &k : baseFilters.keys()) { const auto comparator = baseFilters.value(k); if (comparator.value.canConvert()) { SinkTraceCtx(mLogCtx) << "Executing subquery for property: " << k; const auto result = executeSubquery(comparator.value.value()); baseFilters.insert(k, Query::Comparator(QVariant::fromValue(result), Query::Comparator::In)); } } query.setBaseFilters(baseFilters); QByteArray appliedSorting; //Determine initial set mSource = [&]() { if (!query.ids().isEmpty()) { //We have a set of ids as a starting point QVector ids; for (const auto & id: query.ids()) { ids.append(Identifier::fromDisplayByteArray(id)); } return Source::Ptr::create(ids, this); } else { QSet appliedFilters; auto resultSet = mStore.indexLookup(mType, query, appliedFilters, appliedSorting); if (!appliedFilters.isEmpty()) { //We have an index lookup as starting point return Source::Ptr::create(resultSet, this); } // We do a full scan if there were no indexes available to create the initial set (this is going to be expensive for large sets). return Source::Ptr::create(mStore.fullScan(mType), this); } }(); FilterBase::Ptr baseSet = mSource; if (!query.getBaseFilters().isEmpty()) { auto filter = Filter::Ptr::create(baseSet, this); //For incremental queries the remaining filters are not sufficient for (const auto &f : query.getBaseFilters().keys()) { filter->propertyFilter.insert(f, query.getFilter(f)); } baseSet = filter; } /* if (appliedSorting.isEmpty() && !query.sortProperty.isEmpty()) { */ /* //Apply manual sorting */ /* baseSet = Sort::Ptr::create(baseSet, query.sortProperty); */ /* } */ //Setup the rest of the filter stages on top of the base set for (const auto &stage : query.getFilterStages()) { if (auto filter = stage.dynamicCast()) { auto f = Filter::Ptr::create(baseSet, this); f->propertyFilter = filter->propertyFilter; baseSet = f; } else if (auto filter = stage.dynamicCast()) { auto reduction = Reduce::Ptr::create(filter->property, filter->selector.property, filter->selector.comparator, baseSet, this); for (const auto &aggregator : filter->aggregators) { reduction->mAggregators << Reduce::Aggregator(aggregator.operation, aggregator.propertyToCollect, aggregator.resultProperty); } + for (const auto &propertySelector : filter->propertySelectors) { + reduction->mSelectors << Reduce::PropertySelector(propertySelector.selector, propertySelector.resultProperty); + } reduction->propertyFilter = query.getBaseFilters(); baseSet = reduction; } else if (auto filter = stage.dynamicCast()) { baseSet = Bloom::Ptr::create(filter->property, baseSet, this); } } mCollector = Collector::Ptr::create(baseSet, this); } QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) { QVector changedKeys; mStore.readRevisions(baseRevision, mType, [&](const Key &key) { changedKeys << key; }); return changedKeys; } ResultSet DataStoreQuery::update(qint64 baseRevision) { SinkTraceCtx(mLogCtx) << "Executing query update from revision " << baseRevision << " to revision " << mStore.maxRevision(); auto incrementalResultSet = loadIncrementalResultSet(baseRevision); SinkTraceCtx(mLogCtx) << "Incremental changes: " << incrementalResultSet; mSource->add(incrementalResultSet); ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { if (mCollector->next([this, callback](const ResultSet::Result &result) { SinkTraceCtx(mLogCtx) << "Got incremental result: " << result.entity.identifier() << operationName(result.operation); callback(result); })) { return true; } return false; }; return ResultSet(generator, [this]() { mCollector->skip(); }); } void DataStoreQuery::updateComplete() { mSource->mIncrementalIds.clear(); auto source = mCollector; while (source) { source->updateComplete(); source = source->mSource; } } ResultSet DataStoreQuery::execute() { SinkTraceCtx(mLogCtx) << "Executing query"; Q_ASSERT(mCollector); ResultSet::ValueGenerator generator = [this](const ResultSet::Callback &callback) -> bool { if (mCollector->next([this, callback](const ResultSet::Result &result) { if (result.operation != Sink::Operation_Removal) { SinkTraceCtx(mLogCtx) << "Got initial result: " << result.entity.identifier() << result.operation; callback(ResultSet::Result{result.entity, Sink::Operation_Creation, result.aggregateValues, result.aggregateIds}); } })) { return true; } return false; }; return ResultSet(generator, [this]() { mCollector->skip(); }); } diff --git a/common/query.h b/common/query.h index ea26aa5f..1dabfe42 100644 --- a/common/query.h +++ b/common/query.h @@ -1,619 +1,647 @@ /* * Copyright (C) 2014 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . */ #pragma once #include "sink_export.h" #include #include #include #include "applicationdomaintype.h" namespace Sink { class SINK_EXPORT QueryBase { public: struct SINK_EXPORT Comparator { enum Comparators { Invalid, Equals, Contains, In, Within, Overlap, Fulltext }; Comparator(); Comparator(const QVariant &v); Comparator(const QVariant &v, Comparators c); bool matches(const QVariant &v) const; bool operator==(const Comparator &other) const; QVariant value; Comparators comparator; }; class SINK_EXPORT Filter { public: QByteArrayList ids; QHash propertyFilter; bool operator==(const Filter &other) const; }; QueryBase() = default; QueryBase(const QByteArray &type) : mType(type) {} bool operator==(const QueryBase &other) const; Comparator getFilter(const QByteArray &property) const { return mBaseFilterStage.propertyFilter.value({property}); } Comparator getFilter(const QByteArrayList &properties) const { return mBaseFilterStage.propertyFilter.value(properties); } template Comparator getFilter() const { return getFilter(T::name); } template Comparator getFilter() const { return getFilter({T1::name, T2::name, Rest::name...}); } bool hasFilter(const QByteArray &property) const { return mBaseFilterStage.propertyFilter.contains({property}); } template bool hasFilter() const { return hasFilter(T::name); } void setId(const QByteArray &id) { mId = id; } QByteArray id() const { return mId; } void setBaseFilters(const QHash &filter) { mBaseFilterStage.propertyFilter = filter; } void setFilter(const Filter &filter) { mBaseFilterStage = filter; } QHash getBaseFilters() const { return mBaseFilterStage.propertyFilter; } Filter getFilter() const { return mBaseFilterStage; } QByteArrayList ids() const { return mBaseFilterStage.ids; } void filter(const QByteArray &id) { mBaseFilterStage.ids << id; } void filter(const QByteArrayList &ids) { mBaseFilterStage.ids << ids; } void filter(const QByteArray &property, const QueryBase::Comparator &comparator) { mBaseFilterStage.propertyFilter.insert({property}, comparator); } void filter(const QByteArrayList &properties, const QueryBase::Comparator &comparator) { mBaseFilterStage.propertyFilter.insert(properties, comparator); } void setType(const QByteArray &type) { mType = type; } template void setType() { setType(ApplicationDomain::getTypeName()); } QByteArray type() const { return mType; } void setSortProperty(const QByteArray &property) { mSortProperty = property; } QByteArray sortProperty() const { return mSortProperty; } class FilterStage { public: virtual ~FilterStage(){}; }; QList> getFilterStages() { return mFilterStages; } class Reduce : public FilterStage { public: class Selector { public: enum Comparator { Min, //get the minimum value Max, //get the maximum value - First //Get the first result we get }; template static Selector max() { return Selector(SelectionProperty::name, Max); } + template + static Selector min() + { + return Selector(SelectionProperty::name, Min); + } + Selector(const QByteArray &p, Comparator c) : property(p), comparator(c) { } QByteArray property; Comparator comparator; }; + struct PropertySelector { + QByteArray resultProperty; + Selector selector; + }; + class Aggregator { public: enum Operation { Count, Collect }; Aggregator(const QByteArray &p, Operation o, const QByteArray &c = QByteArray()) : resultProperty(p), operation(o), propertyToCollect(c) { } QByteArray resultProperty; Operation operation; QByteArray propertyToCollect; }; Reduce(const QByteArray &p, const Selector &s) : property(p), selector(s) { } - Reduce &count(const QByteArray &propertyName = "count") + Reduce &count(const QByteArray &resultProperty = "count") { - aggregators << Aggregator(propertyName, Aggregator::Count); + aggregators << Aggregator(resultProperty, Aggregator::Count); return *this; } + /** + * Collect all properties and make them available as a QList as the virtual properite with the name @param resultProperty + */ template - Reduce &collect(const QByteArray &propertyName) + Reduce &collect(const QByteArray &resultProperty) { - aggregators << Aggregator(propertyName, Aggregator::Collect, T::name); + aggregators << Aggregator(resultProperty, Aggregator::Collect, T::name); return *this; } template Reduce &collect() { - aggregators << Aggregator(QByteArray{T::name} + QByteArray{"Collected"}, Aggregator::Collect, T::name); + return collect(QByteArray{T::name} + QByteArray{"Collected"}); + } + + /** + * Select a property and make it available as the virtual properite with the name @param resultProperty. + * + * This allows to make a different choice for this property than for the main selector of the reduction, + * so we can e.g. select the subject of the first email sorted by date, while otherwise selecting the latest email. + * + * Please note that this will reuse the selection property of the main selector. + */ + template + Reduce &select(Selector::Comparator comparator, const QByteArray &resultProperty) + { + propertySelectors << PropertySelector{resultProperty, Selector{T::name, comparator}}; return *this; } + template + Reduce &select(Selector::Comparator comparator) + { + return select(comparator, QByteArray{T::name} + QByteArray{"Selected"}); + } + + //Reduce on property QByteArray property; Selector selector; QList aggregators; - - //TODO add aggregate functions like: - //.count() - //.collect(); - //... - // - //Potentially pass-in an identifier under which the result will be available in the result set. + QList propertySelectors; }; Reduce &reduce(const QByteArray &name, const Reduce::Selector &s) { auto reduction = QSharedPointer::create(name, s); mFilterStages << reduction; return *reduction; } template Reduce &reduce(const Reduce::Selector &s) { return reduce(T::name, s); } /** * "Bloom" on a property. * * For every encountered value of a property, * a result set is generated containing all entries with the same value. * * Example: * For an input set of one mail; return all emails with the same threadId. */ class Bloom : public FilterStage { public: //Property to bloom on QByteArray property; Bloom(const QByteArray &p) : property(p) { } }; template void bloom() { mFilterStages << QSharedPointer::create(T::name); } private: Filter mBaseFilterStage; QList> mFilterStages; QByteArray mType; QByteArray mSortProperty; QByteArray mId; }; /** * A query that matches a set of entities. */ class SINK_EXPORT Query : public QueryBase { public: enum Flag { NoFlags = 0, /** Leave the query running and continuously update the result set. */ LiveQuery = 1, /** Run the query synchronously. */ SynchronousQuery = 2, /** Include status updates via notifications */ UpdateStatus = 4 }; Q_DECLARE_FLAGS(Flags, Flag) template Query &request() { requestedProperties << T::name; return *this; } template Query &requestTree() { mParentProperty = T::name; return *this; } Query &requestTree(const QByteArray &parentProperty) { mParentProperty = parentProperty; return *this; } QByteArray parentProperty() const { return mParentProperty; } template Query &sort() { setSortProperty(T::name); return *this; } template Query &filter(const typename T::Type &value) { filter(T::name, QVariant::fromValue(value)); return *this; } template Query &containsFilter(const QByteArray &value) { static_assert(std::is_same::value, "The contains filter is only implemented for QByteArray in QByteArrayList"); QueryBase::filter(T::name, QueryBase::Comparator(QVariant::fromValue(value), QueryBase::Comparator::Contains)); return *this; } template Query &filter(const QueryBase::Comparator &comparator) { QueryBase::filter(T::name, comparator); return *this; } template Query &filter(const QueryBase::Comparator &comparator) { QueryBase::filter({T1::name, T2::name, Rest::name...}, comparator); return *this; } Query &filter(const QByteArray &id) { QueryBase::filter(id); return *this; } Query &filter(const QByteArrayList &ids) { QueryBase::filter(ids); return *this; } Query &filter(const QByteArray &property, const QueryBase::Comparator &comparator) { QueryBase::filter(property, comparator); return *this; } template Query &filter(const ApplicationDomain::Entity &value) { filter(T::name, QVariant::fromValue(ApplicationDomain::Reference{value.identifier()})); return *this; } template Query &filter(const Query &query) { auto q = query; q.setType(ApplicationDomain::getTypeName()); filter(T::name, QVariant::fromValue(q)); return *this; } Query(const ApplicationDomain::Entity &value) : mLimit(0) { filter(value.identifier()); resourceFilter(value.resourceInstanceIdentifier()); } Query(Flags flags = Flags()) : mLimit(0), mFlags(flags) { } QByteArrayList requestedProperties; void setFlags(Flags flags) { mFlags = flags; } Flags flags() const { return mFlags; } bool liveQuery() const { return mFlags.testFlag(LiveQuery); } bool synchronousQuery() const { return mFlags.testFlag(SynchronousQuery); } Query &limit(int l) { mLimit = l; return *this; } int limit() const { return mLimit; } Filter getResourceFilter() const { return mResourceFilter; } Query &resourceFilter(const QByteArray &id) { mResourceFilter.ids << id; return *this; } template Query &resourceFilter(const ApplicationDomain::ApplicationDomainType &entity) { mResourceFilter.propertyFilter.insert({T::name}, Comparator(entity.identifier())); return *this; } Query &resourceFilter(const QByteArray &name, const Comparator &comparator) { mResourceFilter.propertyFilter.insert({name}, comparator); return *this; } template Query &resourceContainsFilter(const QVariant &value) { return resourceFilter(T::name, Comparator(value, Comparator::Contains)); } template Query &resourceFilter(const QVariant &value) { return resourceFilter(T::name, value); } private: friend class SyncScope; int mLimit; Flags mFlags; Filter mResourceFilter; QByteArray mParentProperty; }; class SyncScope : public QueryBase { public: using QueryBase::QueryBase; SyncScope() = default; SyncScope(const Query &other) : QueryBase(other), mResourceFilter(other.mResourceFilter) { } template SyncScope(const T &o) : QueryBase() { resourceFilter(o.resourceInstanceIdentifier()); filter(o.identifier()); setType(ApplicationDomain::getTypeName()); } Query::Filter getResourceFilter() const { return mResourceFilter; } SyncScope &resourceFilter(const QByteArray &id) { mResourceFilter.ids << id; return *this; } template SyncScope &resourceFilter(const ApplicationDomain::ApplicationDomainType &entity) { mResourceFilter.propertyFilter.insert({T::name}, Comparator(entity.identifier())); return *this; } SyncScope &resourceFilter(const QByteArray &name, const Comparator &comparator) { mResourceFilter.propertyFilter.insert({name}, comparator); return *this; } template SyncScope &resourceContainsFilter(const QVariant &value) { return resourceFilter(T::name, Comparator(value, Comparator::Contains)); } template SyncScope &resourceFilter(const QVariant &value) { return resourceFilter(T::name, value); } template SyncScope &filter(const Query::Comparator &comparator) { return filter(T::name, comparator); } SyncScope &filter(const QByteArray &id) { QueryBase::filter(id); return *this; } SyncScope &filter(const QByteArrayList &ids) { QueryBase::filter(ids); return *this; } SyncScope &filter(const QByteArray &property, const Query::Comparator &comparator) { QueryBase::filter(property, comparator); return *this; } private: Query::Filter mResourceFilter; }; } SINK_EXPORT QDebug operator<<(QDebug dbg, const Sink::QueryBase::Comparator &); SINK_EXPORT QDebug operator<<(QDebug dbg, const Sink::QueryBase &); SINK_EXPORT QDebug operator<<(QDebug dbg, const Sink::Query &); SINK_EXPORT QDataStream &operator<< (QDataStream &stream, const Sink::QueryBase &query); SINK_EXPORT QDataStream &operator>> (QDataStream &stream, Sink::QueryBase &query); Q_DECLARE_OPERATORS_FOR_FLAGS(Sink::Query::Flags) Q_DECLARE_METATYPE(Sink::QueryBase); Q_DECLARE_METATYPE(Sink::Query); Q_DECLARE_METATYPE(Sink::SyncScope); diff --git a/common/standardqueries.h b/common/standardqueries.h index 10f66aed..f2adfdef 100644 --- a/common/standardqueries.h +++ b/common/standardqueries.h @@ -1,75 +1,76 @@ /* * Copyright (C) 2016 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . */ #pragma once #include "query.h" namespace Sink { namespace StandardQueries { /** * Returns the complete thread, containing all mails from all folders. */ static Query completeThread(const ApplicationDomain::Mail &mail) { Sink::Query query; query.setId("completethread"); if (!mail.resourceInstanceIdentifier().isEmpty()) { query.resourceFilter(mail.resourceInstanceIdentifier()); } query.filter(mail.identifier()); query.sort(); query.bloom(); return query; } /** * Returns thread leaders only, sorted by date. */ static Query threadLeaders(const ApplicationDomain::Folder &folder) { Sink::Query query; query.setId("threadleaders"); if (!folder.resourceInstanceIdentifier().isEmpty()) { query.resourceFilter(folder.resourceInstanceIdentifier()); } query.filter(folder); query.sort(); query.reduce(Query::Reduce::Selector::max()) .count() + .select(Query::Reduce::Selector::Min) .collect() .collect(); return query; } /** * Outgoing mails. */ static Query outboxMails() { Sink::Query query; query.setId("outbox"); query.resourceContainsFilter(ApplicationDomain::ResourceCapabilities::Mail::transport); query.sort(); return query; } } } diff --git a/tests/querytest.cpp b/tests/querytest.cpp index 0aaeb9d2..fe5b4a87 100644 --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1,1960 +1,1966 @@ #include #include #include #include "resource.h" #include "store.h" #include "resourcecontrol.h" #include "commands.h" #include "resourceconfig.h" #include "log.h" #include "modelresult.h" #include "test.h" #include "testutils.h" #include "applicationdomaintype.h" #include "queryrunner.h" #include "adaptorfactoryregistry.h" #include "fulltextindex.h" #include #include #include using namespace Sink; using namespace Sink::ApplicationDomain; /** * Test of the query system using the dummy resource. * * This test requires the dummy resource installed. */ class QueryTest : public QObject { Q_OBJECT private slots: void initTestCase() { Sink::Test::initTest(); auto factory = Sink::ResourceFactory::load("sink.dummy"); QVERIFY(factory); ResourceConfig::addResource("sink.dummy.instance1", "sink.dummy"); ResourceConfig::configureResource("sink.dummy.instance1", {{"populate", true}}); VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void cleanup() { VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void init() { qDebug(); qDebug() << "-----------------------------------------"; qDebug(); } void testSerialization() { auto type = QByteArray("type"); auto sort = QByteArray("sort"); Sink::QueryBase::Filter filter; filter.ids << "id"; filter.propertyFilter.insert({"foo"}, QVariant::fromValue(QByteArray("bar"))); Sink::Query query; query.setFilter(filter); query.setType(type); query.setSortProperty(sort); QByteArray data; { QDataStream stream(&data, QIODevice::WriteOnly); stream << query; } Sink::Query deserializedQuery; { QDataStream stream(&data, QIODevice::ReadOnly); stream >> deserializedQuery; } QCOMPARE(deserializedQuery.type(), type); QCOMPARE(deserializedQuery.sortProperty(), sort); QCOMPARE(deserializedQuery.getFilter().ids, filter.ids); QCOMPARE(deserializedQuery.getFilter().propertyFilter.keys(), filter.propertyFilter.keys()); QCOMPARE(deserializedQuery.getFilter().propertyFilter, filter.propertyFilter); } void testNoResources() { // Test Sink::Query query; query.resourceFilter("foobar"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } void testSingle() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); } void testSingleWithDelay() { // Setup auto mail = Mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testFilter() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); VERIFYEXEC(Sink::Store::create(mail)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter("folder1"); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value(); { mail->setFolder("folder2"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 0); { mail->setFolder("folder1"); VERIFYEXEC(Sink::Store::modify(*mail)); } QTRY_COMPARE(model->rowCount(), 1); } void testById() { QByteArray id; // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); VERIFYEXEC(Sink::Store::create(mail)); mail.setExtractedMessageId("test2"); VERIFYEXEC(Sink::Store::create(mail)); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed Sink::Store::synchronize(query).exec().waitForFinished(); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QVERIFY(model->rowCount() >= 1); id = model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->identifier(); } // Test { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(id); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); //Try a non-existing id query.filter("{87fcea5e-8d2e-408e-bb8d-b27b9dcf5e92}"); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } } void testFolder() { // Setup { Folder folder("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); auto folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); } void testFolderTree() { // Setup { auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch after the data is available and don't rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QCOMPARE(model->rowCount(model->index(0, 0)), 1); } void testIncrementalFolderTree() { // Setup auto folder = ApplicationDomainType::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); // Test Sink::Query query{Sink::Query::LiveQuery}; query.resourceFilter("sink.dummy.instance1"); query.requestTree(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); auto subfolder = ApplicationDomainType::createEntity("sink.dummy.instance1"); subfolder.setParent(folder.identifier()); VERIFYEXEC(Sink::Store::create(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Ensure the folder appears QTRY_COMPARE(model->rowCount(model->index(0, 0)), 1); //...and dissapears again after removal VERIFYEXEC(Sink::Store::remove(subfolder)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(model->index(0, 0)), 0); } void testMailByMessageId() { // Setup { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setProperty("sender", "doe@example.org"); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } /* * Filter by two properties to make sure that we also use a non-index based filter. */ void testMailByMessageIdAndFolder() { // Setup Folder::Ptr folderEntity; { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder(folderEntity->identifier()); Sink::Store::create(mail).exec().waitForFinished(); Mail mail1("sink.dummy.instance1"); mail1.setExtractedMessageId("test1"); mail1.setFolder("foobar"); Sink::Store::create(mail1).exec().waitForFinished(); Mail mail2("sink.dummy.instance1"); mail2.setExtractedMessageId("test2"); mail2.setFolder(folderEntity->identifier()); Sink::Store::create(mail2).exec().waitForFinished(); } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.filter("test1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } void testMailByFolderSortedByDate() { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testSecond"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-1)); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLatest"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date); Sink::Store::create(mail).exec().waitForFinished(); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testLast"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.limit(1); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); // The model is not sorted, but the limited set is sorted, so we can only test for the latest result. QCOMPARE(model->rowCount(), 1); QCOMPARE(model->index(0, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testLatest")); model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); // We can't make any assumptions about the order of the indexes // QCOMPARE(model->index(1, 0).data(Sink::Store::DomainObjectRole).value()->getProperty("messageId").toByteArray(), QByteArray("testSecond")); //New revisions always go through { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("testInjected"); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(-2)); Sink::Store::create(mail).exec().waitForFinished(); } QTRY_COMPARE(model->rowCount(), 3); //Ensure we can continue fetching after the incremental update model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); //Ensure we have fetched all model->fetchMore(QModelIndex()); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); } void testReactToNewResource() { Sink::Query query; query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(QModelIndex()), 0); auto res = DummyResource::create(""); VERIFYEXEC(Sink::Store::create(res)); auto folder = Folder::create(res.identifier()); VERIFYEXEC(Sink::Store::create(folder)); QTRY_COMPARE(model->rowCount(QModelIndex()), 1); VERIFYEXEC(Sink::Store::remove(res)); } void testAccountFilter() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup QString accountName("name"); QString accountIcon("icon"); auto account1 = ApplicationDomainType::createEntity(); account1.setAccountType("maildir"); account1.setName(accountName); account1.setIcon(accountIcon); VERIFYEXEC(Store::create(account1)); auto account2 = ApplicationDomainType::createEntity(); account2.setAccountType("maildir"); account2.setName(accountName); account2.setIcon(accountIcon); VERIFYEXEC(Store::create(account2)); auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setAccount(account1); Store::create(resource1).exec().waitForFinished(); auto resource2 = ApplicationDomainType::createEntity(); resource2.setResourceType("sink.dummy"); resource2.setAccount(account2); Store::create(resource2).exec().waitForFinished(); { Folder folder1(resource1.identifier()); VERIFYEXEC(Sink::Store::create(folder1)); Folder folder2(resource2.identifier()); VERIFYEXEC(Sink::Store::create(folder2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier())); // Test Sink::Query query; query.resourceFilter(account1); auto folders = Sink::Store::read(query); QCOMPARE(folders.size(), 1); } void testSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); auto mails = Sink::Store::read(query); QCOMPARE(mails.size(), 1); QCOMPARE(mails.first().getMessageId(), QByteArray("mail1")); } void testLiveSubquery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setSpecialPurpose(QByteArrayList() << "purpose1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); folder2.setSpecialPurpose(QByteArrayList() << "purpose2"); VERIFYEXEC(Sink::Store::create(folder2)); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail1"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.filter(Sink::Query().containsFilter("purpose1")); query.request(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //This folder should not make it through the query { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } //But this one should { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } QTRY_COMPARE(model->rowCount(), 2); } void testResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Sink::Store::create(Folder{resource1.identifier()})); VERIFYEXEC(Sink::Store::create(Folder{resource2.identifier()})); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource1.identifier())); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource2.identifier())); // We fetch before the data is available and rely on the live query mechanism to deliver the actual data auto folders = Sink::Store::read(Sink::Query{}.resourceContainsFilter("cap1")); QCOMPARE(folders.size(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testFilteredLiveResourceSubQuery() { using namespace Sink; using namespace Sink::ApplicationDomain; //Setup auto resource1 = ApplicationDomainType::createEntity(); resource1.setResourceType("sink.dummy"); resource1.setCapabilities(QByteArrayList() << "cap1"); VERIFYEXEC(Store::create(resource1)); VERIFYEXEC(Store::create(Folder{resource1.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource1.identifier())); auto model = Sink::Store::loadModel(Query{Query::LiveQuery}.resourceContainsFilter("cap1")); QTRY_COMPARE(model->rowCount(), 1); auto resource2 = ApplicationDomainType::createEntity(); resource2.setCapabilities(QByteArrayList() << "cap2"); resource2.setResourceType("sink.dummy"); VERIFYEXEC(Store::create(resource2)); VERIFYEXEC(Store::create(Folder{resource2.identifier()})); VERIFYEXEC(ResourceControl::flushMessageQueue(resource2.identifier())); //The new resource should be filtered and thus not make it in here QCOMPARE(model->rowCount(), 1); //TODO this should be part of the regular cleanup between tests VERIFYEXEC(Store::remove(resource1)); VERIFYEXEC(Store::remove(resource2)); } void testLivequeryUnmatchInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setFolder(folder2); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testLivequeryRemoveOneInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); //Setup two folders with a mail each, ensure we only get the mail from the folder that matches the folder filter. Query query; query.setId("testLivequeryUnmatch"); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.setFlags(Query::LiveQuery); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); QCOMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 2); //After the removal, the thread size should be reduced by one { VERIFYEXEC(Sink::Store::remove(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); QTRY_COMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 1); //After the second removal, the thread should be gone { VERIFYEXEC(Sink::Store::remove(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 0); } void testDontUpdateNonLiveQuery() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; //Not a live query query.setFlags(Query::Flags{}); query.setId("testNoLiveQuery"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("senders"); query.sort(); query.request(); QVERIFY(!query.liveQuery()); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTest::qWait(100); QCOMPARE(mail->getUnread(), false); } void testLivequeryModifcationUpdateInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.setFlags(Query::LiveQuery); query.request(); auto model = Sink::Store::loadModel(query); QTRY_COMPARE(model->rowCount(), 1); //After the modifcation the mail should have vanished. { mail1.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail1)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getUnread(), true); QCOMPARE(mail->getProperty("count").toInt(), 1); QCOMPARE(mail->getProperty("folders").toList().size(), 1); } void testReductionUpdate() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setUnread(false); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryUnmatch"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader should change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setUnread(false); mail2.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail2"}); QCOMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 0); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testFilteredReductionUpdate() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilteredReductionUpdate"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //Ensure we don't end up with a mail in the thread that was filtered //This tests the case of an otherwise emtpy thread on purpose. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("filtered"); mail.setFolder(folder2); mail.setExtractedDate(QDateTime{QDate{2017, 2, 3}, QTime{11, 0, 0}}); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QCOMPARE(model->rowCount(), 0); //Ensure the non-filtered still get through. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("not-filtered"); mail.setFolder(folder1); mail.setExtractedDate(QDateTime{QDate{2017, 2, 3}, QTime{11, 0, 0}}); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); } /* * Two messages in the same thread. The first get's filtered, the second one makes it. */ void testFilteredReductionUpdateInSameThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilteredReductionUpdate"); query.setFlags(Query::LiveQuery); query.filter(folder1); query.reduce(Query::Reduce::Selector::max()).count("count"); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); //The first message will be filtered (but would be aggreagted together with the message that passes) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); //Ensure that we can deal with a modification to the filtered message mail.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QCOMPARE(model->rowCount(), 0); //Ensure the non-filtered still gets through. { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); //Ensure that we can deal with a modification to the filtered message mail.setUnread(true); VERIFYEXEC(Sink::Store::modify(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); QCOMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 1); //Ensure another entity still results in a modification { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("aggregatedId"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value()->getProperty("count").toInt(), 2); } void testBloom() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.resourceFilter("sink.dummy.instance1"); query.setId("testFilterCreationInThread"); query.filter(mail1.identifier()); query.bloom(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); } void testLivequeryFilterCreationInThread() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testFilterCreationInThread"); query.resourceFilter("sink.dummy.instance1"); query.filter(mail1.identifier()); query.bloom(); query.sort(); query.setFlags(Query::LiveQuery); query.request(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //This modification should make it through { //This should not trigger an entity already in model warning mail1.setUnread(false); VERIFYEXEC(Sink::Store::modify(mail1)); } //This mail should make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail2"); mail.setFolder(folder1); VERIFYEXEC(Sink::Store::create(mail)); } //This mail shouldn't make it through { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail3"); mail.setFolder(folder2); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); QTest::qWait(100); QCOMPARE(model->rowCount(), 2); //From mail2 QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 0); //From the modification QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } void testLivequeryThreadleaderChange() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto mail1 = Mail::createEntity("sink.dummy.instance1"); mail1.setExtractedMessageId("mail1"); mail1.setFolder(folder1); mail1.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail1)); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()).count("count").collect("folders"); query.sort(); query.request(); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); QSignalSpy insertedSpy(model.data(), &QAbstractItemModel::rowsInserted); QSignalSpy removedSpy(model.data(), &QAbstractItemModel::rowsRemoved); QSignalSpy changedSpy(model.data(), &QAbstractItemModel::dataChanged); QSignalSpy layoutChangedSpy(model.data(), &QAbstractItemModel::layoutChanged); QSignalSpy resetSpy(model.data(), &QAbstractItemModel::modelReset); //The leader shouldn't change to mail2 after the modification { auto mail2 = Mail::createEntity("sink.dummy.instance1"); mail2.setExtractedMessageId("mail2"); mail2.setFolder(folder1); mail2.setExtractedDate(earlier); VERIFYEXEC(Sink::Store::create(mail2)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail1"}); QTRY_COMPARE(mail->getProperty("count").toInt(), 2); QCOMPARE(mail->getProperty("folders").toList().size(), 2); } QCOMPARE(insertedSpy.size(), 0); QCOMPARE(removedSpy.size(), 0); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //The leader should change to mail3 after the modification { auto mail3 = Mail::createEntity("sink.dummy.instance1"); mail3.setExtractedMessageId("mail3"); mail3.setFolder(folder1); mail3.setExtractedDate(later); VERIFYEXEC(Sink::Store::create(mail3)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QTRY_COMPARE(mail->getMessageId(), QByteArray{"mail3"}); QCOMPARE(mail->getProperty("count").toInt(), 3); QCOMPARE(mail->getProperty("folders").toList().size(), 3); } //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 1); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); //Nothing should change on third mail in separate folder { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("mail4"); mail.setFolder(folder2); mail.setExtractedDate(now); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); QTRY_COMPARE(model->rowCount(), 2); //This should eventually be just one modification instead of remove + add (See datastorequery reduce component) QCOMPARE(insertedSpy.size(), 2); QCOMPARE(removedSpy.size(), 1); QCOMPARE(changedSpy.size(), 1); QCOMPARE(layoutChangedSpy.size(), 0); QCOMPARE(resetSpy.size(), 0); } /* * Ensure that we handle the situation properly if the thread-leader doesn't match a property filter. */ void testFilteredThreadLeader() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); auto folder2 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder2)); QDateTime earlier{QDate{2017, 2, 3}, QTime{9, 0, 0}}; QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}}; auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { auto mail = Mail::createEntity("sink.dummy.instance1"); + mail.setExtractedSubject(messageid); mail.setExtractedMessageId(messageid); mail.setFolder(folder); mail.setExtractedDate(date); mail.setImportant(important); return mail; }; VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, earlier, false))); VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, later, true))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setId("testLivequeryThreadleaderChange"); query.setFlags(Query::LiveQuery); - query.reduce(Query::Reduce::Selector::max()).count().collect(); + query.reduce(Query::Reduce::Selector::max()) + .count() + .collect() + .select(Query::Reduce::Selector::Min, "subjectSelected"); query.sort(); query.request(); + query.request(); query.filter(false); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); { auto mail = model->data(model->index(0, 0, QModelIndex{}), Sink::Store::DomainObjectRole).value(); QCOMPARE(mail->getMessageId(), QByteArray{"mail1"}); QCOMPARE(mail->count(), 2); QCOMPARE(mail->getCollectedProperty().size(), 2); + QCOMPARE(mail->getProperty("subjectSelected").toString(), QString{"mail2"}); } } void testQueryRunnerDontMissUpdates() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); VERIFYEXEC(Sink::Store::create(folder1)); QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}}; auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) { auto mail = Mail::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId(messageid); mail.setFolder(folder); mail.setExtractedDate(date); mail.setImportant(important); return mail; }; VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false))); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setFlags(Query::LiveQuery); Sink::ResourceContext resourceContext{"sink.dummy.instance1", "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}; Sink::Log::Context logCtx; auto runner = new QueryRunner(query, resourceContext, ApplicationDomain::getTypeName(), logCtx); runner->delayNextQuery(); auto emitter = runner->emitter(); QList added; emitter->onAdded([&](Mail::Ptr mail) { added << mail; }); emitter->fetch(); VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, now, false))); QTRY_COMPARE(added.size(), 2); runner->delayNextQuery(); VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, now, false))); //The second revision update is supposed to come in while the initial revision update is still in the query. //So wait a bit to make sure the query is currently runnning. QTest::qWait(500); VERIFYEXEC(Sink::Store::create(createMail("mail4", folder1, now, false))); QTRY_COMPARE(added.size(), 4); } /* * This test excercises the scenario where a fetchMore is triggered after * the revision is already updated in storage, but the incremental query was not run yet. * This resulted in lost modification updates. */ void testQueryRunnerDontMissUpdatesWithFetchMore() { // Setup auto folder1 = Folder::createEntity("sink.dummy.instance1"); folder1.setName("name1"); VERIFYEXEC(Sink::Store::create(folder1)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Query query; query.setFlags(Query::LiveQuery); Sink::ResourceContext resourceContext{"sink.dummy.instance1", "sink.dummy", Sink::AdaptorFactoryRegistry::instance().getFactories("sink.dummy")}; Sink::Log::Context logCtx; auto runner = new QueryRunner(query, resourceContext, ApplicationDomain::getTypeName(), logCtx); auto emitter = runner->emitter(); QList added; emitter->onAdded([&](Folder::Ptr folder) { added << folder; }); QList modified; emitter->onModified([&](Folder::Ptr folder) { modified << folder; }); emitter->fetch(); QTRY_COMPARE(added.size(), 1); QCOMPARE(modified.size(), 0); runner->ignoreRevisionChanges(true); folder1.setName("name2"); VERIFYEXEC(Sink::Store::modify(folder1)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); emitter->fetch(); runner->ignoreRevisionChanges(false); runner->triggerRevisionChange(); QTRY_COMPARE(added.size(), 1); QTRY_COMPARE(modified.size(), 1); } /* * This test is here to ensure we don't crash if we call removeFromDisk with a running query. */ void testRemoveFromDiskWithRunningQuery() { // FIXME: we currently crash QSKIP("Skipping because this produces a crash."); { // Setup Folder::Ptr folderEntity; const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0)); { Folder folder("sink.dummy.instance1"); Sink::Store::create(folder).exec().waitForFinished(); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value(); QVERIFY(!folderEntity->identifier().isEmpty()); //Add enough data so the query takes long enough that we remove the data from disk whlie the query is ongoing. for (int i = 0; i < 100; i++) { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("test" + QByteArray::number(i)); mail.setFolder(folderEntity->identifier()); mail.setExtractedDate(date.addDays(i)); Sink::Store::create(mail).exec().waitForFinished(); } } // Test Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(*folderEntity); query.sort(); query.setFlags(Query::LiveQuery); query.reduce(Query::Reduce::Selector::max()) .count("count") .collect("unreadCollected") .collect("importantCollected"); // Ensure all local data is processed VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1")); auto model = Sink::Store::loadModel(query); } //FIXME: this will result in a crash in the above still running query. VERIFYEXEC(Sink::Store::removeDataFromDisk(QByteArray("sink.dummy.instance1"))); } void testMailFulltext() { QByteArray id1; QByteArray id2; // Setup { { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Subject To Search"); msg->setBody("This is the searchable body bar. unique sender2"); msg->from()->from7BitString("\"The Sender\""); msg->to()->from7BitString("\"Foo Bar\""); msg->assemble(); auto mail = ApplicationDomainType::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("test1"); mail.setFolder("folder1"); mail.setMimeMessage(msg->encodedContent()); VERIFYEXEC(Sink::Store::create(mail)); id1 = mail.identifier(); } { auto msg = KMime::Message::Ptr::create(); msg->subject()->from7BitString("Stuff to Search"); msg->setBody("Body foo bar"); msg->from()->from7BitString("\"Another Sender2\""); msg->assemble(); auto mail = ApplicationDomainType::createEntity("sink.dummy.instance1"); mail.setExtractedMessageId("test2"); mail.setFolder("folder2"); mail.setMimeMessage(msg->encodedContent()); VERIFYEXEC(Sink::Store::create(mail)); id2 = mail.identifier(); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { FulltextIndex index("sink.dummy.instance1", Sink::Storage::DataStore::ReadOnly); qInfo() << QString("Found document 1 with terms: ") + index.getIndexContent(id1).terms.join(", "); qInfo() << QString("Found document 2 with terms: ") + index.getIndexContent(id2).terms.join(", "); } } // Test // Default search { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject To Search"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } // Phrase search { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("\"Subject To Search\""), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("\"Stuff to Search\""), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); } //Operators { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject AND search"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject OR search"), QueryBase::Comparator::Fulltext)); QCOMPARE(Sink::Store::read(query).size(), 2); } //Case-insensitive { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Case-insensitive { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subject"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Partial match { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("subj"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by body { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("searchable"), QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by folder { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder1"); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by folder { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QString("Subject"), QueryBase::Comparator::Fulltext)); query.filter("folder2"); QCOMPARE(Sink::Store::read(query).size(), 0); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 2); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("Sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 2); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender@example"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("The Sender"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); } //Filter by sender { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("sender2@unique.com"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id2); } //Filter by recipient { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("foo-bar@example.org"), Sink::QueryBase::Comparator::Fulltext)); const auto list = Sink::Store::read(query); QCOMPARE(list.size(), 1); QCOMPARE(list.first().identifier(), id1); } //Filter by recipient { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter({}, Sink::QueryBase::Comparator(QString("foo-bar@example.com"), Sink::QueryBase::Comparator::Fulltext)); QCOMPARE(Sink::Store::read(query).size(), 0); } } void mailsWithDates() { { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); mail.setExtractedMessageId("message1"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:50:00Z", Qt::ISODate)); mail.setExtractedMessageId("message2"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); mail.setExtractedMessageId("message3"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedMessageId("message4"); VERIFYEXEC(Sink::Store::create(mail)); } { Mail mail("sink.dummy.instance1"); mail.setExtractedDate(QDateTime::fromString("2078-05-23T13:49:41Z", Qt::ISODate)); mail.setExtractedMessageId("message5"); VERIFYEXEC(Sink::Store::create(mail)); } VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); } void testMailDate() { mailsWithDates(); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-27T13:49:41Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 0); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } } void testMailRange() { mailsWithDates(); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-25T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 2); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 3); } { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2118-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); } } void testOverlap() { auto createEvent = [] (const QString &start, const QString &end) { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString(start, Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString(end, Qt::ISODate)); Event event("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC(Sink::Store::create(event)); }; createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); createEvent("2018-05-23T13:00:00Z", "2018-05-23T14:00:00Z"); createEvent("2018-05-23T14:00:00Z", "2018-05-23T15:00:00Z"); createEvent("2018-05-24T12:00:00Z", "2018-05-24T14:00:00Z"); //Long event that spans multiple buckets createEvent("2018-05-30T22:00:00", "2019-04-25T03:00:00"); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); auto findInRange = [] (const QString &start, const QString &end) { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString(start, Qt::ISODate), QDateTime::fromString(end, Qt::ISODate) }, QueryBase::Comparator::Overlap)); return Sink::Store::read(query); }; //Find all QCOMPARE(findInRange("2018-05-22T12:00:00Z", "2018-05-30T13:00:00Z").size(), 4); //Find none on day without events QCOMPARE(findInRange("2018-05-22T12:00:00Z", "2018-05-22T13:00:00Z").size(), 0); //Find none on day with events QCOMPARE(findInRange("2018-05-24T10:00:00Z", "2018-05-24T11:00:00Z").size(), 0); //Find on same day QCOMPARE(findInRange("2018-05-23T12:30:00Z", "2018-05-23T12:31:00Z").size(), 1); //Find on different days QCOMPARE(findInRange("2018-05-22T12:30:00Z", "2018-05-23T12:00:00Z").size(), 1); QCOMPARE(findInRange("2018-05-23T14:30:00Z", "2018-05-23T16:00:00Z").size(), 1); //Find long range event QCOMPARE(findInRange("2018-07-23T14:30:00Z", "2018-10-23T16:00:00Z").size(), 1); } void testOverlapLive() { auto createEvent = [] (const QString &start, const QString &end) { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString(start, Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString(end, Qt::ISODate)); Event event = Event::createEntity("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC_RET(Sink::Store::create(event), {}); return event; }; createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); createEvent("2018-05-23T13:00:00Z", "2018-05-23T14:00:00Z"); createEvent("2018-05-23T14:00:00Z", "2018-05-23T15:00:00Z"); createEvent("2018-05-24T12:00:00Z", "2018-05-24T14:00:00Z"); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); { Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString("2018-05-22T12:00:00Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:00:00Z", Qt::ISODate) }, QueryBase::Comparator::Overlap)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 4); auto event1 = createEvent("2018-05-23T12:00:00Z", "2018-05-23T13:00:00Z"); auto event2 = createEvent("2018-05-31T12:00:00Z", "2018-05-31T13:00:00Z"); QTRY_COMPARE(model->rowCount(), 5); VERIFYEXEC(Sink::Store::remove(event1)); VERIFYEXEC(Sink::Store::remove(event2)); QTRY_COMPARE(model->rowCount(), 4); } } void testRecurringEvents() { auto icalEvent = KCalCore::Event::Ptr::create(); icalEvent->setSummary("test"); icalEvent->setDtStart(QDateTime::fromString("2018-05-10T13:00:00Z", Qt::ISODate)); icalEvent->setDtEnd(QDateTime::fromString("2018-05-10T14:00:00Z", Qt::ISODate)); icalEvent->recurrence()->setWeekly(3); Event event = Event::createEntity("sink.dummy.instance1"); event.setIcal(KCalCore::ICalFormat().toICalString(icalEvent).toUtf8()); VERIFYEXEC(Sink::Store::create(event)); VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); Sink::Query query; query.resourceFilter("sink.dummy.instance1"); query.setFlags(Query::LiveQuery); query.filter(QueryBase::Comparator( QVariantList{ QDateTime::fromString("2018-05-15T12:00:00Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:00:00Z", Qt::ISODate) }, QueryBase::Comparator::Overlap)); auto model = Sink::Store::loadModel(query); QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); QCOMPARE(model->rowCount(), 1); VERIFYEXEC(Sink::Store::remove(event)); QTRY_COMPARE(model->rowCount(), 0); } }; QTEST_MAIN(QueryTest) #include "querytest.moc"