diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -38,7 +38,7 @@ MAPPER.addMapping(&Sink::ApplicationDomain::Buffer::ENTITYTYPE::LOWERCASEPROPERTY, &Sink::ApplicationDomain::Buffer::ENTITYTYPE##Builder::add_##LOWERCASEPROPERTY); typedef IndexConfig, + SortedIndex, ValueIndex, ValueIndex, ValueIndex, @@ -64,7 +64,8 @@ > AddressbookIndexConfig; typedef IndexConfig + ValueIndex, + SortedIndex > EventIndexConfig; typedef IndexConfig +template class SortedIndex { public: @@ -78,6 +78,22 @@ } }; +template +class SortedIndex +{ +public: + static void configure(TypeIndex &index) + { + index.addSortedProperty(); + } + + template + static QMap databases() + { + return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", 1}}; + } +}; + template class SecondaryIndex { diff --git a/common/index.h b/common/index.h --- a/common/index.h +++ b/common/index.h @@ -40,6 +40,10 @@ bool matchSubStringKeys = false); QByteArray lookup(const QByteArray &key); + void rangeLookup(const QByteArray &lowerBound, const QByteArray &upperBound, + const std::function &resultHandler, + const std::function &errorHandler); + private: Q_DISABLE_COPY(Index); Sink::Storage::DataStore::Transaction mTransaction; diff --git a/common/index.cpp b/common/index.cpp --- a/common/index.cpp +++ b/common/index.cpp @@ -59,3 +59,17 @@ lookup(key, [&](const QByteArray &value) { result = QByteArray(value.constData(), value.size()); }, [](const Index::Error &) { }); return result; } + +void Index::rangeLookup(const QByteArray &lowerBound, const QByteArray &upperBound, + const std::function &resultHandler, + const std::function &errorHandler) +{ + mDb.findAllInRange(lowerBound, upperBound, + [&](const QByteArray &key, const QByteArray &value) { + resultHandler(value); + }, + [&](const Sink::Storage::DataStore::Error &error) { + SinkWarningCtx(mLogCtx) << "Error while retrieving value:" << error << mName; + errorHandler(Error(error.store, error.code, error.message)); + }); +} diff --git a/common/query.h b/common/query.h --- a/common/query.h +++ b/common/query.h @@ -36,6 +36,7 @@ Equals, Contains, In, + Within, Fulltext }; diff --git a/common/query.cpp b/common/query.cpp --- a/common/query.cpp +++ b/common/query.cpp @@ -132,8 +132,8 @@ bool QueryBase::operator==(const QueryBase &other) const { - auto ret = mType == other.mType - && mSortProperty == other.mSortProperty + auto ret = mType == other.mType + && mSortProperty == other.mSortProperty && mBaseFilterStage == other.mBaseFilterStage; return ret; } @@ -171,6 +171,14 @@ return false; } return value.value().contains(v.toByteArray()); + case Within: { + auto range = value.value>(); + if (range.size() < 2) { + return false; + } + + return range[0] <= v && v <= range[1]; + } case Fulltext: case Invalid: default: diff --git a/common/typeindex.h b/common/typeindex.h --- a/common/typeindex.h +++ b/common/typeindex.h @@ -38,6 +38,8 @@ template void addProperty(const QByteArray &property); + template + void addSortedProperty(const QByteArray &property); template void addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty); @@ -54,9 +56,9 @@ } template - void addPropertyWithSorting() + void addSortedProperty() { - addPropertyWithSorting(T::name); + addSortedProperty(T::name); } template @@ -112,14 +114,17 @@ friend class Sink::Storage::EntityStore; void updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; + QByteArray sortedIndexName(const QByteArray &property) const; Sink::Log::Context mLogCtx; QByteArray mType; QByteArrayList mProperties; - QMap mSortedProperties; + QByteArrayList mSortedProperties; + QMap mGroupedSortedProperties; // QMap mSecondaryProperties; QList mCustomIndexer; Sink::Storage::DataStore::Transaction *mTransaction; QHash> mIndexer; - QHash> mSortIndexer; + QHash> mSortIndexer; + QHash> mGroupedSortIndexer; }; diff --git a/common/typeindex.cpp b/common/typeindex.cpp --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -21,9 +21,12 @@ #include "log.h" #include "index.h" #include "fulltextindex.h" + #include #include +#include + using namespace Sink; static QByteArray getByteArray(const QVariant &value) @@ -50,15 +53,34 @@ return "toplevel"; } -static QByteArray toSortableByteArray(const QDateTime &date) + +static QByteArray toSortableByteArrayImpl(const QDateTime &date) { // Sort invalid last if (!date.isValid()) { return QByteArray::number(std::numeric_limits::max()); } - return QByteArray::number(std::numeric_limits::max() - date.toTime_t()); + static unsigned int uint_num_digits = std::log10(std::numeric_limits::max()) + 1; + return QByteArray::number(std::numeric_limits::max() - date.toTime_t()).rightJustified(uint_num_digits, '0'); } +static QByteArray toSortableByteArray(const QVariant &value) +{ + if (!value.isValid()) { + // FIXME: we don't know the type, so we don't know what to return + // This mean we're fixing every sorted index keys to use unsigned int + return QByteArray::number(std::numeric_limits::max()); + } + + switch (value.type()) { + case QMetaType::QDateTime: + return toSortableByteArrayImpl(value.toDateTime()); + default: + SinkWarning() << "Not knowing how to convert a" << value.typeName() + << "to a sortable key, falling back to default conversion"; + return getByteArray(value); + } +} TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) { @@ -72,6 +94,11 @@ return mType + ".index." + property + ".sort." + sortProperty; } +QByteArray TypeIndex::sortedIndexName(const QByteArray &property) const +{ + return mType + ".index." + property + ".sorted"; +} + template <> void TypeIndex::addProperty(const QByteArray &property) { @@ -137,6 +164,22 @@ addProperty(property); } +template <> +void TypeIndex::addSortedProperty(const QByteArray &property) +{ + auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, + Sink::Storage::DataStore::Transaction &transaction) { + const auto sortableDate = toSortableByteArray(value); + if (add) { + Index(sortedIndexName(property), transaction).add(sortableDate, identifier); + } else { + Index(sortedIndexName(property), transaction).remove(sortableDate, identifier); + } + }; + mSortIndexer.insert(property, indexer); + mSortedProperties << property; +} + template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { @@ -149,8 +192,8 @@ Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); } }; - mSortIndexer.insert(property + sortProperty, indexer); - mSortedProperties.insert(property, sortProperty); + mGroupedSortIndexer.insert(property + sortProperty, indexer); + mGroupedSortedProperties.insert(property, sortProperty); } template <> @@ -166,10 +209,15 @@ auto indexer = mIndexer.value(property); indexer(add, identifier, value, transaction); } - for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { + for (const auto &property : mSortedProperties) { + const auto value = entity.getProperty(property); + auto indexer = mSortIndexer.value(property); + indexer(add, identifier, value, transaction); + } + for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { const auto value = entity.getProperty(it.key()); const auto sortValue = entity.getProperty(it.value()); - auto indexer = mSortIndexer.value(it.key() + it.value()); + auto indexer = mGroupedSortIndexer.value(it.key() + it.value()); indexer(add, identifier, value, sortValue, transaction); } for (const auto &indexer : mCustomIndexer) { @@ -207,22 +255,60 @@ updateIndex(false, identifier, entity, transaction, resourceInstanceId); } -static QVector indexLookup(Index &index, QueryBase::Comparator filter) +static QVector indexLookup(Index &index, QueryBase::Comparator filter, + std::function valueToKey = getByteArray) { QVector keys; QByteArrayList lookupKeys; if (filter.comparator == Query::Comparator::Equals) { - lookupKeys << getByteArray(filter.value); + lookupKeys << valueToKey(filter.value); } else if (filter.comparator == Query::Comparator::In) { - lookupKeys = filter.value.value(); + for(const QVariant &value : filter.value.value()) { + lookupKeys << valueToKey(value); + } } else { Q_ASSERT(false); } for (const auto &lookupKey : lookupKeys) { index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, - [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, true); + [lookupKey](const Index::Error &error) { + SinkWarning() << "Lookup error in index: " << error.message << lookupKey; + }, + true); + } + return keys; +} + +static QVector sortedIndexLookup(Index &index, QueryBase::Comparator filter) +{ + if (filter.comparator == Query::Comparator::In || filter.comparator == Query::Comparator::Contains) { + SinkWarning() << "In and Contains comparison not supported on sorted indexes"; + } + + if (filter.comparator != Query::Comparator::Within) { + return indexLookup(index, filter, toSortableByteArray); + } + + QVector keys; + + QByteArray lowerBound, upperBound; + auto bounds = filter.value.value(); + if (bounds[0].canConvert()) { + // Inverse the bounds because dates are stored newest first + upperBound = toSortableByteArray(bounds[0].toDateTime()); + lowerBound = toSortableByteArray(bounds[1].toDateTime()); + } else { + lowerBound = bounds[0].toByteArray(); + upperBound = bounds[1].toByteArray(); } + + index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { keys << value; }, + [bounds](const Index::Error &error) { + SinkWarning() << "Lookup error in index:" << error.message + << "with bounds:" << bounds[0] << bounds[1]; + }); + return keys; } @@ -239,16 +325,27 @@ } } - for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { + for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { Index index(indexName(it.key(), it.value()), transaction); const auto keys = indexLookup(index, query.getFilter(it.key())); appliedFilters << it.key(); appliedSorting = it.value(); - SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; + SinkTraceCtx(mLogCtx) << "Grouped sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; return keys; } } + + for (const auto &property : mSortedProperties) { + if (query.hasFilter(property)) { + Index index(sortedIndexName(property), transaction); + const auto keys = sortedIndexLookup(index, query.getFilter(property)); + appliedFilters << property; + SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << property << " found " << keys.size() << " keys."; + return keys; + } + } + for (const auto &property : mProperties) { if (query.hasFilter(property)) { Index index(indexName(property), transaction); diff --git a/tests/querytest.cpp b/tests/querytest.cpp --- a/tests/querytest.cpp +++ b/tests/querytest.cpp @@ -1510,6 +1510,113 @@ } } + void mailsWithDates() + { + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); + mail.setExtractedMessageId("message1"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-23T13:50:00Z", Qt::ISODate)); + mail.setExtractedMessageId("message2"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); + mail.setExtractedMessageId("message3"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedMessageId("message4"); + VERIFYEXEC(Sink::Store::create(mail)); + } + { + Mail mail("sink.dummy.instance1"); + mail.setExtractedDate(QDateTime::fromString("2078-05-23T13:49:41Z", Qt::ISODate)); + mail.setExtractedMessageId("message5"); + VERIFYEXEC(Sink::Store::create(mail)); + } + VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1")); + } + + void testMailDate() + { + mailsWithDates(); + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-27T13:49:41Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 0); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QDateTime::fromString("2018-05-27T13:50:00Z", Qt::ISODate)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + } + + void testMailRange() + { + mailsWithDates(); + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-23T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 1); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-25T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 2); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2018-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 3); + } + + { + Sink::Query query; + query.resourceFilter("sink.dummy.instance1"); + query.filter(QueryBase::Comparator(QVariantList{QDateTime::fromString("2018-05-22T13:49:41Z", Qt::ISODate), QDateTime::fromString("2118-05-30T13:49:41Z", Qt::ISODate)}, QueryBase::Comparator::Within)); + auto model = Sink::Store::loadModel(query); + QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool()); + QCOMPARE(model->rowCount(), 4); + } + } }; QTEST_MAIN(QueryTest)