diff --git a/common/typeindex.cpp b/common/typeindex.cpp index a897ad0c..180343a6 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -1,343 +1,343 @@ /* Copyright (c) 2015 Christian Mollekopf This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "typeindex.h" #include "log.h" #include "index.h" #include "fulltextindex.h" #include #include using namespace Sink; static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { QByteArray result; QDataStream ds(&result, QIODevice::WriteOnly); ds << value.toDateTime(); return result; } if (value.type() == QVariant::Bool) { return value.toBool() ? "t" : "f"; } if (value.canConvert()) { const auto ba = value.value().value; if (!ba.isEmpty()) { return ba; } } if (value.isValid() && !value.toByteArray().isEmpty()) { return value.toByteArray(); } // LMDB can't handle empty keys, so use something different return "toplevel"; } static QByteArray toSortableByteArray(const QDateTime &date) { // Sort invalid last if (!date.isValid()) { return QByteArray::number(std::numeric_limits::max()); } return QByteArray::number(std::numeric_limits::max() - date.toTime_t()); } TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) { } QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &sortProperty) const { if (sortProperty.isEmpty()) { return mType + ".index." + property; } return mType + ".index." + property + ".sort." + sortProperty; } template <> void TypeIndex::addProperty(const QByteArray &property) { auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); if (add) { Index(indexName(property), transaction).add(getByteArray(value), identifier); } else { Index(indexName(property), transaction).remove(getByteArray(value), identifier); } }; mIndexer.insert(property, indexer); mProperties << property; } template <> void TypeIndex::addProperty(const QByteArray &property) { auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { if (add) { Index(indexName(property), transaction).add(getByteArray(value), identifier); } else { Index(indexName(property), transaction).remove(getByteArray(value), identifier); } }; mIndexer.insert(property, indexer); mProperties << property; } template <> void TypeIndex::addProperty(const QByteArray &property) { auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); if (add) { Index(indexName(property), transaction).add(getByteArray(value), identifier); } else { Index(indexName(property), transaction).remove(getByteArray(value), identifier); } }; mIndexer.insert(property, indexer); mProperties << property; } template <> void TypeIndex::addProperty(const QByteArray &property) { auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { //SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << getByteArray(value); if (add) { Index(indexName(property), transaction).add(getByteArray(value), identifier); } else { Index(indexName(property), transaction).remove(getByteArray(value), identifier); } }; mIndexer.insert(property, indexer); mProperties << property; } template <> void TypeIndex::addProperty(const QByteArray &property) { addProperty(property); } template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { auto indexer = [=](bool add, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { const auto date = sortValue.toDateTime(); const auto propertyValue = getByteArray(value); if (add) { Index(indexName(property, sortProperty), transaction).add(propertyValue + toSortableByteArray(date), identifier); } else { Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); } }; mSortIndexer.insert(property + sortProperty, indexer); mSortedProperties.insert(property, sortProperty); } template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { addPropertyWithSorting(property, sortProperty); } void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { for (const auto &property : mProperties) { const auto value = entity.getProperty(property); auto indexer = mIndexer.value(property); indexer(add, identifier, value, transaction); } for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { const auto value = entity.getProperty(it.key()); const auto sortValue = entity.getProperty(it.value()); auto indexer = mSortIndexer.value(it.key() + it.value()); indexer(add, identifier, value, sortValue, transaction); } for (const auto &indexer : mCustomIndexer) { indexer->setup(this, &transaction, resourceInstanceId); if (add) { indexer->add(entity); } else { indexer->remove(entity); } } } void TypeIndex::commitTransaction() { for (const auto &indexer : mCustomIndexer) { indexer->commitTransaction(); } } void TypeIndex::abortTransaction() { for (const auto &indexer : mCustomIndexer) { indexer->abortTransaction(); } } void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(true, identifier, entity, transaction, resourceInstanceId); } void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(false, identifier, entity, transaction, resourceInstanceId); } static QVector indexLookup(Index &index, QueryBase::Comparator filter) { QVector keys; QByteArrayList lookupKeys; if (filter.comparator == Query::Comparator::Equals) { lookupKeys << getByteArray(filter.value); } else if (filter.comparator == Query::Comparator::In) { lookupKeys = filter.value.value(); } else { Q_ASSERT(false); } for (const auto &lookupKey : lookupKeys) { index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, true); } return keys; } QVector TypeIndex::query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { const auto baseFilters = query.getBaseFilters(); for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { if (it.value().comparator == QueryBase::Comparator::Fulltext) { FulltextIndex fulltextIndex{resourceInstanceId}; const auto keys = fulltextIndex.lookup(it.value().value.toString()); appliedFilters << it.key(); SinkTraceCtx(mLogCtx) << "Fulltext index lookup found " << keys.size() << " keys."; return keys; } } for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { Index index(indexName(it.key(), it.value()), transaction); const auto keys = indexLookup(index, query.getFilter(it.key())); appliedFilters << it.key(); appliedSorting = it.value(); SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; return keys; } } for (const auto &property : mProperties) { if (query.hasFilter(property)) { Index index(indexName(property), transaction); const auto keys = indexLookup(index, query.getFilter(property)); appliedFilters << property; SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } } SinkTraceCtx(mLogCtx) << "No matching index"; return {}; } QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties; if (mProperties.contains(property)) { QVector keys; Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); index.lookup( lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } else if (mSecondaryProperties.contains(property)) { //Lookups on secondary indexes first lookup the key, and then lookup the results again to resolve to entity id's QVector keys; auto resultProperty = mSecondaryProperties.value(property); QVector secondaryKeys; Index index(indexName(property + resultProperty), transaction); const auto lookupKey = getByteArray(value); index.lookup( lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); SinkTraceCtx(mLogCtx) << "Looked up secondary keys: " << secondaryKeys; for (const auto &secondary : secondaryKeys) { keys += lookup(resultProperty, secondary, transaction); } return keys; } else { SinkWarning() << "Tried to lookup " << property << " but couldn't find value"; } return QVector(); } template <> void TypeIndex::index(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).add(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::index(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).add(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); } template <> void TypeIndex::unindex(const QByteArray &leftName, const QByteArray &rightName, const QVariant &leftValue, const QVariant &rightValue, Sink::Storage::DataStore::Transaction &transaction) { Index(indexName(leftName + rightName), transaction).remove(getByteArray(leftValue), getByteArray(rightValue)); } template <> QVector TypeIndex::secondaryLookup(const QByteArray &leftName, const QByteArray &rightName, const QVariant &value) { QVector keys; Index index(indexName(leftName + rightName), *mTransaction); const auto lookupKey = getByteArray(value); index.lookup( - lookupKey, [&](const QByteArray &value) { keys << value; }, [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message << value << lookupKey; }); + lookupKey, [&](const QByteArray &value) { keys << QByteArray{value.constData(), value.size()}; }, [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message << value << lookupKey; }); return keys; } template <> QVector TypeIndex::secondaryLookup(const QByteArray &leftName, const QByteArray &rightName, const QVariant &value) { QVector keys; Index index(indexName(leftName + rightName), *mTransaction); const auto lookupKey = getByteArray(value); index.lookup( - lookupKey, [&](const QByteArray &value) { keys << value; }, [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message << value << lookupKey; }); + lookupKey, [&](const QByteArray &value) { keys << QByteArray{value.constData(), value.size()}; }, [=](const Index::Error &error) { SinkWarning() << "Lookup error in secondary index: " << error.message << value << lookupKey; }); return keys; }