diff --git a/common/datastorequery.h b/common/datastorequery.h --- a/common/datastorequery.h +++ b/common/datastorequery.h @@ -22,7 +22,7 @@ #include "resultset.h" #include "log.h" #include "storage/entitystore.h" - +#include "storage/key.h" class Source; class Bloom; @@ -59,13 +59,13 @@ typedef std::function FilterFunction; typedef std::function BufferCallback; - QVector indexLookup(const QByteArray &property, const QVariant &value); + QVector indexLookup(const QByteArray &property, const QVariant &value); - void readEntity(const QByteArray &key, const BufferCallback &resultCallback); - void readPrevious(const QByteArray &key, const std::function &callback); + void readEntity(const Sink::Storage::Identifier &id, const BufferCallback &resultCallback); + void readPrevious(const Sink::Storage::Identifier &id, const std::function &callback); ResultSet createFilteredSet(ResultSet &resultSet, const FilterFunction &); - QVector loadIncrementalResultSet(qint64 baseRevision); + QVector loadIncrementalResultSet(qint64 baseRevision); void setupQuery(const Sink::QueryBase &query_); QByteArrayList executeSubquery(const Sink::QueryBase &subquery); @@ -96,22 +96,22 @@ virtual ~FilterBase(){} - void readEntity(const QByteArray &key, const std::function &callback) + void readEntity(const Sink::Storage::Identifier &id, const std::function &callback) { Q_ASSERT(mDatastore); - mDatastore->readEntity(key, callback); + mDatastore->readEntity(id, callback); } - QVector indexLookup(const QByteArray &property, const QVariant &value) + QVector indexLookup(const QByteArray &property, const QVariant &value) { Q_ASSERT(mDatastore); return mDatastore->indexLookup(property, value); } - void readPrevious(const QByteArray &key, const std::function &callback) + void readPrevious(const Sink::Storage::Identifier &id, const std::function &callback) { Q_ASSERT(mDatastore); - mDatastore->readPrevious(key, callback); + mDatastore->readPrevious(id, callback); } virtual void skip() { mSource->skip(); } diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -41,17 +41,23 @@ public: typedef QSharedPointer Ptr; - QVector mIds; - QVector::ConstIterator mIt; - QVector mIncrementalIds; - QVector::ConstIterator mIncrementalIt; + QVector mIds; + QVector::ConstIterator mIt; + QVector mIncrementalIds; + QVector::ConstIterator mIncrementalIt; Source (const QVector &ids, DataStoreQuery *store) : FilterBase(store), - mIds(ids), - mIt(mIds.constBegin()) + mIds() { - + // Use reserve + append instead of QVector constructor with size here + // to avoid default construction, which would generate a new random + // Uuid + mIds.reserve(ids.size()); + for (const auto &id : ids) { + mIds.append(Identifier::fromDisplayByteArray(id)); + } + mIt = mIds.constBegin(); } virtual ~Source(){} @@ -63,9 +69,15 @@ } }; - void add(const QVector &ids) + void add(const QVector &keys) { - mIncrementalIds = ids; + // Use clear + reserve + append instead of resize here to avoid default + // construction + mIncrementalIds.clear(); + mIncrementalIds.reserve(keys.size()); + for (const auto &key : keys) { + mIncrementalIds.append(key.identifier()); + } mIncrementalIt = mIncrementalIds.constBegin(); } @@ -284,7 +296,7 @@ if (!matchesFilter(entity)) { return; } - reducedAndFilteredResults << r; + reducedAndFilteredResults << r.toDisplayByteArray(); Q_ASSERT(operation != Sink::Operation_Removal); for (auto &aggregator : mAggregators) { if (!aggregator.property.isEmpty()) { @@ -318,7 +330,8 @@ if (v.isNull() && result.operation == Sink::Operation_Removal) { //For removals we have to read the last revision to get a value, and thus be able to find the correct thread. QVariant reductionValue; - readPrevious(result.entity.identifier(), [&] (const ApplicationDomain::ApplicationDomainType &prev) { + const auto id = Identifier::fromDisplayByteArray(result.entity.identifier()); + readPrevious(id, [&] (const ApplicationDomain::ApplicationDomainType &prev) { Q_ASSERT(result.entity.identifier() == prev.identifier()); reductionValue = prev.getProperty(mReductionProperty); }); @@ -345,7 +358,7 @@ return; } mSelectedValues.insert(reductionValueBa, reductionResult.selection); - readEntity(reductionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { + readEntity(Identifier::fromDisplayByteArray(reductionResult.selection), [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation operation) { callback({entity, operation, reductionResult.aggregateValues, reductionResult.aggregateIds}); foundValue = true; }); @@ -368,13 +381,13 @@ if (oldSelectionResult == selectionResult.selection) { mSelectedValues.insert(reductionValueBa, selectionResult.selection); Q_ASSERT(!selectionResult.selection.isEmpty()); - readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { + readEntity(Identifier::fromDisplayByteArray(selectionResult.selection), [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Modification, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } else { //remove old result Q_ASSERT(!oldSelectionResult.isEmpty()); - readEntity(oldSelectionResult, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { + readEntity(Identifier::fromDisplayByteArray(oldSelectionResult), [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Removal}); }); @@ -383,7 +396,7 @@ //add new result mSelectedValues.insert(reductionValueBa, selectionResult.selection); Q_ASSERT(!selectionResult.selection.isEmpty()); - readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { + readEntity(Identifier::fromDisplayByteArray(selectionResult.selection), [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Creation, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } @@ -477,17 +490,17 @@ return state; } -void DataStoreQuery::readEntity(const QByteArray &key, const BufferCallback &resultCallback) +void DataStoreQuery::readEntity(const Identifier &id, const BufferCallback &resultCallback) { - mStore.readLatest(mType, key, resultCallback); + mStore.readLatest(mType, id, resultCallback); } -void DataStoreQuery::readPrevious(const QByteArray &key, const std::function &callback) +void DataStoreQuery::readPrevious(const Identifier &id, const std::function &callback) { - mStore.readPrevious(mType, key, mStore.maxRevision(), callback); + mStore.readPrevious(mType, id, mStore.maxRevision(), callback); } -QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) +QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) { return mStore.indexLookup(mType, property, value); } @@ -654,10 +667,10 @@ mCollector = Collector::Ptr::create(baseSet, this); } -QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) +QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) { - QVector changedKeys; - mStore.readRevisions(baseRevision, mType, [&](const QByteArray &key) { + QVector changedKeys; + mStore.readRevisions(baseRevision, mType, [&](const Key &key) { changedKeys << key; }); return changedKeys; diff --git a/common/storage/entitystore.h b/common/storage/entitystore.h --- a/common/storage/entitystore.h +++ b/common/storage/entitystore.h @@ -59,7 +59,7 @@ QVector fullScan(const QByteArray &type); QVector indexLookup(const QByteArray &type, const QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting); - QVector indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value); + QVector indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value); void indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value, const std::function &callback); template void indexLookup(const QVariant &value, const std::function &callback) { @@ -67,10 +67,13 @@ } ///Returns the uid and buffer. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns an entity. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns an entity and operation. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns a copy @@ -94,10 +97,10 @@ } - void readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback); - void readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback); + void readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision, const std::function callback); + void readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision, const std::function callback); ///Returns a copy - ApplicationDomainType readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision); + ApplicationDomainType readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision); template T readPrevious(const QByteArray &uid, qint64 revision) { @@ -115,7 +118,7 @@ }); } - void readRevisions(qint64 baseRevision, const QByteArray &type, const std::function &callback); + void readRevisions(qint64 baseRevision, const QByteArray &type, const std::function &callback); ///Db contains entity (but may already be marked as removed bool contains(const QByteArray &type, const QByteArray &uid); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -461,11 +461,11 @@ return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction(), d->resourceContext.instanceId()); } -QVector EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value) +QVector EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value) { if (!d->exists()) { SinkTraceCtx(d->logCtx) << "Database is not existing: " << type; - return QVector(); + return {}; } return d->typeIndex(type).lookup(property, value, d->getTransaction()); } @@ -476,9 +476,9 @@ SinkTraceCtx(d->logCtx) << "Database is not existing: " << type; return; } - auto list = d->typeIndex(type).lookup(property, value, d->getTransaction()); - for (const auto &uid : list) { - callback(uid); + auto list = indexLookup(type, property, value); + for (const auto &id : list) { + callback(id.toDisplayByteArray()); } /* Index index(type + ".index." + property, d->transaction); */ /* index.lookup(value, [&](const QByteArray &sinkId) { */ @@ -489,29 +489,25 @@ /* }); */ } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &key, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const Identifier &id, const std::function callback) { Q_ASSERT(d); - Q_ASSERT(!key.isEmpty()); - // TODO: we shouldn't pass whole keys to this function - // check the testSingle test from querytest - const auto internalKey = [&key]() { - if(key.size() == Identifier::DISPLAY_REPR_SIZE) { - return Identifier::fromDisplayByteArray(key).toInternalByteArray(); - } else { - return Key::fromDisplayByteArray(key).toInternalByteArray(); - } - }(); + const auto internalKey = id.toInternalByteArray(); auto db = DataStore::mainDatabase(d->getTransaction(), type); db.findLatest(internalKey, [=](const QByteArray &key, const QByteArray &value) { const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); callback(uid, Sink::EntityBuffer(value.data(), value.size())); }, - [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << key; }); + [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << id; }); } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + +void EntityStore::readLatest(const QByteArray &type, const Identifier &uid, const std::function callback) { readLatest(type, uid, [&](const QByteArray &uid, const EntityBuffer &buffer) { //TODO cache max revision for the duration of the transaction. @@ -519,7 +515,12 @@ }); } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + +void EntityStore::readLatest(const QByteArray &type, const Identifier &uid, const std::function callback) { readLatest(type, uid, [&](const QByteArray &uid, const EntityBuffer &buffer) { //TODO cache max revision for the duration of the transaction. @@ -527,6 +528,11 @@ }); } +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + ApplicationDomain::ApplicationDomainType EntityStore::readLatest(const QByteArray &type, const QByteArray &uid) { ApplicationDomainType dt; @@ -573,7 +579,7 @@ }); } -void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedType, const std::function &callback) +void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedType, const std::function &callback) { qint64 revisionCounter = baseRevision; const qint64 topRevision = DataStore::maxRevision(d->getTransaction()); @@ -592,15 +598,15 @@ const auto key = Key(Identifier::fromDisplayByteArray(uid), revisionCounter); revisionCounter++; - callback(key.toDisplayByteArray()); + callback(key); } } -void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback) +void EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision, const std::function callback) { auto db = DataStore::mainDatabase(d->getTransaction(), type); qint64 latestRevision = 0; - const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); + const auto internalUid = id.toInternalByteArray(); db.scan(internalUid, [&latestRevision, revision](const QByteArray &key, const QByteArray &) -> bool { const auto foundRevision = Key::fromInternalByteArray(key).revision().toQint64(); @@ -610,21 +616,21 @@ return true; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read current value from storage: " << error.message; }, true); - const auto key = Key(Identifier::fromDisplayByteArray(uid), latestRevision); + const auto key = Key(id, latestRevision); readEntity(type, key.toDisplayByteArray(), callback); } -void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback) +void EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision, const std::function callback) { - readPrevious(type, uid, revision, [&](const QByteArray &uid, const EntityBuffer &buffer) { + readPrevious(type, id, revision, [&](const QByteArray &uid, const EntityBuffer &buffer) { callback(d->createApplicationDomainType(type, uid, DataStore::maxRevision(d->getTransaction()), buffer)); }); } -ApplicationDomain::ApplicationDomainType EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision) +ApplicationDomain::ApplicationDomainType EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision) { ApplicationDomainType dt; - readPrevious(type, uid, revision, [&](const ApplicationDomainType &entity) { + readPrevious(type, id, revision, [&](const ApplicationDomainType &entity) { dt = *ApplicationDomainType::getInMemoryRepresentation(entity, entity.availableProperties()); }); return dt; diff --git a/common/storage/key.h b/common/storage/key.h --- a/common/storage/key.h +++ b/common/storage/key.h @@ -76,6 +76,9 @@ static const constexpr size_t INTERNAL_REPR_SIZE = Identifier::INTERNAL_REPR_SIZE + Revision::INTERNAL_REPR_SIZE; static const constexpr size_t DISPLAY_REPR_SIZE = Identifier::DISPLAY_REPR_SIZE + Revision::DISPLAY_REPR_SIZE; + // Just to be able to store keys into Qt containers + [[deprecated("Don't use the default constructor")]] + Key() : id(), rev(0) {} Key(const Identifier &id, const Revision &rev) : id(id), rev(rev) {} QByteArray toInternalByteArray() const; diff --git a/common/typeindex.h b/common/typeindex.h --- a/common/typeindex.h +++ b/common/typeindex.h @@ -95,7 +95,7 @@ void remove(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QVector query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); - QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); + QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); template QVector secondaryLookup(const QVariant &value) diff --git a/common/typeindex.cpp b/common/typeindex.cpp --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -427,17 +427,17 @@ return {}; } -QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, +QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties; if (mProperties.contains(property)) { - QVector keys; + QVector keys; Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); index.lookup(lookupKey, [&](const QByteArray &value) { - keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); + keys << Identifier::fromInternalByteArray(value); }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; @@ -447,7 +447,7 @@ } else if (mSecondaryProperties.contains(property)) { // Lookups on secondary indexes first lookup the key, and then lookup the results again to // resolve to entity id's - QVector keys; + QVector keys; auto resultProperty = mSecondaryProperties.value(property); QVector secondaryKeys; @@ -466,7 +466,7 @@ } else { SinkWarning() << "Tried to lookup " << property << " but couldn't find value"; } - return QVector(); + return {}; } template <>