diff --git a/common/datastorequery.h b/common/datastorequery.h --- a/common/datastorequery.h +++ b/common/datastorequery.h @@ -22,7 +22,7 @@ #include "resultset.h" #include "log.h" #include "storage/entitystore.h" - +#include "storage/key.h" class Source; class Bloom; @@ -59,13 +59,13 @@ typedef std::function FilterFunction; typedef std::function BufferCallback; - QVector indexLookup(const QByteArray &property, const QVariant &value); + QVector indexLookup(const QByteArray &property, const QVariant &value); - void readEntity(const QByteArray &key, const BufferCallback &resultCallback); - void readPrevious(const QByteArray &key, const std::function &callback); + void readEntity(const Sink::Storage::Identifier &id, const BufferCallback &resultCallback); + void readPrevious(const Sink::Storage::Identifier &id, const std::function &callback); ResultSet createFilteredSet(ResultSet &resultSet, const FilterFunction &); - QVector loadIncrementalResultSet(qint64 baseRevision); + QVector loadIncrementalResultSet(qint64 baseRevision); void setupQuery(const Sink::QueryBase &query_); QByteArrayList executeSubquery(const Sink::QueryBase &subquery); @@ -96,22 +96,22 @@ virtual ~FilterBase(){} - void readEntity(const QByteArray &key, const std::function &callback) + void readEntity(const Sink::Storage::Identifier &id, const std::function &callback) { Q_ASSERT(mDatastore); - mDatastore->readEntity(key, callback); + mDatastore->readEntity(id, callback); } - QVector indexLookup(const QByteArray &property, const QVariant &value) + QVector indexLookup(const QByteArray &property, const QVariant &value) { Q_ASSERT(mDatastore); return mDatastore->indexLookup(property, value); } - void readPrevious(const QByteArray &key, const std::function &callback) + void readPrevious(const Sink::Storage::Identifier &id, const std::function &callback) { Q_ASSERT(mDatastore); - mDatastore->readPrevious(key, callback); + mDatastore->readPrevious(id, callback); } virtual void skip() { mSource->skip(); } diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp --- a/common/datastorequery.cpp +++ b/common/datastorequery.cpp @@ -41,17 +41,20 @@ public: typedef QSharedPointer Ptr; - QVector mIds; - QVector::ConstIterator mIt; - QVector mIncrementalIds; - QVector::ConstIterator mIncrementalIt; + QVector mIds; + QVector::ConstIterator mIt; + QVector mIncrementalIds; + QVector::ConstIterator mIncrementalIt; Source (const QVector &ids, DataStoreQuery *store) : FilterBase(store), - mIds(ids), - mIt(mIds.constBegin()) + mIds() { - + mIds.reserve(ids.size()); + for (const auto &id : ids) { + mIds.append(Identifier::fromDisplayByteArray(id)); + } + mIt = mIds.constBegin(); } virtual ~Source(){} @@ -63,9 +66,13 @@ } }; - void add(const QVector &ids) + void add(const QVector &keys) { - mIncrementalIds = ids; + mIncrementalIds.clear(); + mIncrementalIds.reserve(keys.size()); + for (const auto &key : keys) { + mIncrementalIds.append(key.identifier()); + } mIncrementalIt = mIncrementalIds.constBegin(); } @@ -222,7 +229,7 @@ QSet mReducedValues; QSet mIncrementallyReducedValues; - QHash mSelectedValues; + QHash mSelectedValues; QByteArray mReductionProperty; QByteArray mSelectionProperty; QueryBase::Reduce::Selector::Comparator mSelectionComparator; @@ -263,16 +270,16 @@ } struct ReductionResult { - QByteArray selection; + Identifier selection; QVector aggregateIds; QMap aggregateValues; }; ReductionResult reduceOnValue(const QVariant &reductionValue) { QMap aggregateValues; QVariant selectionResultValue; - QByteArray selectionResult; + Identifier selectionResult; const auto results = indexLookup(mReductionProperty, reductionValue); for (auto &aggregator : mAggregators) { aggregator.reset(); @@ -284,7 +291,7 @@ if (!matchesFilter(entity)) { return; } - reducedAndFilteredResults << r; + reducedAndFilteredResults << r.toDisplayByteArray(); Q_ASSERT(operation != Sink::Operation_Removal); for (auto &aggregator : mAggregators) { if (!aggregator.property.isEmpty()) { @@ -296,7 +303,7 @@ auto selectionValue = entity.getProperty(mSelectionProperty); if (!selectionResultValue.isValid() || compare(selectionValue, selectionResultValue, mSelectionComparator)) { selectionResultValue = selectionValue; - selectionResult = entity.identifier(); + selectionResult = Identifier::fromDisplayByteArray(entity.identifier()); } }); } @@ -318,7 +325,8 @@ if (v.isNull() && result.operation == Sink::Operation_Removal) { //For removals we have to read the last revision to get a value, and thus be able to find the correct thread. QVariant reductionValue; - readPrevious(result.entity.identifier(), [&] (const ApplicationDomain::ApplicationDomainType &prev) { + const auto id = Identifier::fromDisplayByteArray(result.entity.identifier()); + readPrevious(id, [&] (const ApplicationDomain::ApplicationDomainType &prev) { Q_ASSERT(result.entity.identifier() == prev.identifier()); reductionValue = prev.getProperty(mReductionProperty); }); @@ -341,7 +349,7 @@ auto reductionResult = reduceOnValue(reductionValue); //This can happen if we get a removal message from a filtered entity and all entites of the reduction are filtered. - if (reductionResult.selection.isEmpty()) { + if (reductionResult.selection.isNull()) { return; } mSelectedValues.insert(reductionValueBa, reductionResult.selection); @@ -362,27 +370,27 @@ //If mSelectedValues did not contain the value, oldSelectionResult will be empty.(Happens if entites have been filtered) auto oldSelectionResult = mSelectedValues.take(reductionValueBa); SinkTraceCtx(mDatastore->mLogCtx) << "Old selection result: " << oldSelectionResult << " New selection result: " << selectionResult.selection; - if (selectionResult.selection.isEmpty() && oldSelectionResult.isEmpty()) { + if (selectionResult.selection.isNull() && oldSelectionResult.isNull()) { //Nothing to do, the item was filtered before, and still is. } else if (oldSelectionResult == selectionResult.selection) { mSelectedValues.insert(reductionValueBa, selectionResult.selection); - Q_ASSERT(!selectionResult.selection.isEmpty()); + Q_ASSERT(!selectionResult.selection.isNull()); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Modification, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); } else { //remove old result - if (!oldSelectionResult.isEmpty()) { + if (!oldSelectionResult.isNull()) { readEntity(oldSelectionResult, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Removal}); }); } //If the last item has been removed, then there's nothing to add - if (!selectionResult.selection.isEmpty()) { + if (!selectionResult.selection.isNull()) { //add new result mSelectedValues.insert(reductionValueBa, selectionResult.selection); - Q_ASSERT(!selectionResult.selection.isEmpty()); + Q_ASSERT(!selectionResult.selection.isNull()); readEntity(selectionResult.selection, [&](const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Operation) { callback({entity, Sink::Operation_Creation, selectionResult.aggregateValues, selectionResult.aggregateIds}); }); @@ -477,17 +485,17 @@ return state; } -void DataStoreQuery::readEntity(const QByteArray &key, const BufferCallback &resultCallback) +void DataStoreQuery::readEntity(const Identifier &id, const BufferCallback &resultCallback) { - mStore.readLatest(mType, key, resultCallback); + mStore.readLatest(mType, id, resultCallback); } -void DataStoreQuery::readPrevious(const QByteArray &key, const std::function &callback) +void DataStoreQuery::readPrevious(const Identifier &id, const std::function &callback) { - mStore.readPrevious(mType, key, mStore.maxRevision(), callback); + mStore.readPrevious(mType, id, mStore.maxRevision(), callback); } -QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) +QVector DataStoreQuery::indexLookup(const QByteArray &property, const QVariant &value) { return mStore.indexLookup(mType, property, value); } @@ -654,10 +662,10 @@ mCollector = Collector::Ptr::create(baseSet, this); } -QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) +QVector DataStoreQuery::loadIncrementalResultSet(qint64 baseRevision) { - QVector changedKeys; - mStore.readRevisions(baseRevision, mType, [&](const QByteArray &key) { + QVector changedKeys; + mStore.readRevisions(baseRevision, mType, [&](const Key &key) { changedKeys << key; }); return changedKeys; diff --git a/common/storage/entitystore.h b/common/storage/entitystore.h --- a/common/storage/entitystore.h +++ b/common/storage/entitystore.h @@ -59,18 +59,21 @@ QVector fullScan(const QByteArray &type); QVector indexLookup(const QByteArray &type, const QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting); - QVector indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value); + QVector indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value); void indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value, const std::function &callback); template void indexLookup(const QVariant &value, const std::function &callback) { return indexLookup(ApplicationDomain::getTypeName(), PropertyType::name, value, callback); } ///Returns the uid and buffer. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns an entity. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns an entity and operation. Note that the memory only remains valid until the next operation or transaction end. + void readLatest(const QByteArray &type, const Identifier &uid, const std::function callback); void readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback); ///Returns a copy @@ -94,10 +97,10 @@ } - void readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback); - void readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback); + void readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision, const std::function callback); + void readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision, const std::function callback); ///Returns a copy - ApplicationDomainType readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision); + ApplicationDomainType readPrevious(const QByteArray &type, const Sink::Storage::Identifier &id, qint64 revision); template T readPrevious(const QByteArray &uid, qint64 revision) { @@ -115,7 +118,7 @@ }); } - void readRevisions(qint64 baseRevision, const QByteArray &type, const std::function &callback); + void readRevisions(qint64 baseRevision, const QByteArray &type, const std::function &callback); ///Db contains entity (but may already be marked as removed bool contains(const QByteArray &type, const QByteArray &uid); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -461,11 +461,11 @@ return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction(), d->resourceContext.instanceId()); } -QVector EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value) +QVector EntityStore::indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value) { if (!d->exists()) { SinkTraceCtx(d->logCtx) << "Database is not existing: " << type; - return QVector(); + return {}; } return d->typeIndex(type).lookup(property, value, d->getTransaction()); } @@ -476,9 +476,9 @@ SinkTraceCtx(d->logCtx) << "Database is not existing: " << type; return; } - auto list = d->typeIndex(type).lookup(property, value, d->getTransaction()); - for (const auto &uid : list) { - callback(uid); + auto list = indexLookup(type, property, value); + for (const auto &id : list) { + callback(id.toDisplayByteArray()); } /* Index index(type + ".index." + property, d->transaction); */ /* index.lookup(value, [&](const QByteArray &sinkId) { */ @@ -489,44 +489,50 @@ /* }); */ } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &key, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const Identifier &id, const std::function callback) { Q_ASSERT(d); - Q_ASSERT(!key.isEmpty()); - // TODO: we shouldn't pass whole keys to this function - // check the testSingle test from querytest - const auto internalKey = [&key]() { - if(key.size() == Identifier::DISPLAY_REPR_SIZE) { - return Identifier::fromDisplayByteArray(key).toInternalByteArray(); - } else { - return Key::fromDisplayByteArray(key).toInternalByteArray(); - } - }(); + const auto internalKey = id.toInternalByteArray(); auto db = DataStore::mainDatabase(d->getTransaction(), type); db.findLatest(internalKey, [=](const QByteArray &key, const QByteArray &value) { const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); callback(uid, Sink::EntityBuffer(value.data(), value.size())); }, - [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << key; }); + [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << id; }); } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + +void EntityStore::readLatest(const QByteArray &type, const Identifier &uid, const std::function callback) { readLatest(type, uid, [&](const QByteArray &uid, const EntityBuffer &buffer) { //TODO cache max revision for the duration of the transaction. callback(d->createApplicationDomainType(type, uid, DataStore::maxRevision(d->getTransaction()), buffer)); }); } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + +void EntityStore::readLatest(const QByteArray &type, const Identifier &uid, const std::function callback) { readLatest(type, uid, [&](const QByteArray &uid, const EntityBuffer &buffer) { //TODO cache max revision for the duration of the transaction. callback(d->createApplicationDomainType(type, uid, DataStore::maxRevision(d->getTransaction()), buffer), buffer.operation()); }); } +void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +{ + readLatest(type, Identifier::fromDisplayByteArray(uid), callback); +} + ApplicationDomain::ApplicationDomainType EntityStore::readLatest(const QByteArray &type, const QByteArray &uid) { ApplicationDomainType dt; @@ -573,7 +579,7 @@ }); } -void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedType, const std::function &callback) +void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedType, const std::function &callback) { qint64 revisionCounter = baseRevision; const qint64 topRevision = DataStore::maxRevision(d->getTransaction()); @@ -592,15 +598,15 @@ const auto key = Key(Identifier::fromDisplayByteArray(uid), revisionCounter); revisionCounter++; - callback(key.toDisplayByteArray()); + callback(key); } } -void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback) +void EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision, const std::function callback) { auto db = DataStore::mainDatabase(d->getTransaction(), type); qint64 latestRevision = 0; - const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); + const auto internalUid = id.toInternalByteArray(); db.scan(internalUid, [&latestRevision, revision](const QByteArray &key, const QByteArray &) -> bool { const auto foundRevision = Key::fromInternalByteArray(key).revision().toQint64(); @@ -610,21 +616,21 @@ return true; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read current value from storage: " << error.message; }, true); - const auto key = Key(Identifier::fromDisplayByteArray(uid), latestRevision); + const auto key = Key(id, latestRevision); readEntity(type, key.toDisplayByteArray(), callback); } -void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback) +void EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision, const std::function callback) { - readPrevious(type, uid, revision, [&](const QByteArray &uid, const EntityBuffer &buffer) { + readPrevious(type, id, revision, [&](const QByteArray &uid, const EntityBuffer &buffer) { callback(d->createApplicationDomainType(type, uid, DataStore::maxRevision(d->getTransaction()), buffer)); }); } -ApplicationDomain::ApplicationDomainType EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision) +ApplicationDomain::ApplicationDomainType EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision) { ApplicationDomainType dt; - readPrevious(type, uid, revision, [&](const ApplicationDomainType &entity) { + readPrevious(type, id, revision, [&](const ApplicationDomainType &entity) { dt = *ApplicationDomainType::getInMemoryRepresentation(entity, entity.availableProperties()); }); return dt; diff --git a/common/storage/key.h b/common/storage/key.h --- a/common/storage/key.h +++ b/common/storage/key.h @@ -37,14 +37,20 @@ static const constexpr size_t INTERNAL_REPR_SIZE = 16; static const constexpr size_t DISPLAY_REPR_SIZE = 38; - Identifier() : uid(QUuid::createUuid()){}; + Identifier() = default; + static Identifier createIdentifier(); QByteArray toInternalByteArray() const; static Identifier fromInternalByteArray(const QByteArray &bytes); QString toDisplayString() const; QByteArray toDisplayByteArray() const; static Identifier fromDisplayByteArray(const QByteArray &bytes); + bool isNull() const; + + bool operator==(const Identifier &other) const; + bool operator!=(const Identifier &other) const; + private: explicit Identifier(const QUuid &uid) : uid(uid) {} QUuid uid; @@ -66,6 +72,9 @@ static Revision fromDisplayByteArray(const QByteArray &bytes); qint64 toQint64() const; + bool operator==(const Revision &other) const; + bool operator!=(const Revision &other) const; + private: qint64 rev; }; @@ -76,6 +85,7 @@ static const constexpr size_t INTERNAL_REPR_SIZE = Identifier::INTERNAL_REPR_SIZE + Revision::INTERNAL_REPR_SIZE; static const constexpr size_t DISPLAY_REPR_SIZE = Identifier::DISPLAY_REPR_SIZE + Revision::DISPLAY_REPR_SIZE; + Key() : id(), rev(0) {} Key(const Identifier &id, const Revision &rev) : id(id), rev(rev) {} QByteArray toInternalByteArray() const; @@ -87,6 +97,11 @@ const Revision &revision() const; void setRevision(const Revision &newRev); + bool isNull() const; + + bool operator==(const Key &other) const; + bool operator!=(const Key &other) const; + private: Identifier id; Revision rev; diff --git a/common/storage/key.cpp b/common/storage/key.cpp --- a/common/storage/key.cpp +++ b/common/storage/key.cpp @@ -46,8 +46,14 @@ // Identifier +Identifier Identifier::createIdentifier() +{ + return Identifier(QUuid::createUuid()); +} + QByteArray Identifier::toInternalByteArray() const { + Q_ASSERT(!uid.isNull()); return uid.toRfc4122(); } @@ -73,6 +79,21 @@ return Identifier(QUuid(bytes)); } +bool Identifier::isNull() const +{ + return uid.isNull(); +} + +bool Identifier::operator==(const Identifier &other) const +{ + return uid == other.uid; +} + +bool Identifier::operator!=(const Identifier &other) const +{ + return !(*this == other); +} + // Revision QByteArray Revision::toInternalByteArray() const @@ -107,6 +128,16 @@ return rev; } +bool Revision::operator==(const Revision &other) const +{ + return rev == other.rev; +} + +bool Revision::operator!=(const Revision &other) const +{ + return !(*this == other); +} + // Key QByteArray Key::toInternalByteArray() const @@ -154,3 +185,19 @@ { rev = newRev; } + +bool Key::isNull() const +{ + return id.isNull(); +} + +bool Key::operator==(const Key &other) const +{ + return (id == other.id) && (rev == other.rev); +} + +bool Key::operator!=(const Key &other) const +{ + return !(*this == other); +} + diff --git a/common/typeindex.h b/common/typeindex.h --- a/common/typeindex.h +++ b/common/typeindex.h @@ -95,7 +95,7 @@ void remove(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QVector query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); - QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); + QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); template QVector secondaryLookup(const QVariant &value) diff --git a/common/typeindex.cpp b/common/typeindex.cpp --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -427,17 +427,17 @@ return {}; } -QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, +QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties; if (mProperties.contains(property)) { - QVector keys; + QVector keys; Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); index.lookup(lookupKey, [&](const QByteArray &value) { - keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); + keys << Identifier::fromInternalByteArray(value); }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; @@ -447,7 +447,7 @@ } else if (mSecondaryProperties.contains(property)) { // Lookups on secondary indexes first lookup the key, and then lookup the results again to // resolve to entity id's - QVector keys; + QVector keys; auto resultProperty = mSecondaryProperties.value(property); QVector secondaryKeys; @@ -466,7 +466,7 @@ } else { SinkWarning() << "Tried to lookup " << property << " but couldn't find value"; } - return QVector(); + return {}; } template <>