diff --git a/autotests/server/fakedatastore.cpp b/autotests/server/fakedatastore.cpp index d384e423c..7f6101b82 100644 --- a/autotests/server/fakedatastore.cpp +++ b/autotests/server/fakedatastore.cpp @@ -1,322 +1,322 @@ /* Copyright (c) 2014 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "fakedatastore.h" #include "dbpopulator.h" #include "storage/dbconfig.h" #include "inspectablenotificationcollector.h" #include "akonadischema.h" #include "storage/dbinitializer.h" #include using namespace Akonadi::Server; Q_DECLARE_METATYPE(PimItem) Q_DECLARE_METATYPE(PimItem::List) Q_DECLARE_METATYPE(Collection) Q_DECLARE_METATYPE(Flag) Q_DECLARE_METATYPE(QVector) Q_DECLARE_METATYPE(Tag) Q_DECLARE_METATYPE(QVector) Q_DECLARE_METATYPE(MimeType) Q_DECLARE_METATYPE(QList) namespace Akonadi { namespace Server { class FakeDataStoreFactory : public DataStoreFactory { public: FakeDataStoreFactory() = default; ~FakeDataStoreFactory() override = default; DataStore * createStore() override { return new FakeDataStore(); } }; } } Akonadi::Server::FakeDataStore::FakeDataStore() : DataStore() , mPopulateDb(true) { mNotificationCollector = std::make_unique(this); } FakeDataStore::~FakeDataStore() { } void FakeDataStore::registerFactory() { sFactory.reset(new FakeDataStoreFactory); } bool FakeDataStore::init() { if (!DataStore::init()) { return false; } if (mPopulateDb) { DbPopulator dbPopulator; if (!dbPopulator.run()) { qWarning() << "Failed to populate database"; return false; } } return true; } bool FakeDataStore::setItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col, bool silent) { mChanges.insert(QStringLiteral("setItemsFlags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(flags) << QVariant::fromValue(col) << silent); return DataStore::setItemsFlags(items, flags, flagsChanged, col, silent); } bool FakeDataStore::appendItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, bool checkIfExists, const Collection &col, bool silent) { mChanges.insert(QStringLiteral("appendItemsFlags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(flags) << checkIfExists << QVariant::fromValue(col) << silent); return DataStore::appendItemsFlags(items, flags, flagsChanged, checkIfExists, col, silent); } bool FakeDataStore::removeItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col, bool silent) { mChanges.insert(QStringLiteral("removeItemsFlags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(flags) << QVariant::fromValue(col) << silent); return DataStore::removeItemsFlags(items, flags, flagsChanged, col, silent); } bool FakeDataStore::setItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { mChanges.insert(QStringLiteral("setItemsTags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(tags) << silent); return DataStore::setItemsTags(items, tags, tagsChanged, silent); } bool FakeDataStore::appendItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool checkIfExists, const Collection &col, bool silent) { mChanges.insert(QStringLiteral("appendItemsTags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(tags) << checkIfExists << QVariant::fromValue(col) << silent); return DataStore::appendItemsTags(items, tags, tagsChanged, checkIfExists, col, silent); } bool FakeDataStore::removeItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { mChanges.insert(QStringLiteral("removeItemsTags"), QVariantList() << QVariant::fromValue(items) << QVariant::fromValue(tags) << silent); return DataStore::removeItemsTags(items, tags, tagsChanged, silent); } bool FakeDataStore::removeItemParts(const PimItem &item, const QSet &parts) { mChanges.insert(QStringLiteral("remoteItemParts"), QVariantList() << QVariant::fromValue(item) << QVariant::fromValue(parts)); return DataStore::removeItemParts(item, parts); } bool FakeDataStore::invalidateItemCache(const PimItem &item) { mChanges.insert(QStringLiteral("invalidateItemCache"), QVariantList() << QVariant::fromValue(item)); return DataStore::invalidateItemCache(item); } bool FakeDataStore::appendCollection(Collection &collection, const QStringList &mimeTypes, const QMap &attributes) { mChanges.insert(QStringLiteral("appendCollection"), QVariantList() << QVariant::fromValue(collection) << mimeTypes << QVariant::fromValue(attributes)); return DataStore::appendCollection(collection, mimeTypes, attributes); } bool FakeDataStore::cleanupCollection(Collection &collection) { mChanges.insert(QStringLiteral("cleanupCollection"), QVariantList() << QVariant::fromValue(collection)); return DataStore::cleanupCollection(collection); } bool FakeDataStore::cleanupCollection_slow(Collection &collection) { mChanges.insert(QStringLiteral("cleanupCollection_slow"), QVariantList() << QVariant::fromValue(collection)); return DataStore::cleanupCollection_slow(collection); } bool FakeDataStore::moveCollection(Collection &collection, const Collection &newParent) { mChanges.insert(QStringLiteral("moveCollection"), QVariantList() << QVariant::fromValue(collection) << QVariant::fromValue(newParent)); return DataStore::moveCollection(collection, newParent); } bool FakeDataStore::appendMimeTypeForCollection(qint64 collectionId, const QStringList &mimeTypes) { mChanges.insert(QStringLiteral("appendMimeTypeForCollection"), QVariantList() << collectionId << QVariant::fromValue(mimeTypes)); return DataStore::appendMimeTypeForCollection(collectionId, mimeTypes); } void FakeDataStore::activeCachePolicy(Collection &col) { mChanges.insert(QStringLiteral("activeCachePolicy"), QVariantList() << QVariant::fromValue(col)); return DataStore::activeCachePolicy(col); } bool FakeDataStore::appendPimItem(QVector &parts, const QVector &flags, const MimeType &mimetype, const Collection &collection, const QDateTime &dateTime, const QString &remote_id, const QString &remoteRevision, const QString &gid, PimItem &pimItem) { mChanges.insert(QStringLiteral("appendPimItem"), QVariantList() << QVariant::fromValue(mimetype) << QVariant::fromValue(collection) << dateTime << remote_id << remoteRevision << gid); return DataStore::appendPimItem(parts, flags, mimetype, collection, dateTime, remote_id, remoteRevision, gid, pimItem); } -bool FakeDataStore::cleanupPimItems(const PimItem::List &items) +bool FakeDataStore::cleanupPimItems(const PimItem::List &items, bool silent) { mChanges.insert(QStringLiteral("cleanupPimItems"), - QVariantList() << QVariant::fromValue(items)); - return DataStore::cleanupPimItems(items); + QVariantList() << QVariant::fromValue(items) << silent); + return DataStore::cleanupPimItems(items, silent); } bool FakeDataStore::unhidePimItem(PimItem &pimItem) { mChanges.insert(QStringLiteral("unhidePimItem"), QVariantList() << QVariant::fromValue(pimItem)); return DataStore::unhidePimItem(pimItem); } bool FakeDataStore::unhideAllPimItems() { mChanges.insert(QStringLiteral("unhideAllPimItems"), QVariantList()); return DataStore::unhideAllPimItems(); } bool FakeDataStore::addCollectionAttribute(const Collection &col, const QByteArray &key, const QByteArray &value, bool silent) { mChanges.insert(QStringLiteral("addCollectionAttribute"), QVariantList() << QVariant::fromValue(col) << key << value << silent); return DataStore::addCollectionAttribute(col, key, value, silent); } bool FakeDataStore::removeCollectionAttribute(const Collection &col, const QByteArray &key) { mChanges.insert(QStringLiteral("removeCollectionAttribute"), QVariantList() << QVariant::fromValue(col) << key); return DataStore::removeCollectionAttribute(col, key); } bool FakeDataStore::beginTransaction(const QString &name) { mChanges.insert(QStringLiteral("beginTransaction"), QVariantList() << name); return DataStore::beginTransaction(name); } bool FakeDataStore::commitTransaction() { mChanges.insert(QStringLiteral("commitTransaction"), QVariantList()); return DataStore::commitTransaction(); } bool FakeDataStore::rollbackTransaction() { mChanges.insert(QStringLiteral("rollbackTransaction"), QVariantList()); return DataStore::rollbackTransaction(); } void FakeDataStore::setPopulateDb(bool populate) { mPopulateDb = populate; } diff --git a/autotests/server/fakedatastore.h b/autotests/server/fakedatastore.h index 3854a63d9..b5b59e349 100644 --- a/autotests/server/fakedatastore.h +++ b/autotests/server/fakedatastore.h @@ -1,139 +1,139 @@ /* Copyright (c) 2014 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef AKONADI_SERVER_FAKEDATASTORE_H #define AKONADI_SERVER_FAKEDATASTORE_H #include "storage/datastore.h" namespace Akonadi { namespace Server { class FakeDataStoreFactory; class FakeDataStore : public DataStore { Q_OBJECT friend class FakeDataStoreFactory; public: ~FakeDataStore() override; static void registerFactory(); bool init() override; QMap changes() const { return mChanges; } bool setItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged = nullptr, const Collection &col = Collection(), bool silent = false) override; bool appendItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged = nullptr, bool checkIfExists = true, const Collection &col = Collection(), bool silent = false) override; bool removeItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged = nullptr, const Collection &col = Collection(), bool silent = false) override; bool setItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool silent = false) override; bool appendItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool checkIfExists = true, const Collection &col = Collection(), bool silent = false) override; bool removeItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool silent = false) override; bool removeItemParts(const PimItem &item, const QSet &parts) override; bool invalidateItemCache(const PimItem &item) override; bool appendCollection(Collection &collection, const QStringList &mimeTypes, const QMap &attributes) override; bool cleanupCollection(Collection &collection) override; bool cleanupCollection_slow(Collection &collection) override; bool moveCollection(Collection &collection, const Collection &newParent) override; virtual bool appendMimeTypeForCollection(qint64 collectionId, const QStringList &mimeTypes) override; void activeCachePolicy(Collection &col) override; bool appendPimItem(QVector &parts, const QVector &flags, const MimeType &mimetype, const Collection &collection, const QDateTime &dateTime, const QString &remote_id, const QString &remoteRevision, const QString &gid, PimItem &pimItem) override; - bool cleanupPimItems(const PimItem::List &items) override; + bool cleanupPimItems(const PimItem::List &items, bool silent = false) override; bool unhidePimItem(PimItem &pimItem) override; bool unhideAllPimItems() override; bool addCollectionAttribute(const Collection &col, const QByteArray &key, const QByteArray &value, bool silent = false) override; bool removeCollectionAttribute(const Collection &col, const QByteArray &key) override; bool beginTransaction(const QString &name = QString()) override; bool rollbackTransaction() override; bool commitTransaction() override; void setPopulateDb(bool populate); protected: FakeDataStore(); QMap mChanges; private: bool populateDatabase(); bool mPopulateDb; }; } } #endif // AKONADI_SERVER_FAKEDATASTORE_H diff --git a/src/server/handler/itemcreatehandler.cpp b/src/server/handler/itemcreatehandler.cpp index 4d5286cbb..d2728184d 100644 --- a/src/server/handler/itemcreatehandler.cpp +++ b/src/server/handler/itemcreatehandler.cpp @@ -1,451 +1,512 @@ /*************************************************************************** * Copyright (C) 2007 by Robert Zwerus * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "itemcreatehandler.h" #include "itemfetchhelper.h" #include "connection.h" #include "preprocessormanager.h" #include "handlerhelper.h" #include "storage/datastore.h" #include "storage/transaction.h" #include "storage/parttypehelper.h" #include "storage/dbconfig.h" #include "storage/partstreamer.h" #include "storage/parthelper.h" #include "storage/selectquerybuilder.h" +#include "storage/itemretrievalmanager.h" #include +#include "shared/akranges.h" +#include "shared/akscopeguard.h" + #include //std::accumulate using namespace Akonadi; using namespace Akonadi::Server; bool ItemCreateHandler::buildPimItem(const Protocol::CreateItemCommand &cmd, PimItem &item, Collection &parentCol) { parentCol = HandlerHelper::collectionFromScope(cmd.collection(), connection()); if (!parentCol.isValid()) { return failureResponse(QStringLiteral("Invalid parent collection")); } if (parentCol.isVirtual()) { return failureResponse(QStringLiteral("Cannot append item into virtual collection")); } MimeType mimeType = MimeType::retrieveByNameOrCreate(cmd.mimeType()); if (!mimeType.isValid()) { return failureResponse(QStringLiteral("Unable to create mimetype '") % cmd.mimeType() % QStringLiteral("'.")); } item.setRev(0); item.setSize(cmd.itemSize()); item.setMimeTypeId(mimeType.id()); item.setCollectionId(parentCol.id()); item.setDatetime(cmd.dateTime()); if (cmd.remoteId().isEmpty()) { // from application item.setDirty(true); } else { // from resource item.setRemoteId(cmd.remoteId()); item.setDirty(false); } item.setRemoteRevision(cmd.remoteRevision()); item.setGid(cmd.gid()); item.setAtime(QDateTime::currentDateTimeUtc()); return true; } bool ItemCreateHandler::insertItem(const Protocol::CreateItemCommand &cmd, PimItem &item, const Collection &parentCol) { if (!item.datetime().isValid()) { item.setDatetime(QDateTime::currentDateTimeUtc()); } if (!item.insert()) { return failureResponse(QStringLiteral("Failed to append item")); } // set message flags const QSet flags = cmd.mergeModes() == Protocol::CreateItemCommand::None ? cmd.flags() : cmd.addedFlags(); if (!flags.isEmpty()) { // This will hit an entry in cache inserted there in buildPimItem() const Flag::List flagList = HandlerHelper::resolveFlags(flags); bool flagsChanged = false; if (!storageBackend()->appendItemsFlags({item}, flagList, &flagsChanged, false, parentCol, true)) { return failureResponse("Unable to append item flags."); } } const Scope tags = cmd.mergeModes() == Protocol::CreateItemCommand::None ? cmd.tags() : cmd.addedTags(); if (!tags.isEmpty()) { const Tag::List tagList = HandlerHelper::tagsFromScope(tags, connection()); bool tagsChanged = false; if (!storageBackend()->appendItemsTags({item}, tagList, &tagsChanged, false, parentCol, true)) { return failureResponse(QStringLiteral("Unable to append item tags.")); } } // Handle individual parts qint64 partSizes = 0; PartStreamer streamer(connection(), item); Q_FOREACH (const QByteArray &partName, cmd.parts()) { qint64 partSize = 0; try { streamer.stream(true, partName, partSize); } catch (const PartStreamerException &e) { return failureResponse(e.what()); } partSizes += partSize; } const Protocol::Attributes attrs = cmd.attributes(); for (auto iter = attrs.cbegin(), end = attrs.cend(); iter != end; ++iter) { try { streamer.streamAttribute(true, iter.key(), iter.value()); } catch (const PartStreamerException &e) { return failureResponse(e.what()); } } // TODO: Try to avoid this addition query if (partSizes > item.size()) { item.setSize(partSizes); item.update(); } // Preprocessing if (PreprocessorManager::instance()->isActive()) { Part hiddenAttribute; hiddenAttribute.setPimItemId(item.id()); hiddenAttribute.setPartType(PartTypeHelper::fromFqName(QStringLiteral(AKONADI_ATTRIBUTE_HIDDEN))); hiddenAttribute.setData(QByteArray()); hiddenAttribute.setDatasize(0); // TODO: Handle errors? Technically, this is not a critical issue as no data are lost PartHelper::insert(&hiddenAttribute); } const bool seen = flags.contains(AKONADI_FLAG_SEEN) || flags.contains(AKONADI_FLAG_IGNORED); notify(item, seen, item.collection()); sendResponse(item, Protocol::CreateItemCommand::None); return true; } bool ItemCreateHandler::mergeItem(const Protocol::CreateItemCommand &cmd, PimItem &newItem, PimItem ¤tItem, const Collection &parentCol) { bool needsUpdate = false; QSet changedParts; if (!newItem.remoteId().isEmpty() && currentItem.remoteId() != newItem.remoteId()) { currentItem.setRemoteId(newItem.remoteId()); changedParts.insert(AKONADI_PARAM_REMOTEID); needsUpdate = true; } if (!newItem.remoteRevision().isEmpty() && currentItem.remoteRevision() != newItem.remoteRevision()) { currentItem.setRemoteRevision(newItem.remoteRevision()); changedParts.insert(AKONADI_PARAM_REMOTEREVISION); needsUpdate = true; } if (!newItem.gid().isEmpty() && currentItem.gid() != newItem.gid()) { currentItem.setGid(newItem.gid()); changedParts.insert(AKONADI_PARAM_GID); needsUpdate = true; } if (newItem.datetime().isValid() && newItem.datetime() != currentItem.datetime()) { currentItem.setDatetime(newItem.datetime()); needsUpdate = true; } if (newItem.size() > 0 && newItem.size() != currentItem.size()) { currentItem.setSize(newItem.size()); needsUpdate = true; } const Collection col = Collection::retrieveById(parentCol.id()); if (cmd.flags().isEmpty() && !cmd.flagsOverwritten()) { bool flagsAdded = false, flagsRemoved = false; if (!cmd.addedFlags().isEmpty()) { const auto addedFlags = HandlerHelper::resolveFlags(cmd.addedFlags()); storageBackend()->appendItemsFlags({currentItem}, addedFlags, &flagsAdded, true, col, true); } if (!cmd.removedFlags().isEmpty()) { const auto removedFlags = HandlerHelper::resolveFlags(cmd.removedFlags()); storageBackend()->removeItemsFlags({currentItem}, removedFlags, &flagsRemoved, col, true); } if (flagsAdded || flagsRemoved) { changedParts.insert(AKONADI_PARAM_FLAGS); needsUpdate = true; } } else { bool flagsChanged = false; QSet flagNames = cmd.flags(); static QVector localFlagsToPreserve = { "$ATTACHMENT", "$INVITATION", "$ENCRYPTED", "$SIGNED", "$WATCHED" }; // Make sure we don't overwrite some local-only flags that can't come // through from Resource during ItemSync, like $ATTACHMENT, because the // resource is not aware of them (they are usually assigned by client // upon inspecting the payload) Q_FOREACH (const Flag ¤tFlag, currentItem.flags()) { const QByteArray currentFlagName = currentFlag.name().toLatin1(); if (localFlagsToPreserve.contains(currentFlagName)) { flagNames.insert(currentFlagName); } } const auto flags = HandlerHelper::resolveFlags(flagNames); storageBackend()->setItemsFlags({currentItem}, flags, &flagsChanged, col, true); if (flagsChanged) { changedParts.insert(AKONADI_PARAM_FLAGS); needsUpdate = true; } } if (cmd.tags().isEmpty()) { bool tagsAdded = false, tagsRemoved = false; if (!cmd.addedTags().isEmpty()) { const auto addedTags = HandlerHelper::tagsFromScope(cmd.addedTags(), connection()); storageBackend()->appendItemsTags({currentItem}, addedTags, &tagsAdded, true, col, true); } if (!cmd.removedTags().isEmpty()) { const Tag::List removedTags = HandlerHelper::tagsFromScope(cmd.removedTags(), connection()); storageBackend()->removeItemsTags({currentItem}, removedTags, &tagsRemoved, true); } if (tagsAdded || tagsRemoved) { changedParts.insert(AKONADI_PARAM_TAGS); needsUpdate = true; } } else { bool tagsChanged = false; const auto tags = HandlerHelper::tagsFromScope(cmd.tags(), connection()); storageBackend()->setItemsTags({currentItem}, tags, &tagsChanged, true); if (tagsChanged) { changedParts.insert(AKONADI_PARAM_TAGS); needsUpdate = true; } } const Part::List existingParts = Part::retrieveFiltered(Part::pimItemIdColumn(), currentItem.id()); QMap partsSizes; for (const Part &part : existingParts) { partsSizes.insert(PartTypeHelper::fullName(part.partType()).toLatin1(), part.datasize()); } PartStreamer streamer(connection(), currentItem); Q_FOREACH (const QByteArray &partName, cmd.parts()) { bool changed = false; qint64 partSize = 0; try { streamer.stream(true, partName, partSize, &changed); } catch (const PartStreamerException &e) { return failureResponse(e.what()); } if (changed) { changedParts.insert(partName); partsSizes.insert(partName, partSize); needsUpdate = true; } } const qint64 size = std::accumulate(partsSizes.begin(), partsSizes.end(), 0); if (size > currentItem.size()) { currentItem.setSize(size); needsUpdate = true; } if (needsUpdate) { currentItem.setRev(qMax(newItem.rev(), currentItem.rev()) + 1); currentItem.setAtime(QDateTime::currentDateTimeUtc()); // Only mark dirty when merged from application currentItem.setDirty(!connection()->context()->resource().isValid()); // Store all changes if (!currentItem.update()) { return failureResponse("Failed to store merged item"); } notify(currentItem, currentItem.collection(), changedParts); } sendResponse(currentItem, cmd.mergeModes()); return true; } bool ItemCreateHandler::sendResponse(const PimItem &item, Protocol::CreateItemCommand::MergeModes mergeModes) { if (mergeModes & Protocol::CreateItemCommand::Silent || mergeModes & Protocol::CreateItemCommand::None) { Protocol::FetchItemsResponse resp; resp.setId(item.id()); resp.setMTime(item.datetime()); Handler::sendResponse(std::move(resp)); return true; } Protocol::ItemFetchScope fetchScope; fetchScope.setAncestorDepth(Protocol::ItemFetchScope::ParentAncestor); fetchScope.setFetch(Protocol::ItemFetchScope::AllAttributes | Protocol::ItemFetchScope::FullPayload | Protocol::ItemFetchScope::CacheOnly | Protocol::ItemFetchScope::Flags | Protocol::ItemFetchScope::GID | Protocol::ItemFetchScope::MTime | Protocol::ItemFetchScope::RemoteID | Protocol::ItemFetchScope::RemoteRevision | Protocol::ItemFetchScope::Size | Protocol::ItemFetchScope::Tags); ImapSet set; set.add(QVector() << item.id()); Scope scope; scope.setUidSet(set); ItemFetchHelper fetchHelper(connection(), scope, fetchScope, Protocol::TagFetchScope{}); if (!fetchHelper.fetchItems()) { return failureResponse("Failed to retrieve item"); } return true; } bool ItemCreateHandler::notify(const PimItem &item, bool seen, const Collection &collection) { storageBackend()->notificationCollector()->itemAdded(item, seen, collection); if (PreprocessorManager::instance()->isActive()) { // enqueue the item for preprocessing PreprocessorManager::instance()->beginHandleItem(item, storageBackend()); } return true; } bool ItemCreateHandler::notify(const PimItem &item, const Collection &collection, const QSet &changedParts) { if (!changedParts.isEmpty()) { storageBackend()->notificationCollector()->itemChanged(item, changedParts, collection); } return true; } +void ItemCreateHandler::recoverFromMultipleMergeCandidates(const PimItem::List &items, const Collection &collection) +{ + // HACK HACK HACK: When this happens within ItemSync, we are running inside a client-side + // transaction, so just calling commit here won't have any effect, since this handler will + // ultimately fail and the client will rollback the transaction. To circumvent this, we + // will forcibly commit the transaction, do our changes here within a new transaction and + // then we open a new transaction so that the client won't notice. + + int transactionDepth = 0; + while (storageBackend()->inTransaction()) { + ++transactionDepth; + storageBackend()->commitTransaction(); + } + const AkScopeGuard restoreTransaction([&]() { + for (int i = 0; i < transactionDepth; ++i) { + storageBackend()->beginTransaction(QStringLiteral("RestoredTransactionAfterMMCRecovery")); + } + }); + + Transaction transaction(storageBackend(), QStringLiteral("MMC Recovery Transaction")); + + // If any of the conflicting items is dirty or does not have a remote ID, we don't want to remove + // them as it would cause data loss. There's a chance next changeReplay will fix this, so + // next time the ItemSync hits this multiple merge candidates, all changes will be committed + // and this check will succeed + if (items | any([](const auto &item) { return item.dirty() || item.remoteId().isEmpty(); })) { + qCWarning(AKONADISERVER_LOG) << "Automatic multiple merge candidates recovery failed: at least one of the candidates has uncommitted changes!"; + return; + } + + // This cannot happen with ItemSync, but in theory could happen during individual GID merge. + if (items | any([collection](const auto &item) { return item.collectionId() != collection.id(); })) { + qCWarning(AKONADISERVER_LOG) << "Automatic multiple merge candidates recovery failed: all candidates do not belong to the same collection."; + return; + } + + storageBackend()->cleanupPimItems(items, DataStore::Silent); + if (!transaction.commit()) { + qCWarning(AKONADISERVER_LOG) << "Automatic multiple merge candidates recovery failed: failed to commit database transaction."; + return; + } + + + // Schedule a new sync of the collection, one that will succeed + const auto resource = collection.resource().name(); + QMetaObject::invokeMethod(ItemRetrievalManager::instance(), "triggerCollectionSync", + Qt::QueuedConnection, + Q_ARG(QString, resource), Q_ARG(qint64, collection.id())); + + qCInfo(AKONADISERVER_LOG) << "Automatic multiple merge candidates recovery successful: conflicting items" << (items | transform([](const auto &i) { return i.id(); }) | toQVector) + << "in collection" << collection.name() << "(ID:" << collection.id() << ") were removed and a new sync was scheduled in the resource" + << resource; +} + bool ItemCreateHandler::parseStream() { const auto &cmd = Protocol::cmdCast(m_command); // FIXME: The streaming/reading of all item parts can hold the transaction for // unnecessary long time -> should we wrap the PimItem into one transaction // and try to insert Parts independently? In case we fail to insert a part, // it's not a problem as it can be re-fetched at any time, except for attributes. Transaction transaction(storageBackend(), QStringLiteral("ItemCreateHandler")); ExternalPartStorageTransaction storageTrx; PimItem item; Collection parentCol; if (!buildPimItem(cmd, item, parentCol)) { return false; } - if (cmd.mergeModes() == Protocol::CreateItemCommand::None) { if (!insertItem(cmd, item, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse(QStringLiteral("Failed to commit transaction")); } storageTrx.commit(); } else { // Merging is always restricted to the same collection SelectQueryBuilder qb; qb.setForUpdate(); qb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, parentCol.id()); Query::Condition rootCondition(Query::Or); Query::Condition mergeCondition(Query::And); if (cmd.mergeModes() & Protocol::CreateItemCommand::GID) { mergeCondition.addValueCondition(PimItem::gidColumn(), Query::Equals, item.gid()); } if (cmd.mergeModes() & Protocol::CreateItemCommand::RemoteID) { mergeCondition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, item.remoteId()); } rootCondition.addCondition(mergeCondition); // If an Item with matching RID but empty GID exists during GID merge, // merge into this item instead of creating a new one if (cmd.mergeModes() & Protocol::CreateItemCommand::GID && !item.remoteId().isEmpty()) { mergeCondition = Query::Condition(Query::And); mergeCondition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, item.remoteId()); mergeCondition.addValueCondition(PimItem::gidColumn(), Query::Equals, QLatin1String("")); rootCondition.addCondition(mergeCondition); } qb.addCondition(rootCondition); if (!qb.exec()) { return failureResponse("Failed to query database for item"); } const QVector result = qb.result(); if (result.isEmpty()) { // No item with such GID/RID exists, so call ItemCreateHandler::insert() and behave // like if this was a new item if (!insertItem(cmd, item, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse("Failed to commit transaction"); } storageTrx.commit(); } else if (result.count() == 1) { // Item with matching GID/RID combination exists, so merge this item into it // and send itemChanged() PimItem existingItem = result.at(0); if (!mergeItem(cmd, item, existingItem, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse("Failed to commit transaction"); } storageTrx.commit(); } else { - qCWarning(AKONADISERVER_LOG) << "Multiple merge candidates:"; + qCWarning(AKONADISERVER_LOG) << "Multiple merge candidates, will attempt to recover:"; for (const PimItem &item : result) { qCWarning(AKONADISERVER_LOG) << "\tID:" << item.id() << ", RID:" << item.remoteId() << ", GID:" << item.gid() << ", Collection:" << item.collection().name() << "(" << item.collectionId() << ")" << ", Resource:" << item.collection().resource().name() << "(" << item.collection().resourceId() << ")"; } - // Nor GID or RID are guaranteed to be unique, so make sure we don't merge - // something we don't want + + transaction.commit(); // commit the current transaction, before we attempt MMC recovery + recoverFromMultipleMergeCandidates(result, parentCol); + + // Even if the recovery was successful, indicate error to force the client to abort the + // sync, since we've interfered with the overall state. return failureResponse(QStringLiteral("Multiple merge candidates in collection '%1', aborting").arg(item.collection().name())); } } return successResponse(); } diff --git a/src/server/handler/itemcreatehandler.h b/src/server/handler/itemcreatehandler.h index a6dafe742..4fdbbc3f9 100644 --- a/src/server/handler/itemcreatehandler.h +++ b/src/server/handler/itemcreatehandler.h @@ -1,70 +1,74 @@ /*************************************************************************** * Copyright (C) 2007 by Robert Zwerus * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef AKONADI_ITEMCREATEHANDLER_H_ #define AKONADI_ITEMCREATEHANDLER_H_ #include "handler.h" #include "entities.h" namespace Akonadi { namespace Server { +class Transaction; + /** @ingroup akonadi_server_handler Handler for the X-AKAPPEND command. This command is used to append an item with multiple parts. */ class ItemCreateHandler: public Handler { public: ~ItemCreateHandler() override = default; bool parseStream() override; private: bool buildPimItem(const Protocol::CreateItemCommand &cmd, PimItem &item, Collection &parentCollection); bool insertItem(const Protocol::CreateItemCommand &cmd, PimItem &item, const Collection &parentCollection); bool mergeItem(const Protocol::CreateItemCommand &cmd, PimItem &newItem, PimItem ¤tItem, const Collection &parentCollection); bool sendResponse(const PimItem &item, Protocol::CreateItemCommand::MergeModes mergeModes); bool notify(const PimItem &item, bool seen, const Collection &collection); bool notify(const PimItem &item, const Collection &collection, const QSet &changedParts); + + void recoverFromMultipleMergeCandidates(const PimItem::List &items, const Collection &collection); }; } // namespace Server } // namespace Akonadi #endif diff --git a/src/server/storage/datastore.cpp b/src/server/storage/datastore.cpp index 4f7bf8908..890176438 100644 --- a/src/server/storage/datastore.cpp +++ b/src/server/storage/datastore.cpp @@ -1,1517 +1,1519 @@ /*************************************************************************** * Copyright (C) 2006 by Andreas Gungl * * Copyright (C) 2007 by Robert Zwerus * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "datastore.h" #include "akonadi.h" #include "collectionstatistics.h" #include "dbconfig.h" #include "dbinitializer.h" #include "dbupdater.h" #include "notificationmanager.h" #include "tracer.h" #include "transaction.h" #include "selectquerybuilder.h" #include "handlerhelper.h" #include "countquerybuilder.h" #include "parthelper.h" #include "handler.h" #include "collectionqueryhelper.h" #include "akonadischema.h" #include "parttypehelper.h" #include "querycache.h" #include "queryhelper.h" #include "akonadiserver_debug.h" #include "storagedebugger.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; static QMutex sTransactionMutex; bool DataStore::s_hasForeignKeyConstraints = false; static QThreadStorage sInstances; #define TRANSACTION_MUTEX_LOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.lock() #define TRANSACTION_MUTEX_UNLOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.unlock() #define setBoolPtr(ptr, val) \ { \ if ((ptr)) { \ *(ptr) = (val); \ } \ } DataStore *DataStoreFactory::createStore() { return new DataStore(); } std::unique_ptr DataStore::sFactory = std::make_unique(); /*************************************************************************** * DataStore * ***************************************************************************/ DataStore::DataStore() : QObject() , m_dbOpened(false) , m_transactionLevel(0) , m_keepAliveTimer(nullptr) { if (DbConfig::configuredDatabase()->driverName() == QLatin1String("QMYSQL")) { // Send a dummy query to MySQL every 1 hour to keep the connection alive, // otherwise MySQL just drops the connection and our subsequent queries fail // without properly reporting the error m_keepAliveTimer = new QTimer(this); m_keepAliveTimer->setInterval(3600 * 1000); QObject::connect(m_keepAliveTimer, &QTimer::timeout, this, &DataStore::sendKeepAliveQuery); } } DataStore::~DataStore() { if (m_dbOpened) { close(); } } void DataStore::open() { m_connectionName = QUuid::createUuid().toString() + QString::number(reinterpret_cast(QThread::currentThread())); Q_ASSERT(!QSqlDatabase::contains(m_connectionName)); m_database = QSqlDatabase::addDatabase(DbConfig::configuredDatabase()->driverName(), m_connectionName); DbConfig::configuredDatabase()->apply(m_database); if (!m_database.isValid()) { m_dbOpened = false; return; } m_dbOpened = m_database.open(); if (!m_dbOpened) { debugLastDbError("Cannot open database."); } else { qCDebug(AKONADISERVER_LOG) << "Database" << m_database.databaseName() << "opened using driver" << m_database.driverName(); } StorageDebugger::instance()->addConnection(reinterpret_cast(this), QThread::currentThread()->objectName()); connect(QThread::currentThread(), &QThread::objectNameChanged, this, [this](const QString &name) { if (!name.isEmpty()) { StorageDebugger::instance()->changeConnection(reinterpret_cast(this), name); } }); DbConfig::configuredDatabase()->initSession(m_database); if (m_keepAliveTimer) { m_keepAliveTimer->start(); } } QSqlDatabase DataStore::database() { if (!m_dbOpened) { open(); } return m_database; } void DataStore::close() { if (m_keepAliveTimer) { m_keepAliveTimer->stop(); } if (!m_dbOpened) { return; } if (inTransaction()) { // By setting m_transactionLevel to '1' here, we skip all nested transactions // and rollback the outermost transaction. m_transactionLevel = 1; rollbackTransaction(); } QueryCache::clear(); m_database.close(); m_database = QSqlDatabase(); QSqlDatabase::removeDatabase(m_connectionName); StorageDebugger::instance()->removeConnection(reinterpret_cast(this)); m_dbOpened = false; } bool DataStore::init() { Q_ASSERT(QThread::currentThread() == QCoreApplication::instance()->thread()); AkonadiSchema schema; DbInitializer::Ptr initializer = DbInitializer::createInstance(database(), &schema); if (!initializer->run()) { qCCritical(AKONADISERVER_LOG) << initializer->errorMsg(); return false; } s_hasForeignKeyConstraints = initializer->hasForeignKeyConstraints(); if (QFile::exists(QStringLiteral(":dbupdate.xml"))) { DbUpdater updater(database(), QStringLiteral(":dbupdate.xml")); if (!updater.run()) { return false; } } else { qCWarning(AKONADISERVER_LOG) << "Warning: dbupdate.xml not found, skipping updates"; } if (!initializer->updateIndexesAndConstraints()) { qCCritical(AKONADISERVER_LOG) << initializer->errorMsg(); return false; } // enable caching for some tables MimeType::enableCache(true); Flag::enableCache(true); Resource::enableCache(true); Collection::enableCache(true); PartType::enableCache(true); return true; } NotificationCollector *DataStore::notificationCollector() { if (!mNotificationCollector) { mNotificationCollector = std::make_unique(this); } return mNotificationCollector.get(); } DataStore *DataStore::self() { if (!sInstances.hasLocalData()) { sInstances.setLocalData(sFactory->createStore()); } return sInstances.localData(); } bool DataStore::hasDataStore() { return sInstances.hasLocalData(); } /* --- ItemFlags ----------------------------------------------------- */ bool DataStore::setItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col_, bool silent) { QSet removedFlags; QSet addedFlags; QVariantList insIds; QVariantList insFlags; Query::Condition delConds(Query::Or); Collection col = col_; setBoolPtr(flagsChanged, false); for (const PimItem &item : items) { const Flag::List itemFlags = item.flags(); for (const Flag &flag : itemFlags) { if (!flags.contains(flag)) { removedFlags << flag.name(); Query::Condition cond; cond.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::Equals, item.id()); cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), Query::Equals, flag.id()); delConds.addCondition(cond); } } for (const Flag &flag : flags) { if (!itemFlags.contains(flag)) { addedFlags << flag.name(); insIds << item.id(); insFlags << flag.id(); } } if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } } if (!removedFlags.empty()) { QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); qb.addCondition(delConds); if (!qb.exec()) { return false; } } if (!addedFlags.empty()) { QueryBuilder qb2(PimItemFlagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemFlagRelation::leftColumn(), insIds); qb2.setColumnValue(PimItemFlagRelation::rightColumn(), insFlags); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { return false; } } if (!silent && (!addedFlags.isEmpty() || !removedFlags.isEmpty())) { QSet addedFlagsBa, removedFlagsBa; for (const auto &addedFlag : qAsConst(addedFlags)) { addedFlagsBa.insert(addedFlag.toLatin1()); } for (const auto &removedFlag : qAsConst(removedFlags)) { removedFlagsBa.insert(removedFlag.toLatin1()); } notificationCollector()->itemsFlagsChanged(items, addedFlagsBa, removedFlagsBa, col); } setBoolPtr(flagsChanged, (addedFlags != removedFlags)); return true; } bool DataStore::doAppendItemsFlag(const PimItem::List &items, const Flag &flag, const QSet &existing, const Collection &col_, bool silent) { Collection col = col_; QVariantList flagIds; QVariantList appendIds; PimItem::List appendItems; for (const PimItem &item : items) { if (existing.contains(item.id())) { continue; } flagIds << flag.id(); appendIds << item.id(); appendItems << item; if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } } if (appendItems.isEmpty()) { return true; // all items have the desired flags already } QueryBuilder qb2(PimItemFlagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemFlagRelation::leftColumn(), appendIds); qb2.setColumnValue(PimItemFlagRelation::rightColumn(), flagIds); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to append flag" << flag.name() << "to Items" << appendIds; return false; } if (!silent) { notificationCollector()->itemsFlagsChanged(appendItems, {flag.name().toLatin1()}, {}, col); } return true; } bool DataStore::appendItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, bool checkIfExists, const Collection &col, bool silent) { QVariantList itemsIds; itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds.append(item.id()); } setBoolPtr(flagsChanged, false); for (const Flag &flag : flags) { QSet existing; if (checkIfExists) { QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Select); Query::Condition cond; cond.addValueCondition(PimItemFlagRelation::rightColumn(), Query::Equals, flag.id()); cond.addValueCondition(PimItemFlagRelation::leftColumn(), Query::In, itemsIds); qb.addColumn(PimItemFlagRelation::leftColumn()); qb.addCondition(cond); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to retrieve existing flags for Items " << itemsIds; return false; } QSqlQuery query = qb.query(); if (query.driver()->hasFeature(QSqlDriver::QuerySize)) { //The query size feature is not supported by the sqllite driver if (query.size() == items.count()) { continue; } setBoolPtr(flagsChanged, true); } while (query.next()) { existing << query.value(0).value(); } if (!query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (existing.size() != items.count()) { setBoolPtr(flagsChanged, true); } } query.finish(); } if (!doAppendItemsFlag(items, flag, existing, col, silent)) { return false; } } return true; } bool DataStore::removeItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col_, bool silent) { Collection col = col_; QSet removedFlags; QVariantList itemsIds; QVariantList flagsIds; setBoolPtr(flagsChanged, false); itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds << item.id(); if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } for (int i = 0; i < flags.count(); ++i) { const QString flagName = flags[i].name(); if (!removedFlags.contains(flagName)) { flagsIds << flags[i].id(); removedFlags << flagName; } } } // Delete all given flags from all given items in one go QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); Query::Condition cond(Query::And); cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), Query::In, flagsIds); cond.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::In, itemsIds); qb.addCondition(cond); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to remove flags" << flags << "from Items" << itemsIds; return false; } if (qb.query().numRowsAffected() != 0) { setBoolPtr(flagsChanged, true); if (!silent) { QSet removedFlagsBa; for (const auto &remoteFlag : qAsConst(removedFlags)) { removedFlagsBa.insert(remoteFlag.toLatin1()); } notificationCollector()->itemsFlagsChanged(items, {}, removedFlagsBa, col); } } return true; } /* --- ItemTags ----------------------------------------------------- */ bool DataStore::setItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { QSet removedTags; QSet addedTags; QVariantList insIds; QVariantList insTags; Query::Condition delConds(Query::Or); setBoolPtr(tagsChanged, false); for (const PimItem &item : items) { const Tag::List itemTags = item.tags(); for (const Tag &tag : itemTags) { if (!tags.contains(tag)) { // Remove tags from items that had it set removedTags << tag.id(); Query::Condition cond; cond.addValueCondition(PimItemTagRelation::leftFullColumnName(), Query::Equals, item.id()); cond.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::Equals, tag.id()); delConds.addCondition(cond); } } for (const Tag &tag : tags) { if (!itemTags.contains(tag)) { // Add tags to items that did not have the tag addedTags << tag.id(); insIds << item.id(); insTags << tag.id(); } } } if (!removedTags.empty()) { QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Delete); qb.addCondition(delConds); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << removedTags << "from Items"; return false; } } if (!addedTags.empty()) { QueryBuilder qb2(PimItemTagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemTagRelation::leftColumn(), insIds); qb2.setColumnValue(PimItemTagRelation::rightColumn(), insTags); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to add tags" << addedTags << "to Items"; return false; } } if (!silent && (!addedTags.empty() || !removedTags.empty())) { notificationCollector()->itemsTagsChanged(items, addedTags, removedTags); } setBoolPtr(tagsChanged, (addedTags != removedTags)); return true; } bool DataStore::doAppendItemsTag(const PimItem::List &items, const Tag &tag, const QSet &existing, const Collection &col, bool silent) { QVariantList tagIds; QVariantList appendIds; PimItem::List appendItems; for (const PimItem &item : items) { if (existing.contains(item.id())) { continue; } tagIds << tag.id(); appendIds << item.id(); appendItems << item; } if (appendItems.isEmpty()) { return true; // all items have the desired tags already } QueryBuilder qb2(PimItemTagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemTagRelation::leftColumn(), appendIds); qb2.setColumnValue(PimItemTagRelation::rightColumn(), tagIds); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to append tag" << tag << "to Items" << appendItems; return false; } if (!silent) { notificationCollector()->itemsTagsChanged(appendItems, {tag.id()}, {}, col); } return true; } bool DataStore::appendItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool checkIfExists, const Collection &col, bool silent) { QVariantList itemsIds; itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds.append(item.id()); } setBoolPtr(tagsChanged, false); for (const Tag &tag : tags) { QSet existing; if (checkIfExists) { QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Select); Query::Condition cond; cond.addValueCondition(PimItemTagRelation::rightColumn(), Query::Equals, tag.id()); cond.addValueCondition(PimItemTagRelation::leftColumn(), Query::In, itemsIds); qb.addColumn(PimItemTagRelation::leftColumn()); qb.addCondition(cond); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to retrieve existing tag" << tag << "for Items" << itemsIds; return false; } QSqlQuery query = qb.query(); if (query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (query.size() == items.count()) { continue; } setBoolPtr(tagsChanged, true); } while (query.next()) { existing << query.value(0).value(); } if (!query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (existing.size() != items.count()) { setBoolPtr(tagsChanged, true); } } query.finish(); } if (!doAppendItemsTag(items, tag, existing, col, silent)) { return false; } } return true; } bool DataStore::removeItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { QSet removedTags; QVariantList itemsIds; QVariantList tagsIds; setBoolPtr(tagsChanged, false); itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds << item.id(); for (int i = 0; i < tags.count(); ++i) { const qint64 tagId = tags[i].id(); if (!removedTags.contains(tagId)) { tagsIds << tagId; removedTags << tagId; } } } // Delete all given tags from all given items in one go QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Delete); Query::Condition cond(Query::And); cond.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::In, tagsIds); cond.addValueCondition(PimItemTagRelation::leftFullColumnName(), Query::In, itemsIds); qb.addCondition(cond); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << tagsIds << "from Items" << itemsIds; return false; } if (qb.query().numRowsAffected() != 0) { setBoolPtr(tagsChanged, true); if (!silent) { notificationCollector()->itemsTagsChanged(items, QSet(), removedTags); } } return true; } bool DataStore::removeTags(const Tag::List &tags, bool silent) { // Currently the "silent" argument is only for API symmetry Q_UNUSED(silent); QVariantList removedTagsIds; QSet removedTags; removedTagsIds.reserve(tags.count()); removedTags.reserve(tags.count()); for (const Tag &tag : tags) { removedTagsIds << tag.id(); removedTags << tag.id(); } // Get all PIM items that we will untag SelectQueryBuilder itemsQuery; itemsQuery.addJoin(QueryBuilder::LeftJoin, PimItemTagRelation::tableName(), PimItemTagRelation::leftFullColumnName(), PimItem::idFullColumnName()); itemsQuery.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::In, removedTagsIds); if (!itemsQuery.exec()) { qCWarning(AKONADISERVER_LOG) << "Removing tags failed: failed to query Items for given tags" << removedTagsIds; return false; } const PimItem::List items = itemsQuery.result(); if (!items.isEmpty()) { notificationCollector()->itemsTagsChanged(items, QSet(), removedTags); } Q_FOREACH (const Tag &tag, tags) { // Emit special tagRemoved notification for each resource that owns the tag QueryBuilder qb(TagRemoteIdResourceRelation::tableName(), QueryBuilder::Select); qb.addColumn(TagRemoteIdResourceRelation::remoteIdFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), TagRemoteIdResourceRelation::resourceIdFullColumnName(), Resource::idFullColumnName()); qb.addColumn(Resource::nameFullColumnName()); qb.addValueCondition(TagRemoteIdResourceRelation::tagIdFullColumnName(), Query::Equals, tag.id()); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Removing tags failed: failed to retrieve RIDs for tag" << tag.id(); return false; } // Emit specialized notifications for each resource QSqlQuery query = qb.query(); while (query.next()) { const QString rid = query.value(0).toString(); const QByteArray resource = query.value(1).toByteArray(); notificationCollector()->tagRemoved(tag, resource, rid); } query.finish(); // And one for clients - without RID notificationCollector()->tagRemoved(tag, QByteArray(), QString()); } // Just remove the tags, table constraints will take care of the rest QueryBuilder qb(Tag::tableName(), QueryBuilder::Delete); qb.addValueCondition(Tag::idColumn(), Query::In, removedTagsIds); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << removedTagsIds; return false; } return true; } /* --- ItemParts ----------------------------------------------------- */ bool DataStore::removeItemParts(const PimItem &item, const QSet &parts) { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName()); qb.addValueCondition(Part::pimItemIdFullColumnName(), Query::Equals, item.id()); qb.addCondition(PartTypeHelper::conditionFromFqNames(parts)); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Removing item parts failed: failed to query parts" << parts << "from Item " << item.id(); return false; } const Part::List existingParts = qb.result(); for (Part part : qAsConst(existingParts)) { //krazy:exclude=foreach if (!PartHelper::remove(&part)) { qCWarning(AKONADISERVER_LOG) << "Failed to remove part" << part.id() << "(" << part.partType().ns() << ":" << part.partType().name() << ") from Item" << item.id(); return false; } } notificationCollector()->itemChanged(item, parts); return true; } bool DataStore::invalidateItemCache(const PimItem &item) { // find all payload item parts SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName()); qb.addValueCondition(Part::pimItemIdFullColumnName(), Query::Equals, item.id()); qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant()); qb.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QLatin1String("PLD")); qb.addValueCondition(PimItem::dirtyFullColumnName(), Query::Equals, false); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to invalidate cache for Item" << item.id(); return false; } const Part::List parts = qb.result(); // clear data field for (Part part : parts) { if (!PartHelper::truncate(part)) { qCWarning(AKONADISERVER_LOG) << "Failed to truncate payload part" << part.id() << "(" << part.partType().ns() << ":" << part.partType().name() << ") of Item" << item.id(); return false; } } return true; } /* --- Collection ------------------------------------------------------ */ bool DataStore::appendCollection(Collection &collection, const QStringList &mimeTypes, const QMap &attributes) { // no need to check for already existing collection with the same name, // a unique index on parent + name prevents that in the database if (!collection.insert()) { qCWarning(AKONADISERVER_LOG) << "Failed to append Collection" << collection.name() << "in resource" << collection.resource().name(); return false; } if (!appendMimeTypeForCollection(collection.id(), mimeTypes)) { qCWarning(AKONADISERVER_LOG) << "Failed to append mimetypes" << mimeTypes << "to new collection" << collection.name() << "(ID" << collection.id() << ") in resource" << collection.resource().name(); return false; } for (auto it = attributes.cbegin(), end = attributes.cend(); it != end; ++it) { if (!addCollectionAttribute(collection, it.key(), it.value(), true)) { qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << it.key() << "to new collection" << collection.name() << "(ID" << collection.id() << ") in resource" << collection.resource().name(); return false; } } notificationCollector()->collectionAdded(collection); return true; } bool DataStore::cleanupCollection(Collection &collection) { if (!s_hasForeignKeyConstraints) { return cleanupCollection_slow(collection); } // db will do most of the work for us, we just deal with notifications and external payload parts here Q_ASSERT(s_hasForeignKeyConstraints); // collect item deletion notifications const PimItem::List items = collection.items(); const QByteArray resource = collection.resource().name().toLatin1(); // generate the notification before actually removing the data // TODO: we should try to get rid of this, requires client side changes to resources and Monitor though notificationCollector()->itemsRemoved(items, collection, resource); // remove all external payload parts QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName()); qb.addValueCondition(Collection::idFullColumnName(), Query::Equals, collection.id()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External); qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant()); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to cleanup collection" << collection.name() << "(ID" << collection.id() << "):" << "Failed to query existing payload parts"; return false; } try { while (qb.query().next()) { ExternalPartStorage::self()->removePartFile( ExternalPartStorage::resolveAbsolutePath(qb.query().value(0).toByteArray())); } } catch (const PartHelperException &e) { qb.query().finish(); qCWarning(AKONADISERVER_LOG) << "PartHelperException while cleaning up collection" << collection.name() << "(ID" << collection.id() << "):" << e.what(); return false; } qb.query().finish(); // delete the collection itself, referential actions will do the rest notificationCollector()->collectionRemoved(collection); return collection.remove(); } bool DataStore::cleanupCollection_slow(Collection &collection) { Q_ASSERT(!s_hasForeignKeyConstraints); // delete the content const PimItem::List items = collection.items(); const QByteArray resource = collection.resource().name().toLatin1(); notificationCollector()->itemsRemoved(items, collection, resource); for (const PimItem &item : items) { if (!item.clearFlags()) { // TODO: move out of loop and use only a single query qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" << "failed: error clearing items flags"; return false; } if (!PartHelper::remove(Part::pimItemIdColumn(), item.id())) { // TODO: reduce to single query qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" << "failed: error clearing item payload parts"; return false; } if (!PimItem::remove(PimItem::idColumn(), item.id())) { // TODO: move into single querya qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" << "failed: error clearing items"; return false; } if (!Entity::clearRelation(item.id(), Entity::Right)) { // TODO: move into single query qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" << "failed: error clearing linked items"; return false; } } // delete collection mimetypes collection.clearMimeTypes(); Collection::clearPimItems(collection.id()); // delete attributes Q_FOREACH (CollectionAttribute attr, collection.attributes()) { //krazy:exclude=foreach if (!attr.remove()) { qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() << ")" << "failed: error clearing attribute" << attr.type(); return false; } } // delete the collection itself notificationCollector()->collectionRemoved(collection); return collection.remove(); } static bool recursiveSetResourceId(const Collection &collection, qint64 resourceId) { Transaction transaction(DataStore::self(), QStringLiteral("RECURSIVE SET RESOURCEID")); QueryBuilder qb(Collection::tableName(), QueryBuilder::Update); qb.addValueCondition(Collection::parentIdColumn(), Query::Equals, collection.id()); qb.setColumnValue(Collection::resourceIdColumn(), resourceId); qb.setColumnValue(Collection::remoteIdColumn(), QVariant()); qb.setColumnValue(Collection::remoteRevisionColumn(), QVariant()); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to set resource ID" << resourceId << "to collection" << collection.name() << "(ID" << collection.id() << ")"; return false; } // this is a cross-resource move, so also reset any resource-specific data (RID, RREV, etc) // as well as mark the items dirty to prevent cache purging before they have been written back qb = QueryBuilder(PimItem::tableName(), QueryBuilder::Update); qb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, collection.id()); qb.setColumnValue(PimItem::remoteIdColumn(), QVariant()); qb.setColumnValue(PimItem::remoteRevisionColumn(), QVariant()); const QDateTime now = QDateTime::currentDateTimeUtc(); qb.setColumnValue(PimItem::datetimeColumn(), now); qb.setColumnValue(PimItem::atimeColumn(), now); qb.setColumnValue(PimItem::dirtyColumn(), true); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed reset RID/RREV for PimItems in Collection" << collection.name() << "(ID" << collection.id() << ")"; return false; } transaction.commit(); Q_FOREACH (const Collection &col, collection.children()) { if (!recursiveSetResourceId(col, resourceId)) { return false; } } return true; } bool DataStore::moveCollection(Collection &collection, const Collection &newParent) { if (collection.parentId() == newParent.id()) { return true; } if (!m_dbOpened) { return false; } if (!newParent.isValid()) { qCWarning(AKONADISERVER_LOG) << "Failed to move collection" << collection.name() << "(ID" << collection.id() << "): invalid destination"; return false; } const QByteArray oldResource = collection.resource().name().toLatin1(); int resourceId = collection.resourceId(); const Collection source = collection.parent(); if (newParent.id() > 0) { // not root resourceId = newParent.resourceId(); } if (!CollectionQueryHelper::canBeMovedTo(collection, newParent)) { return false; } collection.setParentId(newParent.id()); if (collection.resourceId() != resourceId) { collection.setResourceId(resourceId); collection.setRemoteId(QString()); collection.setRemoteRevision(QString()); if (!recursiveSetResourceId(collection, resourceId)) { return false; } } if (!collection.update()) { qCWarning(AKONADISERVER_LOG) << "Failed to move Collection" << collection.name() << "(ID" << collection.id() << ")" << "into Collection" << collection.name() << "(ID" << collection.id() << ")"; return false; } notificationCollector()->collectionMoved(collection, source, oldResource, newParent.resource().name().toLatin1()); return true; } bool DataStore::appendMimeTypeForCollection(qint64 collectionId, const QStringList &mimeTypes) { if (mimeTypes.isEmpty()) { return true; } for (const QString &mimeType : mimeTypes) { const auto &mt = MimeType::retrieveByNameOrCreate(mimeType); if (!mt.isValid()) { return false; } if (!Collection::addMimeType(collectionId, mt.id())) { qCWarning(AKONADISERVER_LOG) << "Failed to append mimetype" << mt.name() << "to Collection" << collectionId; return false; } } return true; } void DataStore::activeCachePolicy(Collection &col) { if (!col.cachePolicyInherit()) { return; } Collection parent = col; while (parent.parentId() != 0) { parent = parent.parent(); if (!parent.cachePolicyInherit()) { col.setCachePolicyCheckInterval(parent.cachePolicyCheckInterval()); col.setCachePolicyCacheTimeout(parent.cachePolicyCacheTimeout()); col.setCachePolicySyncOnDemand(parent.cachePolicySyncOnDemand()); col.setCachePolicyLocalParts(parent.cachePolicyLocalParts()); return; } } // ### system default col.setCachePolicyCheckInterval(-1); col.setCachePolicyCacheTimeout(-1); col.setCachePolicySyncOnDemand(false); col.setCachePolicyLocalParts(QStringLiteral("ALL")); } QVector DataStore::virtualCollections(const PimItem &item) { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), Collection::idFullColumnName(), CollectionPimItemRelation::leftFullColumnName()); qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::Equals, item.id()); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to query virtual collections which PimItem" << item.id() << "belongs into"; return QVector(); } return qb.result(); } QMap > DataStore::virtualCollections(const PimItem::List &items) { QueryBuilder qb(CollectionPimItemRelation::tableName(), QueryBuilder::Select); qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), Collection::idFullColumnName(), CollectionPimItemRelation::leftFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), PimItem::idFullColumnName(), CollectionPimItemRelation::rightFullColumnName()); qb.addColumn(Collection::idFullColumnName()); qb.addColumns(QStringList() << PimItem::idFullColumnName() << PimItem::remoteIdFullColumnName() << PimItem::remoteRevisionFullColumnName() << PimItem::mimeTypeIdFullColumnName()); qb.addSortColumn(Collection::idFullColumnName(), Query::Ascending); if (items.count() == 1) { qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::Equals, items.first().id()); } else { QVariantList ids; ids.reserve(items.count()); for (const PimItem &item : items) { ids << item.id(); } qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::In, ids); } if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to query virtual Collections which PimItems" << items << "belong into"; return QMap >(); } QSqlQuery query = qb.query(); QMap > map; query.next(); while (query.isValid()) { const qlonglong collectionId = query.value(0).toLongLong(); QList &pimItems = map[collectionId]; do { PimItem item; item.setId(query.value(1).toLongLong()); item.setRemoteId(query.value(2).toString()); item.setRemoteRevision(query.value(3).toString()); item.setMimeTypeId(query.value(4).toLongLong()); pimItems << item; } while (query.next() && query.value(0).toLongLong() == collectionId); } query.finish(); return map; } /* --- PimItem ------------------------------------------------------- */ bool DataStore::appendPimItem(QVector &parts, const QVector &flags, const MimeType &mimetype, const Collection &collection, const QDateTime &dateTime, const QString &remote_id, const QString &remoteRevision, const QString &gid, PimItem &pimItem) { pimItem.setMimeTypeId(mimetype.id()); pimItem.setCollectionId(collection.id()); if (dateTime.isValid()) { pimItem.setDatetime(dateTime); } if (remote_id.isEmpty()) { // from application pimItem.setDirty(true); } else { // from resource pimItem.setRemoteId(remote_id); pimItem.setDirty(false); } pimItem.setRemoteRevision(remoteRevision); pimItem.setGid(gid); pimItem.setAtime(QDateTime::currentDateTimeUtc()); if (!pimItem.insert()) { qCWarning(AKONADISERVER_LOG) << "Failed to append new PimItem into Collection" << collection.name() << "(ID" << collection.id() << ")"; return false; } // insert every part if (!parts.isEmpty()) { //don't use foreach, the caller depends on knowing the part has changed, see the Append handler for (QVector::iterator it = parts.begin(); it != parts.end(); ++it) { (*it).setPimItemId(pimItem.id()); if ((*it).datasize() < (*it).data().size()) { (*it).setDatasize((*it).data().size()); } // qCDebug(AKONADISERVER_LOG) << "Insert from DataStore::appendPimItem"; if (!PartHelper::insert(&(*it))) { qCWarning(AKONADISERVER_LOG) << "Failed to add part" << it->partType().name() << "to new PimItem" << pimItem.id(); return false; } } } bool seen = false; Q_FOREACH (const Flag &flag, flags) { seen |= (flag.name() == QLatin1String(AKONADI_FLAG_SEEN) || flag.name() == QLatin1String(AKONADI_FLAG_IGNORED)); if (!pimItem.addFlag(flag)) { qCWarning(AKONADISERVER_LOG) << "Failed to add flag" << flag.name() << "to new PimItem" << pimItem.id(); return false; } } // qCDebug(AKONADISERVER_LOG) << "appendPimItem: " << pimItem; notificationCollector()->itemAdded(pimItem, seen, collection); return true; } bool DataStore::unhidePimItem(PimItem &pimItem) { if (!m_dbOpened) { return false; } qCDebug(AKONADISERVER_LOG) << "DataStore::unhidePimItem(" << pimItem << ")"; // FIXME: This is inefficient. Using a bit on the PimItemTable record would probably be some orders of magnitude faster... return removeItemParts(pimItem, { AKONADI_ATTRIBUTE_HIDDEN }); } bool DataStore::unhideAllPimItems() { if (!m_dbOpened) { return false; } qCDebug(AKONADISERVER_LOG) << "DataStore::unhideAllPimItems()"; try { return PartHelper::remove(Part::partTypeIdFullColumnName(), PartTypeHelper::fromFqName(QStringLiteral("ATR"), QStringLiteral("HIDDEN")).id()); } catch (...) { } // we can live with this failing return false; } -bool DataStore::cleanupPimItems(const PimItem::List &items) +bool DataStore::cleanupPimItems(const PimItem::List &items, bool silent) { // generate relation removed notifications - for (const PimItem &item : items) { - SelectQueryBuilder relationQuery; - relationQuery.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, item.id()); - relationQuery.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, item.id()); - relationQuery.setSubQueryMode(Query::Or); + if (!silent) { + for (const PimItem &item : items) { + SelectQueryBuilder relationQuery; + relationQuery.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, item.id()); + relationQuery.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, item.id()); + relationQuery.setSubQueryMode(Query::Or); - if (!relationQuery.exec()) { - throw HandlerException("Failed to obtain relations"); - } - const Relation::List relations = relationQuery.result(); - for (const Relation &relation : relations) { - notificationCollector()->relationRemoved(relation); + if (!relationQuery.exec()) { + throw HandlerException("Failed to obtain relations"); + } + const Relation::List relations = relationQuery.result(); + for (const Relation &relation : relations) { + notificationCollector()->relationRemoved(relation); + } } - } - // generate the notification before actually removing the data - notificationCollector()->itemsRemoved(items); + // generate the notification before actually removing the data + notificationCollector()->itemsRemoved(items); + } // FIXME: Create a single query to do this - Q_FOREACH (const PimItem &item, items) { + for (const auto &item : items) { if (!item.clearFlags()) { qCWarning(AKONADISERVER_LOG) << "Failed to clean up flags from PimItem" << item.id(); return false; } if (!PartHelper::remove(Part::pimItemIdColumn(), item.id())) { qCWarning(AKONADISERVER_LOG) << "Failed to clean up parts from PimItem" << item.id(); return false; } if (!PimItem::remove(PimItem::idColumn(), item.id())) { qCWarning(AKONADISERVER_LOG) << "Failed to remove PimItem" << item.id(); return false; } if (!Entity::clearRelation(item.id(), Entity::Right)) { qCWarning(AKONADISERVER_LOG) << "Failed to remove PimItem" << item.id() << "from linked collections"; return false; } } return true; } bool DataStore::addCollectionAttribute(const Collection &col, const QByteArray &key, const QByteArray &value, bool silent) { SelectQueryBuilder qb; qb.addValueCondition(CollectionAttribute::collectionIdColumn(), Query::Equals, col.id()); qb.addValueCondition(CollectionAttribute::typeColumn(), Query::Equals, key); if (!qb.exec()) { qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() << "(ID" << col.id() << "): Failed to query existing attribute"; return false; } if (!qb.result().isEmpty()) { qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() << "(ID" << col.id() << "): Attribute already exists"; return false; } CollectionAttribute attr; attr.setCollectionId(col.id()); attr.setType(key); attr.setValue(value); if (!attr.insert()) { qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() << "(ID" << col.id() << ")"; return false; } if (!silent) { notificationCollector()->collectionChanged(col, QList() << key); } return true; } bool DataStore::removeCollectionAttribute(const Collection &col, const QByteArray &key) { SelectQueryBuilder qb; qb.addValueCondition(CollectionAttribute::collectionIdColumn(), Query::Equals, col.id()); qb.addValueCondition(CollectionAttribute::typeColumn(), Query::Equals, key); if (!qb.exec()) { throw HandlerException("Unable to query for collection attribute"); } const QVector result = qb.result(); for (CollectionAttribute attr : result) { if (!attr.remove()) { throw HandlerException("Unable to remove collection attribute"); } } if (!result.isEmpty()) { notificationCollector()->collectionChanged(col, QList() << key); return true; } return false; } void DataStore::debugLastDbError(const char *actionDescription) const { qCCritical(AKONADISERVER_LOG) << "Database error:" << actionDescription; qCCritical(AKONADISERVER_LOG) << " Last driver error:" << m_database.lastError().driverText(); qCCritical(AKONADISERVER_LOG) << " Last database error:" << m_database.lastError().databaseText(); Tracer::self()->error("DataStore (Database Error)", QStringLiteral("%1\nDriver said: %2\nDatabase said:%3") .arg(QString::fromLatin1(actionDescription), m_database.lastError().driverText(), m_database.lastError().databaseText())); } void DataStore::debugLastQueryError(const QSqlQuery &query, const char *actionDescription) const { qCCritical(AKONADISERVER_LOG) << "Query error:" << actionDescription; qCCritical(AKONADISERVER_LOG) << " Last error message:" << query.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Last driver error:" << m_database.lastError().driverText(); qCCritical(AKONADISERVER_LOG) << " Last database error:" << m_database.lastError().databaseText(); Tracer::self()->error("DataStore (Database Query Error)", QStringLiteral("%1: %2") .arg(QString::fromLatin1(actionDescription), query.lastError().text())); } // static QString DataStore::dateTimeFromQDateTime(const QDateTime &dateTime) { QDateTime utcDateTime = dateTime; if (utcDateTime.timeSpec() != Qt::UTC) { utcDateTime.toUTC(); } return utcDateTime.toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")); } // static QDateTime DataStore::dateTimeToQDateTime(const QByteArray &dateTime) { return QDateTime::fromString(QString::fromLatin1(dateTime), QStringLiteral("yyyy-MM-dd hh:mm:ss")); } bool DataStore::doRollback() { QSqlDriver *driver = m_database.driver(); QElapsedTimer timer; timer.start(); driver->rollbackTransaction(); StorageDebugger::instance()->removeTransaction(reinterpret_cast(this), false, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { TRANSACTION_MUTEX_UNLOCK; debugLastDbError("DataStore::rollbackTransaction"); return false; } TRANSACTION_MUTEX_UNLOCK; return true; } void DataStore::transactionKilledByDB() { m_transactionKilledByDB = true; cleanupAfterRollback(); Q_EMIT transactionRolledBack(); } bool DataStore::beginTransaction(const QString &name) { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0 || m_transactionKilledByDB) { m_transactionKilledByDB = false; QElapsedTimer timer; timer.start(); TRANSACTION_MUTEX_LOCK; if (DbType::type(m_database) == DbType::Sqlite) { m_database.exec(QStringLiteral("BEGIN IMMEDIATE TRANSACTION")); StorageDebugger::instance()->addTransaction(reinterpret_cast(this), name, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::beginTransaction (SQLITE)"); TRANSACTION_MUTEX_UNLOCK; return false; } } else { m_database.driver()->beginTransaction(); StorageDebugger::instance()->addTransaction(reinterpret_cast(this), name, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::beginTransaction"); TRANSACTION_MUTEX_UNLOCK; return false; } } if (DbType::type(m_database) == DbType::PostgreSQL) { // Make constraints check deferred in PostgreSQL. Allows for // INSERT INTO mimetypetable (name) VALUES ('foo') RETURNING id; // INSERT INTO collectionmimetyperelation (collection_id, mimetype_id) VALUES (x, y) // where "y" refers to the newly inserted mimetype m_database.exec(QStringLiteral("SET CONSTRAINTS ALL DEFERRED")); } } ++m_transactionLevel; return true; } bool DataStore::rollbackTransaction() { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0) { qCWarning(AKONADISERVER_LOG) << "DataStore::rollbackTransaction(): No transaction in progress!"; return false; } --m_transactionLevel; if (m_transactionLevel == 0 && !m_transactionKilledByDB) { doRollback(); cleanupAfterRollback(); Q_EMIT transactionRolledBack(); } return true; } bool DataStore::commitTransaction() { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0) { qCWarning(AKONADISERVER_LOG) << "DataStore::commitTransaction(): No transaction in progress!"; return false; } if (m_transactionLevel == 1) { if (m_transactionKilledByDB) { qCWarning(AKONADISERVER_LOG) << "DataStore::commitTransaction(): Cannot commit, transaction was killed by mysql deadlock handling!"; return false; } QSqlDriver *driver = m_database.driver(); QElapsedTimer timer; timer.start(); driver->commitTransaction(); StorageDebugger::instance()->removeTransaction(reinterpret_cast(this), true, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::commitTransaction"); rollbackTransaction(); return false; } else { TRANSACTION_MUTEX_UNLOCK; m_transactionLevel--; Q_EMIT transactionCommitted(); } } else { m_transactionLevel--; } return true; } bool DataStore::inTransaction() const { return m_transactionLevel > 0; } void DataStore::sendKeepAliveQuery() { if (m_database.isOpen()) { QSqlQuery query(m_database); query.exec(QStringLiteral("SELECT 1")); } } void DataStore::cleanupAfterRollback() { MimeType::invalidateCompleteCache(); Flag::invalidateCompleteCache(); Resource::invalidateCompleteCache(); Collection::invalidateCompleteCache(); PartType::invalidateCompleteCache(); CollectionStatistics::self()->expireCache(); QueryCache::clear(); } diff --git a/src/server/storage/datastore.h b/src/server/storage/datastore.h index f7a2e96c8..d27bdcf23 100644 --- a/src/server/storage/datastore.h +++ b/src/server/storage/datastore.h @@ -1,366 +1,368 @@ /*************************************************************************** * Copyright (C) 2006 by Andreas Gungl * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #ifndef DATASTORE_H #define DATASTORE_H #include #include #include #include #include class QSqlQuery; class QTimer; #include "entities.h" #include "notificationcollector.h" #include namespace Akonadi { namespace Server { class DataStore; class DataStoreFactory { public: virtual ~DataStoreFactory() = default; virtual DataStore *createStore(); }; class NotificationCollector; /** This class handles all the database access.

Database configuration

You can select between various database backends during runtime using the @c $HOME/.config/akonadi/akonadiserverrc configuration file. Example: @verbatim [%General] Driver=QMYSQL [QMYSQL_EMBEDDED] Name=akonadi Options=SERVER_DATADIR=/home/foo/.local/share/akonadi/db_data [QMYSQL] Name=akonadi Host=localhost User=foo Password=***** #Options=UNIX_SOCKET=/home/foo/.local/share/akonadi/socket-bar/mysql.socket StartServer=true ServerPath=/usr/sbin/mysqld [QSQLITE] Name=/home/foo/.local/share/akonadi/akonadi.db @endverbatim Use @c General/Driver to select the QSql driver to use for database access. The following drivers are currently supported, other might work but are untested: - QMYSQL - QMYSQL_EMBEDDED - QSQLITE The options for each driver are read from the corresponding group. The following options are supported, dependent on the driver not all of them might have an effect: - Name: Database name, for sqlite that's the file name of the database. - Host: Hostname of the database server - User: Username for the database server - Password: Password for the database server - Options: Additional options, format is driver-dependent - StartServer: Start the database locally just for Akonadi instead of using an existing one - ServerPath: Path to the server executable */ class DataStore : public QObject { Q_OBJECT public: + const constexpr static bool Silent = true; + /** Closes the database connection and destroys the DataStore object. */ ~DataStore() override; /** Opens the database connection. */ virtual void open(); /** Closes the database connection. */ void close(); /** Initializes the database. Should be called during startup by the main thread. */ virtual bool init(); /** Per thread singleton. */ static DataStore *self(); /** * Returns whether per thread DataStore has been created. */ static bool hasDataStore(); /* --- ItemFlags ----------------------------------------------------- */ virtual bool setItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged = nullptr, const Collection &col = Collection(), bool silent = false); virtual bool appendItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged = nullptr, bool checkIfExists = true, const Collection &col = Collection(), bool silent = false); virtual bool removeItemsFlags(const PimItem::List &items, const QVector &flags, bool *tagsChanged = nullptr, const Collection &collection = Collection(), bool silent = false); /* --- ItemTags ----------------------------------------------------- */ virtual bool setItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool silent = false); virtual bool appendItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool checkIfExists = true, const Collection &col = Collection(), bool silent = false); virtual bool removeItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged = nullptr, bool silent = false); virtual bool removeTags(const Tag::List &tags, bool silent = false); /* --- ItemParts ----------------------------------------------------- */ virtual bool removeItemParts(const PimItem &item, const QSet &parts); // removes all payload parts for this item. virtual bool invalidateItemCache(const PimItem &item); /* --- Collection ------------------------------------------------------ */ virtual bool appendCollection(Collection &collection, const QStringList &mimeTypes, const QMap &attributes); /// removes the given collection and all its content virtual bool cleanupCollection(Collection &collection); /// same as the above but for database backends without working referential actions on foreign keys virtual bool cleanupCollection_slow(Collection &collection); /// moves the collection @p collection to @p newParent. virtual bool moveCollection(Collection &collection, const Collection &newParent); virtual bool appendMimeTypeForCollection(qint64 collectionId, const QStringList &mimeTypes); static QString collectionDelimiter() { return QStringLiteral("/"); } /** Determines the active cache policy for this Collection. The active cache policy is set in the corresponding Collection fields. */ virtual void activeCachePolicy(Collection &col); /// Returns all virtual collections the @p item is linked to QVector virtualCollections(const PimItem &item); QMap< Server::Entity::Id, QList< PimItem > > virtualCollections(const Akonadi::Server::PimItem::List &items); /* --- PimItem ------------------------------------------------------- */ virtual bool appendPimItem(QVector &parts, const QVector &flags, const MimeType &mimetype, const Collection &collection, const QDateTime &dateTime, const QString &remote_id, const QString &remoteRevision, const QString &gid, PimItem &pimItem); /** * Removes the pim item and all referenced data ( e.g. flags ) */ - virtual bool cleanupPimItems(const PimItem::List &items); + virtual bool cleanupPimItems(const PimItem::List &items, bool silent = false); /** * Unhides the specified PimItem. Emits the itemAdded() notification as * the hidden flag is assumed to have been set by appendPimItem() before * pushing the item to the preprocessor chain. The hidden item had his * notifications disabled until now (so for the clients the "unhide" operation * is actually a new item arrival). * * This function does NOT verify if the item was *really* hidden: this is * responsibility of the caller. */ virtual bool unhidePimItem(PimItem &pimItem); /** * Unhides all the items which have the "hidden" flag set. * This function doesn't emit any notification about the items * being unhidden so it's meant to be called only in rare circumstances. * The most notable call to this function is at server startup * when we attempt to restore a clean state of the database. */ virtual bool unhideAllPimItems(); /* --- Collection attributes ------------------------------------------ */ virtual bool addCollectionAttribute(const Collection &col, const QByteArray &key, const QByteArray &value, bool silent = false); /** * Removes the given collection attribute for @p col. * @throws HandlerException on database errors * @returns @c true if the attribute existed, @c false otherwise */ virtual bool removeCollectionAttribute(const Collection &col, const QByteArray &key); /* --- Helper functions ---------------------------------------------- */ /** Begins a transaction. No changes will be written to the database and no notification signal will be emitted unless you call commitTransaction(). @return @c true if successful. */ virtual bool beginTransaction(const QString &name); /** Reverts all changes within the current transaction. */ virtual bool rollbackTransaction(); /** Commits all changes within the current transaction and emits all collected notfication signals. If committing fails, the transaction will be rolled back. */ virtual bool commitTransaction(); /** Returns true if there is a transaction in progress. */ bool inTransaction() const; /** Returns the notification collector of this DataStore object. Use this to listen to change notification signals. */ NotificationCollector *notificationCollector(); /** Returns the QSqlDatabase object. Use this for generating queries yourself. Will [re-]open the database, if it is closed. */ QSqlDatabase database(); /** Sets the current session id. */ void setSessionId(const QByteArray &sessionId) { mSessionId = sessionId; } /** Returns if the database is currently open */ bool isOpened() const { return m_dbOpened; } bool doRollback(); void transactionKilledByDB(); Q_SIGNALS: /** Emitted if a transaction has been successfully committed. */ void transactionCommitted(); /** Emitted if a transaction has been aborted. */ void transactionRolledBack(); protected: /** Creates a new DataStore object and opens it. */ DataStore(); void debugLastDbError(const char *actionDescription) const; void debugLastQueryError(const QSqlQuery &query, const char *actionDescription) const; private: bool doAppendItemsFlag(const PimItem::List &items, const Flag &flag, const QSet &existing, const Collection &col, bool silent); bool doAppendItemsTag(const PimItem::List &items, const Tag &tag, const QSet &existing, const Collection &col, bool silent); /** Converts the given date/time to the database format, i.e. "YYYY-MM-DD HH:MM:SS". @param dateTime the date/time in UTC @return the date/time in database format @see dateTimeToQDateTime */ static QString dateTimeFromQDateTime(const QDateTime &dateTime); /** Converts the given date/time from database format to QDateTime. @param dateTime the date/time in database format @return the date/time as QDateTime @see dateTimeFromQDateTime */ static QDateTime dateTimeToQDateTime(const QByteArray &dateTime); private Q_SLOTS: void sendKeepAliveQuery(); protected: static std::unique_ptr sFactory; std::unique_ptr mNotificationCollector; private: void cleanupAfterRollback(); QString m_connectionName; QSqlDatabase m_database; bool m_dbOpened; bool m_transactionKilledByDB = false; uint m_transactionLevel; struct TransactionQuery { QString query; QVector boundValues; bool isBatch; }; QByteArray mSessionId; QTimer *m_keepAliveTimer = nullptr; static bool s_hasForeignKeyConstraints; friend class DataStoreFactory; }; } // namespace Server } // namespace Akonadi #endif