diff --git a/src/core/itemsync.cpp b/src/core/itemsync.cpp index 6558a6028..df1cc58e0 100644 --- a/src/core/itemsync.cpp +++ b/src/core/itemsync.cpp @@ -1,554 +1,554 @@ /* Copyright (c) 2007 Tobias Koenig Copyright (c) 2007 Volker Krause Copyright (c) 2014 Christian Mollekopf This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "itemsync.h" #include "job_p.h" #include "collection.h" #include "item.h" #include "item_p.h" #include "itemcreatejob.h" #include "itemdeletejob.h" #include "itemfetchjob.h" #include "itemmodifyjob.h" #include "transactionsequence.h" #include "itemfetchscope.h" #include "akonadicore_debug.h" using namespace Akonadi; /** * @internal */ class Akonadi::ItemSyncPrivate : public JobPrivate { public: ItemSyncPrivate(ItemSync *parent) : JobPrivate(parent) , mTransactionMode(ItemSync::SingleTransaction) , mCurrentTransaction(nullptr) , mTransactionJobs(0) , mPendingJobs(0) , mProgress(0) , mTotalItems(-1) , mTotalItemsProcessed(0) , mStreaming(false) , mIncremental(false) , mDeliveryDone(false) , mFinished(false) , mFullListingDone(false) , mProcessingBatch(false) , mDisableAutomaticDeliveryDone(false) , mBatchSize(10) , mMergeMode(Akonadi::ItemSync::RIDMerge) { // we want to fetch all data by default mFetchScope.fetchFullPayload(); mFetchScope.fetchAllAttributes(); } void createOrMerge(const Item &item); void checkDone(); void slotItemsReceived(const Item::List &items); void slotLocalListDone(KJob *job); void slotLocalDeleteDone(KJob *job); void slotLocalChangeDone(KJob *job); void execute(); void processItems(); void processBatch(); void deleteItems(const Item::List &items); void slotTransactionResult(KJob *job); void requestTransaction(); Job *subjobParent() const; void fetchLocalItemsToDelete(); QString jobDebuggingString() const override; bool allProcessed() const; Q_DECLARE_PUBLIC(ItemSync) Collection mSyncCollection; QSet mListedItems; ItemSync::TransactionMode mTransactionMode; TransactionSequence *mCurrentTransaction; int mTransactionJobs; // fetch scope for initial item listing ItemFetchScope mFetchScope; Akonadi::Item::List mRemoteItemQueue; Akonadi::Item::List mRemovedRemoteItemQueue; Akonadi::Item::List mCurrentBatchRemoteItems; Akonadi::Item::List mCurrentBatchRemovedRemoteItems; Akonadi::Item::List mItemsToDelete; // create counter int mPendingJobs; int mProgress; int mTotalItems; int mTotalItemsProcessed; bool mStreaming; bool mIncremental; bool mDeliveryDone; bool mFinished; bool mFullListingDone; bool mProcessingBatch; bool mDisableAutomaticDeliveryDone; int mBatchSize; Akonadi::ItemSync::MergeMode mMergeMode; }; void ItemSyncPrivate::createOrMerge(const Item &item) { Q_Q(ItemSync); // don't try to do anything in error state if (q->error()) { return; } mPendingJobs++; ItemCreateJob *create = new ItemCreateJob(item, mSyncCollection, subjobParent()); ItemCreateJob::MergeOptions merge = ItemCreateJob::Silent; if (mMergeMode == ItemSync::GIDMerge && !item.gid().isEmpty()) { merge |= ItemCreateJob::GID; } else { merge |= ItemCreateJob::RID; } create->setMerge(merge); q->connect(create, &ItemCreateJob::result, q, [this](KJob *job) {slotLocalChangeDone(job);}); } bool ItemSyncPrivate::allProcessed() const { return mDeliveryDone && mCurrentBatchRemoteItems.isEmpty() && mRemoteItemQueue.isEmpty() && mRemovedRemoteItemQueue.isEmpty() && mCurrentBatchRemovedRemoteItems.isEmpty(); } void ItemSyncPrivate::checkDone() { Q_Q(ItemSync); q->setProcessedAmount(KJob::Bytes, mProgress); if (mPendingJobs > 0) { return; } if (mTransactionJobs > 0) { //Commit the current transaction if we're in batch processing mode or done //and wait until the transaction is committed to process the next batch if (mTransactionMode == ItemSync::MultipleTransactions || (mDeliveryDone && mRemoteItemQueue.isEmpty())) { if (mCurrentTransaction) { q->emit transactionCommitted(); mCurrentTransaction->commit(); mCurrentTransaction = nullptr; } return; } } mProcessingBatch = false; if (!mRemoteItemQueue.isEmpty()) { execute(); //We don't have enough items, request more if (!mProcessingBatch) { q->emit readyForNextBatch(mBatchSize - mRemoteItemQueue.size()); } return; } q->emit readyForNextBatch(mBatchSize); if (allProcessed() && !mFinished) { // prevent double result emission, can happen since checkDone() is called from all over the place - qCDebug(AKONADICORE_LOG) << "finished"; + qCDebug(AKONADICORE_LOG) << "ItemSync of collection" << mSyncCollection.id() << "finished"; mFinished = true; q->emitResult(); } } ItemSync::ItemSync(const Collection &collection, QObject *parent) : Job(new ItemSyncPrivate(this), parent) { Q_D(ItemSync); d->mSyncCollection = collection; } ItemSync::~ItemSync() { } void ItemSync::setFullSyncItems(const Item::List &items) { /* * We received a list of items from the server: * * fetch all local id's + rid's only * * check each full sync item whether it's locally available * * if it is modify the item * * if it's not create it * * delete all superfluous items */ Q_D(ItemSync); Q_ASSERT(!d->mIncremental); if (!d->mStreaming) { d->mDeliveryDone = true; } d->mRemoteItemQueue += items; d->mTotalItemsProcessed += items.count(); qCDebug(AKONADICORE_LOG) << "Received: " << items.count() << "In total: " << d->mTotalItemsProcessed << " Wanted: " << d->mTotalItems; if (!d->mDisableAutomaticDeliveryDone && (d->mTotalItemsProcessed == d->mTotalItems)) { d->mDeliveryDone = true; } d->execute(); } void ItemSync::setTotalItems(int amount) { Q_D(ItemSync); Q_ASSERT(!d->mIncremental); Q_ASSERT(amount >= 0); setStreamingEnabled(true); qCDebug(AKONADICORE_LOG) << amount; d->mTotalItems = amount; setTotalAmount(KJob::Bytes, amount); if (!d->mDisableAutomaticDeliveryDone && (d->mTotalItems == 0)) { d->mDeliveryDone = true; d->execute(); } } void ItemSync::setDisableAutomaticDeliveryDone(bool disable) { Q_D(ItemSync); d->mDisableAutomaticDeliveryDone = disable; } void ItemSync::setIncrementalSyncItems(const Item::List &changedItems, const Item::List &removedItems) { /* * We received an incremental listing of items: * * for each changed item: * ** If locally available => modify * ** else => create * * removed items can be removed right away */ Q_D(ItemSync); d->mIncremental = true; if (!d->mStreaming) { d->mDeliveryDone = true; } d->mRemoteItemQueue += changedItems; d->mRemovedRemoteItemQueue += removedItems; d->mTotalItemsProcessed += changedItems.count() + removedItems.count(); qCDebug(AKONADICORE_LOG) << "Received: " << changedItems.count() << "Removed: " << removedItems.count() << "In total: " << d->mTotalItemsProcessed << " Wanted: " << d->mTotalItems; if (!d->mDisableAutomaticDeliveryDone && (d->mTotalItemsProcessed == d->mTotalItems)) { d->mDeliveryDone = true; } d->execute(); } void ItemSync::setFetchScope(ItemFetchScope &fetchScope) { Q_D(ItemSync); d->mFetchScope = fetchScope; } ItemFetchScope &ItemSync::fetchScope() { Q_D(ItemSync); return d->mFetchScope; } void ItemSync::doStart() { } void ItemSyncPrivate::fetchLocalItemsToDelete() { Q_Q(ItemSync); if (mIncremental) { qFatal("This must not be called while in incremental mode"); return; } ItemFetchJob *job = new ItemFetchJob(mSyncCollection, subjobParent()); job->fetchScope().setFetchRemoteIdentification(true); job->fetchScope().setFetchModificationTime(false); job->setDeliveryOption(ItemFetchJob::EmitItemsIndividually); // we only can fetch parts already in the cache, otherwise this will deadlock job->fetchScope().setCacheOnly(true); QObject::connect(job, &ItemFetchJob::itemsReceived, q, [this](const Akonadi::Item::List &lst) { slotItemsReceived(lst); }); QObject::connect(job, &ItemFetchJob::result, q, [this](KJob *job) { slotLocalListDone(job); }); mPendingJobs++; } void ItemSyncPrivate::slotItemsReceived(const Item::List &items) { for (const Akonadi::Item &item : items) { //Don't delete items that have not yet been synchronized if (item.remoteId().isEmpty()) { continue; } if (!mListedItems.contains(item.remoteId())) { mItemsToDelete << Item(item.id()); } } } void ItemSyncPrivate::slotLocalListDone(KJob *job) { mPendingJobs--; if (job->error()) { qCWarning(AKONADICORE_LOG) << job->errorString(); } deleteItems(mItemsToDelete); checkDone(); } QString ItemSyncPrivate::jobDebuggingString() const { // TODO: also print out mIncremental and mTotalItemsProcessed, but they are set after the job // started, so this requires passing jobDebuggingString to jobEnded(). return QStringLiteral("Collection %1 (%2)").arg(mSyncCollection.id()).arg(mSyncCollection.name()); } void ItemSyncPrivate::execute() { //shouldn't happen if (mFinished) { qCWarning(AKONADICORE_LOG) << "Call to execute() on finished job."; Q_ASSERT(false); return; } //not doing anything, start processing if (!mProcessingBatch) { if (mRemoteItemQueue.size() >= mBatchSize || mDeliveryDone) { //we have a new batch to process const int num = qMin(mBatchSize, mRemoteItemQueue.size()); mCurrentBatchRemoteItems.reserve(mBatchSize); std::move(mRemoteItemQueue.begin(), mRemoteItemQueue.begin() + num, std::back_inserter(mCurrentBatchRemoteItems)); mRemoteItemQueue.erase(mRemoteItemQueue.begin(), mRemoteItemQueue.begin() + num); mCurrentBatchRemovedRemoteItems += mRemovedRemoteItemQueue; mRemovedRemoteItemQueue.clear(); } else { //nothing to do, let's wait for more data return; } mProcessingBatch = true; processBatch(); return; } checkDone(); } //process the current batch of items void ItemSyncPrivate::processBatch() { if (mCurrentBatchRemoteItems.isEmpty() && !mDeliveryDone) { return; } //request a transaction, there are items that require processing requestTransaction(); processItems(); // removed if (!mIncremental && allProcessed()) { //the full listing is done and we know which items to remove fetchLocalItemsToDelete(); } else { deleteItems(mCurrentBatchRemovedRemoteItems); mCurrentBatchRemovedRemoteItems.clear(); } checkDone(); } void ItemSyncPrivate::processItems() { // added / updated for (const Item &remoteItem : qAsConst(mCurrentBatchRemoteItems)) { if (remoteItem.remoteId().isEmpty()) { qCWarning(AKONADICORE_LOG) << "Item " << remoteItem.id() << " does not have a remote identifier"; continue; } if (!mIncremental) { mListedItems << remoteItem.remoteId(); } createOrMerge(remoteItem); } mCurrentBatchRemoteItems.clear(); } void ItemSyncPrivate::deleteItems(const Item::List &itemsToDelete) { Q_Q(ItemSync); // if in error state, better not change anything anymore if (q->error()) { return; } if (itemsToDelete.isEmpty()) { return; } mPendingJobs++; ItemDeleteJob *job = new ItemDeleteJob(itemsToDelete, subjobParent()); q->connect(job, &ItemDeleteJob::result, q, [this](KJob *job) { slotLocalDeleteDone(job); }); // It can happen that the groupware servers report us deleted items // twice, in this case this item delete job will fail on the second try. // To avoid a rollback of the complete transaction we gracefully allow the job // to fail :) TransactionSequence *transaction = qobject_cast(subjobParent()); if (transaction) { transaction->setIgnoreJobFailure(job); } } void ItemSyncPrivate::slotLocalDeleteDone(KJob *job) { if (job->error()) { qCWarning(AKONADICORE_LOG) << "Deleting items from the akonadi database failed:" << job->errorString(); } mPendingJobs--; mProgress++; checkDone(); } void ItemSyncPrivate::slotLocalChangeDone(KJob *job) { if (job->error()) { qCWarning(AKONADICORE_LOG) << "Creating/updating items from the akonadi database failed:" << job->errorString(); } mPendingJobs--; mProgress++; checkDone(); } void ItemSyncPrivate::slotTransactionResult(KJob *job) { --mTransactionJobs; if (mCurrentTransaction == job) { mCurrentTransaction = nullptr; } checkDone(); } void ItemSyncPrivate::requestTransaction() { Q_Q(ItemSync); //we never want parallel transactions, single transaction just makes one big transaction, and multi transaction uses multiple transaction sequentially if (!mCurrentTransaction) { ++mTransactionJobs; mCurrentTransaction = new TransactionSequence(q); mCurrentTransaction->setAutomaticCommittingEnabled(false); QObject::connect(mCurrentTransaction, &TransactionSequence::result, q, [this](KJob *job) { slotTransactionResult(job); }); } } Job *ItemSyncPrivate::subjobParent() const { Q_Q(const ItemSync); if (mCurrentTransaction && mTransactionMode != ItemSync::NoTransaction) { return mCurrentTransaction; } return const_cast(q); } void ItemSync::setStreamingEnabled(bool enable) { Q_D(ItemSync); d->mStreaming = enable; } void ItemSync::deliveryDone() { Q_D(ItemSync); Q_ASSERT(d->mStreaming); d->mDeliveryDone = true; d->execute(); } void ItemSync::slotResult(KJob *job) { if (job->error()) { qCWarning(AKONADICORE_LOG) << "Error during ItemSync: " << job->errorString(); // pretend there were no errors Akonadi::Job::removeSubjob(job); // propagate the first error we got but continue, we might still be fed with stuff from a resource if (!error()) { setError(job->error()); setErrorText(job->errorText()); } } else { Akonadi::Job::slotResult(job); } } void ItemSync::rollback() { Q_D(ItemSync); qCDebug(AKONADICORE_LOG) << "The item sync is being rolled-back."; setError(UserCanceled); if (d->mCurrentTransaction) { d->mCurrentTransaction->rollback(); } d->mDeliveryDone = true; // user wont deliver more data d->execute(); // end this in an ordered way, since we have an error set no real change will be done } void ItemSync::setTransactionMode(ItemSync::TransactionMode mode) { Q_D(ItemSync); d->mTransactionMode = mode; } int ItemSync::batchSize() const { Q_D(const ItemSync); return d->mBatchSize; } void ItemSync::setBatchSize(int size) { Q_D(ItemSync); d->mBatchSize = size; } ItemSync::MergeMode ItemSync::mergeMode() const { Q_D(const ItemSync); return d->mMergeMode; } void ItemSync::setMergeMode(MergeMode mergeMode) { Q_D(ItemSync); d->mMergeMode = mergeMode; } #include "moc_itemsync.cpp" diff --git a/src/private/protocol_p.h b/src/private/protocol_p.h index e2dcb347d..8b8552706 100644 --- a/src/private/protocol_p.h +++ b/src/private/protocol_p.h @@ -1,644 +1,647 @@ /* Copyright (c) 2007 Volker Krause Copyright (c) 2015 Daniel Vrátil Copyright (c) 2016 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef AKONADI_PROTOCOL_COMMON_P_H #define AKONADI_PROTOCOL_COMMON_P_H #include "akonadiprivate_export.h" #include #include #include #include #include #include #include "tristate_p.h" #include "scope_p.h" /** @file protocol_p.h Shared constants used in the communication protocol between the Akonadi server and its clients. */ namespace Akonadi { namespace Protocol { class Factory; class DataStream; class Command; class Response; class ItemFetchScope; class ScopeContext; class ChangeNotification; using Attributes = QMap; } // namespace Protocol } // namespace Akonadi namespace Akonadi { namespace Protocol { AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::Command &cmd); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::Command &cmd); AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::Command &cmd); AKONADIPRIVATE_EXPORT void toJson(const Command *cmd, QJsonObject &json); using CommandPtr = QSharedPointer; class AKONADIPRIVATE_EXPORT Command { public: enum Type : quint8 { Invalid = 0, // Session management Hello = 1, Login, Logout, // Transactions Transaction = 10, // Items CreateItem = 20, CopyItems, DeleteItems, FetchItems, LinkItems, ModifyItems, MoveItems, // Collections CreateCollection = 40, CopyCollection, DeleteCollection, FetchCollections, FetchCollectionStats, ModifyCollection, MoveCollection, // Search Search = 60, SearchResult, StoreSearch, // Tag CreateTag = 70, DeleteTag, FetchTags, ModifyTag, // Relation FetchRelations = 80, ModifyRelation, RemoveRelations, // Resources SelectResource = 90, // Other StreamPayload = 100, // Notifications ItemChangeNotification = 110, CollectionChangeNotification, TagChangeNotification, RelationChangeNotification, SubscriptionChangeNotification, DebugChangeNotification, CreateSubscription, ModifySubscription, // _MaxValue = 127 _ResponseBit = 0x80 // reserved }; explicit Command() = default; explicit Command(const Command &) = default; Command(Command &&) = default; ~Command() = default; Command &operator=(const Command &) = default; Command &operator=(Command &&) = default; bool operator==(const Command &other) const; inline bool operator!=(const Command &other) const { return !operator==(other); } inline Type type() const { return static_cast(mType & ~_ResponseBit); } inline bool isValid() const { return type() != Invalid; } inline bool isResponse() const { return mType & _ResponseBit; } void toJson(QJsonObject &stream) const; protected: explicit Command(quint8 type); quint8 mType = Invalid; // unused 7 bytes private: friend class Factory; friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<(Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::Command &cmd); friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>(Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::Command &cmd); friend AKONADIPRIVATE_EXPORT QDebug operator<<(::QDebug dbg, const Akonadi::Protocol::Command &cmd); friend AKONADIPRIVATE_EXPORT void toJson(const Akonadi::Protocol::Command *cmd, QJsonObject &json); }; +AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, Command::Type type); + } // namespace Protocol } // namespace Akonadi Q_DECLARE_METATYPE(Akonadi::Protocol::Command::Type) Q_DECLARE_METATYPE(Akonadi::Protocol::CommandPtr) + namespace Akonadi { namespace Protocol { AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::Response &cmd); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::Response &cmd); AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::Response &response); using ResponsePtr = QSharedPointer; class AKONADIPRIVATE_EXPORT Response : public Command { public: explicit Response(); explicit Response(const Response &) = default; Response(Response &&) = default; Response &operator=(const Response &) = default; Response &operator=(Response &&) = default; inline void setError(int code, const QString &message) { mErrorCode = code; mErrorMsg = message; } bool operator==(const Response &other) const; inline bool operator!=(const Response &other) const { return !operator==(other); } inline bool isError() const { return mErrorCode > 0; } inline int errorCode() const { return mErrorCode; } inline QString errorMessage() const { return mErrorMsg; } void toJson(QJsonObject &json) const; protected: explicit Response(Command::Type type); int mErrorCode; QString mErrorMsg; private: friend class Factory; friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<(Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::Response &cmd); friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>(Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::Response &cmd); friend AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::Response &cmd); }; } // namespace Protocol } // namespace Akonadi namespace Akonadi { namespace Protocol { template inline const X &cmdCast(const QSharedPointer &p) { return static_cast(*p); } template inline X &cmdCast(QSharedPointer &p) { return static_cast(*p); } class AKONADIPRIVATE_EXPORT Factory { public: static CommandPtr command(Command::Type type); static ResponsePtr response(Command::Type type); private: template friend AKONADIPRIVATE_EXPORT CommandPtr deserialize(QIODevice *device); }; AKONADIPRIVATE_EXPORT void serialize(QIODevice *device, const CommandPtr &command); AKONADIPRIVATE_EXPORT CommandPtr deserialize(QIODevice *device); AKONADIPRIVATE_EXPORT QString debugString(const Command &command); AKONADIPRIVATE_EXPORT inline QString debugString(const CommandPtr &command) { return debugString(*command); } } // namespace Protocol } // namespace Akonadi namespace Akonadi { namespace Protocol { AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ItemFetchScope &scope); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ItemFetchScope &scope); AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ItemFetchScope &scope); class AKONADIPRIVATE_EXPORT ItemFetchScope { public: enum FetchFlag : int { None = 0, CacheOnly = 1 << 0, CheckCachedPayloadPartsOnly = 1 << 1, FullPayload = 1 << 2, AllAttributes = 1 << 3, Size = 1 << 4, MTime = 1 << 5, RemoteRevision = 1 << 6, IgnoreErrors = 1 << 7, Flags = 1 << 8, RemoteID = 1 << 9, GID = 1 << 10, Tags = 1 << 11, Relations = 1 << 12, VirtReferences = 1 << 13 }; Q_DECLARE_FLAGS(FetchFlags, FetchFlag) enum AncestorDepth : ushort { NoAncestor, ParentAncestor, AllAncestors }; explicit ItemFetchScope() = default; ItemFetchScope(const ItemFetchScope &) = default; ItemFetchScope(ItemFetchScope &&other) = default; ~ItemFetchScope() = default; ItemFetchScope &operator=(const ItemFetchScope &) = default; ItemFetchScope &operator=(ItemFetchScope &&) = default; bool operator==(const ItemFetchScope &other) const; inline bool operator!=(const ItemFetchScope &other) const { return !operator==(other); } inline void setRequestedParts(const QVector &requestedParts) { mRequestedParts = requestedParts; } inline QVector requestedParts() const { return mRequestedParts; } QVector requestedPayloads() const; inline void setChangedSince(const QDateTime &changedSince) { mChangedSince = changedSince; } inline QDateTime changedSince() const { return mChangedSince; } inline void setAncestorDepth(AncestorDepth depth) { mAncestorDepth = depth; } inline AncestorDepth ancestorDepth() const { return mAncestorDepth; } inline bool cacheOnly() const { return mFlags & CacheOnly; } inline bool checkCachedPayloadPartsOnly() const { return mFlags & CheckCachedPayloadPartsOnly; } inline bool fullPayload() const { return mFlags & FullPayload; } inline bool allAttributes() const { return mFlags & AllAttributes; } inline bool fetchSize() const { return mFlags & Size; } inline bool fetchMTime() const { return mFlags & MTime; } inline bool fetchRemoteRevision() const { return mFlags & RemoteRevision; } inline bool ignoreErrors() const { return mFlags & IgnoreErrors; } inline bool fetchFlags() const { return mFlags & Flags; } inline bool fetchRemoteId() const { return mFlags & RemoteID; } inline bool fetchGID() const { return mFlags & GID; } inline bool fetchTags() const { return mFlags & Tags; } inline bool fetchRelations() const { return mFlags & Relations; } inline bool fetchVirtualReferences() const { return mFlags & VirtReferences; } void setFetch(FetchFlags attributes, bool fetch = true); bool fetch(FetchFlags flags) const; void toJson(QJsonObject &json) const; private: AncestorDepth mAncestorDepth = NoAncestor; // 2 bytes free FetchFlags mFlags = None; QVector mRequestedParts; QDateTime mChangedSince; friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<(Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ItemFetchScope &scope); friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>(Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ItemFetchScope &scope); friend AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ItemFetchScope &scope); }; } // namespace Protocol } // namespace Akonadi Q_DECLARE_OPERATORS_FOR_FLAGS(Akonadi::Protocol::ItemFetchScope::FetchFlags) namespace Akonadi { namespace Protocol { AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ScopeContext &ctx); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ScopeContext &ctx); AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ScopeContext &ctx); class AKONADIPRIVATE_EXPORT ScopeContext { public: enum Type : uchar { Any = 0, Collection, Tag }; explicit ScopeContext() = default; ScopeContext(Type type, qint64 id); ScopeContext(Type type, const QString &id); ScopeContext(const ScopeContext &) = default; ScopeContext(ScopeContext &&) = default; ~ScopeContext() = default; ScopeContext &operator=(const ScopeContext &) = default; ScopeContext &operator=(ScopeContext &&) = default; bool operator==(const ScopeContext &other) const; inline bool operator!=(const ScopeContext &other) const { return !operator==(other); } inline bool isEmpty() const { return mColCtx.isNull() && mTagCtx.isNull(); } inline void setContext(Type type, qint64 id) { setCtx(type, id); } inline void setContext(Type type, const QString &id) { setCtx(type, id); } inline void clearContext(Type type) { setCtx(type, QVariant()); } inline bool hasContextId(Type type) const { return ctx(type).type() == QVariant::LongLong; } inline qint64 contextId(Type type) const { return hasContextId(type) ? ctx(type).toLongLong() : 0; } inline bool hasContextRID(Type type) const { return ctx(type).type() == QVariant::String; } inline QString contextRID(Type type) const { return hasContextRID(type) ? ctx(type).toString() : QString(); } void toJson(QJsonObject &json) const; private: QVariant mColCtx; QVariant mTagCtx; inline QVariant ctx(Type type) const { return type == Collection ? mColCtx : type == Tag ? mTagCtx : QVariant(); } inline void setCtx(Type type, const QVariant &v) { if (type == Collection) { mColCtx = v; } else if (type == Tag) { mTagCtx = v; } } friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<(Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ScopeContext &context); friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>(Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ScopeContext &context); friend AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ScopeContext &ctx); }; } // namespace Protocol } // namespace akonadi namespace Akonadi { namespace Protocol { class FetchItemsResponse; typedef QSharedPointer FetchItemsResponsePtr; AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ChangeNotification &ntf); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ChangeNotification &ntf); AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ChangeNotification &ntf); using ChangeNotificationPtr = QSharedPointer; using ChangeNotificationList = QVector; class AKONADIPRIVATE_EXPORT ChangeNotification : public Command { public: static QList itemsToUids(const QVector &items); class Relation { public: Relation() = default; Relation(const Relation &) = default; Relation(Relation &&) = default; inline Relation(qint64 leftId, qint64 rightId, const QString &type) : leftId(leftId) , rightId(rightId) , type(type) { } Relation &operator=(const Relation &) = default; Relation &operator=(Relation &&) = default; inline bool operator==(const Relation &other) const { return leftId == other.leftId && rightId == other.rightId && type == other.type; } void toJson(QJsonObject &json) const { json[QStringLiteral("leftId")] = leftId; json[QStringLiteral("rightId")] = rightId; json[QStringLiteral("type")] = type; } qint64 leftId = -1; qint64 rightId = -1; QString type; }; ChangeNotification &operator=(const ChangeNotification &) = default; ChangeNotification &operator=(ChangeNotification &&) = default; bool operator==(const ChangeNotification &other) const; inline bool operator!=(const ChangeNotification &other) const { return !operator==(other); } bool isRemove() const; bool isMove() const; inline QByteArray sessionId() const { return mSessionId; } inline void setSessionId(const QByteArray &sessionId) { mSessionId = sessionId; } inline void addMetadata(const QByteArray &metadata) { mMetaData << metadata; } inline void removeMetadata(const QByteArray &metadata) { mMetaData.removeAll(metadata); } QVector metadata() const { return mMetaData; } static bool appendAndCompress(ChangeNotificationList &list, const ChangeNotificationPtr &msg); void toJson(QJsonObject &json) const; protected: explicit ChangeNotification() = default; explicit ChangeNotification(Command::Type type); ChangeNotification(const ChangeNotification &) = default; ChangeNotification(ChangeNotification &&) = default; QByteArray mSessionId; // For internal use only: Akonadi server can add some additional information // that might be useful when evaluating the notification for example, but // it is never transferred to clients QVector mMetaData; friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<(Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ChangeNotification &ntf); friend AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>(Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ChangeNotification &ntf); friend AKONADIPRIVATE_EXPORT QDebug operator<<(QDebug dbg, const Akonadi::Protocol::ChangeNotification &ntf); }; inline uint qHash(const ChangeNotification::Relation &rel) { return ::qHash(rel.leftId + rel.rightId); } // TODO: Internalize? AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator<<( Akonadi::Protocol::DataStream &stream, const Akonadi::Protocol::ChangeNotification::Relation &relation); AKONADIPRIVATE_EXPORT Akonadi::Protocol::DataStream &operator>>( Akonadi::Protocol::DataStream &stream, Akonadi::Protocol::ChangeNotification::Relation &relation); } // namespace Protocol } // namespace Akonadi Q_DECLARE_METATYPE(Akonadi::Protocol::ChangeNotificationPtr) Q_DECLARE_METATYPE(Akonadi::Protocol::ChangeNotificationList) /******************************************************************************/ // Here comes the actual generated Protocol. See protocol.xml for definitions, // and genprotocol folder for the generator. #include "protocol_gen.h" /******************************************************************************/ // Command parameters #define AKONADI_PARAM_ATR "ATR:" #define AKONADI_PARAM_CACHEPOLICY "CACHEPOLICY" #define AKONADI_PARAM_DISPLAY "DISPLAY" #define AKONADI_PARAM_ENABLED "ENABLED" #define AKONADI_PARAM_FLAGS "FLAGS" #define AKONADI_PARAM_TAGS "TAGS" #define AKONADI_PARAM_GID "GID" #define AKONADI_PARAM_INDEX "INDEX" #define AKONADI_PARAM_MIMETYPE "MIMETYPE" #define AKONADI_PARAM_NAME "NAME" #define AKONADI_PARAM_PARENT "PARENT" #define AKONADI_PARAM_PERSISTENTSEARCH "PERSISTENTSEARCH" #define AKONADI_PARAM_PLD "PLD:" #define AKONADI_PARAM_PLD_RFC822 "PLD:RFC822" #define AKONADI_PARAM_RECURSIVE "RECURSIVE" #define AKONADI_PARAM_REFERENCED "REFERENCED" #define AKONADI_PARAM_REMOTE "REMOTE" #define AKONADI_PARAM_REMOTEID "REMOTEID" #define AKONADI_PARAM_REMOTEREVISION "REMOTEREVISION" #define AKONADI_PARAM_REVISION "REV" #define AKONADI_PARAM_SIZE "SIZE" #define AKONADI_PARAM_SYNC "SYNC" #define AKONADI_PARAM_TAG "TAG" #define AKONADI_PARAM_TYPE "TYPE" #define AKONADI_PARAM_VIRTUAL "VIRTUAL" // Flags #define AKONADI_FLAG_GID "\\Gid" #define AKONADI_FLAG_IGNORED "$IGNORED" #define AKONADI_FLAG_MIMETYPE "\\MimeType" #define AKONADI_FLAG_REMOTEID "\\RemoteId" #define AKONADI_FLAG_REMOTEREVISION "\\RemoteRevision" #define AKONADI_FLAG_TAG "\\Tag" #define AKONADI_FLAG_RTAG "\\RTag" #define AKONADI_FLAG_SEEN "\\SEEN" // Attributes #define AKONADI_ATTRIBUTE_HIDDEN "ATR:HIDDEN" #define AKONADI_ATTRIBUTE_MESSAGES "MESSAGES" #define AKONADI_ATTRIBUTE_UNSEEN "UNSEEN" // special resource names #define AKONADI_SEARCH_RESOURCE "akonadi_search_resource" #endif diff --git a/src/server/akonadi.cpp b/src/server/akonadi.cpp index 860b7f487..d5b714f38 100644 --- a/src/server/akonadi.cpp +++ b/src/server/akonadi.cpp @@ -1,425 +1,427 @@ /*************************************************************************** * Copyright (C) 2006 by Till Adam * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "akonadi.h" #include "handler.h" #include "connection.h" #include "serveradaptor.h" #include "akonadiserver_debug.h" #include "cachecleaner.h" #include "intervalcheck.h" #include "storagejanitor.h" #include "storage/dbconfig.h" #include "storage/datastore.h" #include "notificationmanager.h" #include "resourcemanager.h" #include "tracer.h" #include "utils.h" #include "debuginterface.h" #include "storage/itemretrievalmanager.h" #include "storage/collectionstatistics.h" #include "preprocessormanager.h" #include "search/searchmanager.h" #include "search/searchtaskmanager.h" #include "aklocalserver.h" #include "collectionreferencemanager.h" #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; AkonadiServer *AkonadiServer::s_instance = nullptr; AkonadiServer::AkonadiServer(QObject *parent) : QObject(parent) { // Register bunch of useful types qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType(); qRegisterMetaType("quintptr"); } bool AkonadiServer::init() { + qCInfo(AKONADISERVER_LOG) << "Starting up the Akonadi Server..."; + const QString serverConfigFile = StandardDirs::serverConfigFile(StandardDirs::ReadWrite); QSettings settings(serverConfigFile, QSettings::IniFormat); // Restrict permission to 600, as the file might contain database password in plaintext QFile::setPermissions(serverConfigFile, QFile::ReadOwner | QFile::WriteOwner); if (!DbConfig::configuredDatabase()) { quit(); return false; } if (DbConfig::configuredDatabase()->useInternalServer()) { if (!startDatabaseProcess()) { quit(); return false; } } else { if (!createDatabase()) { quit(); return false; } } DbConfig::configuredDatabase()->setup(); s_instance = this; const QString connectionSettingsFile = StandardDirs::connectionConfigFile(StandardDirs::WriteOnly); QSettings connectionSettings(connectionSettingsFile, QSettings::IniFormat); mCmdServer = new AkLocalServer(this); connect(mCmdServer, QOverload::of(&AkLocalServer::newConnection), this, &AkonadiServer::newCmdConnection); mNotificationManager = new NotificationManager(); mNtfServer = new AkLocalServer(this); // Note: this is a queued connection, as NotificationManager lives in its // own thread connect(mNtfServer, QOverload::of(&AkLocalServer::newConnection), mNotificationManager, &NotificationManager::registerConnection); // TODO: share socket setup with client #ifdef Q_OS_WIN // use the installation prefix as uid QString suffix; if (Instance::hasIdentifier()) { suffix = QStringLiteral("%1-").arg(Instance::identifier()); } suffix += QString::fromUtf8(QUrl::toPercentEncoding(qApp->applicationDirPath())); const QString defaultCmdPipe = QStringLiteral("Akonadi-Cmd-") % suffix; const QString cmdPipe = settings.value(QStringLiteral("Connection/NamedPipe"), defaultCmdPipe).toString(); if (!mCmdServer->listen(cmdPipe)) { qCCritical(AKONADISERVER_LOG) << "Unable to listen on Named Pipe" << cmdPipe; quit(); return false; } const QString defaultNtfPipe = QStringLiteral("Akonadi-Ntf-") % suffix; const QString ntfPipe = settings.value(QStringLiteral("Connection/NtfNamedPipe"), defaultNtfPipe).toString(); if (!mNtfServer->listen(ntfPipe)) { qCCritical(AKONADISERVER_LOG) << "Unable to listen on Named Pipe" << ntfPipe; quit(); return false; } connectionSettings.setValue(QStringLiteral("Data/Method"), QStringLiteral("NamedPipe")); connectionSettings.setValue(QStringLiteral("Data/NamedPipe"), cmdPipe); connectionSettings.setValue(QStringLiteral("Notifications/Method"), QStringLiteral("NamedPipe")); connectionSettings.setValue(QStringLiteral("Notifications/NamedPipe"), ntfPipe); #else const QString socketDir = Utils::preferredSocketDirectory(StandardDirs::saveDir("data")); const QString cmdSocketFile = socketDir % QStringLiteral("/akonadiserver-cmd.socket"); QFile::remove(cmdSocketFile); if (!mCmdServer->listen(cmdSocketFile)) { qCCritical(AKONADISERVER_LOG) << "Unable to listen on Unix socket" << cmdSocketFile; quit(); return false; } const QString ntfSocketFile = socketDir % QStringLiteral("/akonadiserver-ntf.socket"); QFile::remove(ntfSocketFile); if (!mNtfServer->listen(ntfSocketFile)) { qCCritical(AKONADISERVER_LOG) << "Unable to listen on Unix socket" << ntfSocketFile; quit(); return false; } connectionSettings.setValue(QStringLiteral("Data/Method"), QStringLiteral("UnixPath")); connectionSettings.setValue(QStringLiteral("Data/UnixPath"), cmdSocketFile); connectionSettings.setValue(QStringLiteral("Notifications/Method"), QStringLiteral("UnixPath")); connectionSettings.setValue(QStringLiteral("Notifications/UnixPath"), ntfSocketFile); #endif // initialize the database DataStore *db = DataStore::self(); if (!db->database().isOpen()) { qCCritical(AKONADISERVER_LOG) << "Unable to open database" << db->database().lastError().text(); quit(); return false; } if (!db->init()) { qCCritical(AKONADISERVER_LOG) << "Unable to initialize database."; quit(); return false; } Tracer::self(); new DebugInterface(this); ResourceManager::self(); CollectionStatistics::self(); // Initialize the preprocessor manager PreprocessorManager::init(); // Forcibly disable it if configuration says so if (settings.value(QStringLiteral("General/DisablePreprocessing"), false).toBool()) { PreprocessorManager::instance()->setEnabled(false); } if (settings.value(QStringLiteral("Cache/EnableCleaner"), true).toBool()) { mCacheCleaner = new CacheCleaner(); } mIntervalCheck = new IntervalCheck(); mStorageJanitor = new StorageJanitor(); mItemRetrieval = new ItemRetrievalManager(); mAgentSearchManager = new SearchTaskManager(); const QStringList searchManagers = settings.value(QStringLiteral("Search/Manager"), QStringList() << QStringLiteral("Agent")).toStringList(); mSearchManager = new SearchManager(searchManagers); new ServerAdaptor(this); QDBusConnection::sessionBus().registerObject(QStringLiteral("/Server"), this); const QByteArray dbusAddress = qgetenv("DBUS_SESSION_BUS_ADDRESS"); if (!dbusAddress.isEmpty()) { connectionSettings.setValue(QStringLiteral("DBUS/Address"), QLatin1String(dbusAddress)); } QDBusServiceWatcher *watcher = new QDBusServiceWatcher(DBus::serviceName(DBus::Control), QDBusConnection::sessionBus(), QDBusServiceWatcher::WatchForOwnerChange, this); connect(watcher, &QDBusServiceWatcher::serviceOwnerChanged, this, &AkonadiServer::serviceOwnerChanged); // Unhide all the items that are actually hidden. // The hidden flag was probably left out after an (abrupt) // server quit. We don't attempt to resume preprocessing // for the items as we don't actually know at which stage the // operation was interrupted... db->unhideAllPimItems(); // Cleanup referenced collections from the last run CollectionReferenceManager::cleanup(); // We are ready, now register org.freedesktop.Akonadi service to DBus and // the fun can begin if (!QDBusConnection::sessionBus().registerService(DBus::serviceName(DBus::Server))) { qCCritical(AKONADISERVER_LOG) << "Unable to connect to dbus service: " << QDBusConnection::sessionBus().lastError().message(); quit(); return false; } return true; } AkonadiServer::~AkonadiServer() { } template static void quitThread(T &thread) { if (thread) { thread->quit(); thread->wait(); delete thread; thread = nullptr; } } bool AkonadiServer::quit() { if (mAlreadyShutdown) { return true; } mAlreadyShutdown = true; qCDebug(AKONADISERVER_LOG) << "terminating connection threads"; qDeleteAll(mConnections); mConnections.clear(); qCDebug(AKONADISERVER_LOG) << "terminating service threads"; delete mCacheCleaner; delete mIntervalCheck; delete mStorageJanitor; delete mItemRetrieval; delete mAgentSearchManager; delete mSearchManager; delete mNotificationManager; // Terminate the preprocessor manager before the database but after all connections are gone PreprocessorManager::done(); CollectionStatistics::destroy(); if (DbConfig::isConfigured()) { if (DataStore::hasDataStore()) { DataStore::self()->close(); } qCDebug(AKONADISERVER_LOG) << "stopping db process"; stopDatabaseProcess(); } //QSettings settings(StandardDirs::serverConfigFile(), QSettings::IniFormat); const QString connectionSettingsFile = StandardDirs::connectionConfigFile(StandardDirs::WriteOnly); if (!QDir::home().remove(connectionSettingsFile)) { qCCritical(AKONADISERVER_LOG) << "Failed to remove runtime connection config file"; } QTimer::singleShot(0, this, &AkonadiServer::doQuit); return true; } void AkonadiServer::doQuit() { QCoreApplication::exit(); } void AkonadiServer::newCmdConnection(quintptr socketDescriptor) { if (mAlreadyShutdown) { return; } Connection *connection = new Connection(socketDescriptor); connect(connection, &Connection::disconnected, this, &AkonadiServer::connectionDisconnected); mConnections.append(connection); } void AkonadiServer::connectionDisconnected() { auto conn = qobject_cast(sender()); mConnections.removeOne(conn); delete conn; } AkonadiServer *AkonadiServer::instance() { if (!s_instance) { s_instance = new AkonadiServer(); } return s_instance; } bool AkonadiServer::startDatabaseProcess() { if (!DbConfig::configuredDatabase()->useInternalServer()) { qCCritical(AKONADISERVER_LOG) << "Trying to start external database!"; } // create the database directories if they don't exists StandardDirs::saveDir("data"); StandardDirs::saveDir("data", QStringLiteral("file_db_data")); return DbConfig::configuredDatabase()->startInternalServer(); } bool AkonadiServer::createDatabase() { bool success = true; const QLatin1String initCon("initConnection"); QSqlDatabase db = QSqlDatabase::addDatabase(DbConfig::configuredDatabase()->driverName(), initCon); DbConfig::configuredDatabase()->apply(db); db.setDatabaseName(DbConfig::configuredDatabase()->databaseName()); if (!db.isValid()) { qCCritical(AKONADISERVER_LOG) << "Invalid database object during initial database connection"; return false; } if (db.open()) { db.close(); } else { qCCritical(AKONADISERVER_LOG) << "Failed to use database" << DbConfig::configuredDatabase()->databaseName(); qCCritical(AKONADISERVER_LOG) << "Database error:" << db.lastError().text(); qCDebug(AKONADISERVER_LOG) << "Trying to create database now..."; db.close(); db.setDatabaseName(QString()); if (db.open()) { { QSqlQuery query(db); if (!query.exec(QStringLiteral("CREATE DATABASE %1").arg(DbConfig::configuredDatabase()->databaseName()))) { qCCritical(AKONADISERVER_LOG) << "Failed to create database"; qCCritical(AKONADISERVER_LOG) << "Query error:" << query.lastError().text(); qCCritical(AKONADISERVER_LOG) << "Database error:" << db.lastError().text(); success = false; } } // make sure query is destroyed before we close the db db.close(); } else { qCCritical(AKONADISERVER_LOG) << "Failed to connect to database!"; qCCritical(AKONADISERVER_LOG) << "Database error:" << db.lastError().text(); success = false; } } return success; } void AkonadiServer::stopDatabaseProcess() { if (!DbConfig::configuredDatabase()->useInternalServer()) { // closing initConnection this late to work around QTBUG-63108 QSqlDatabase::removeDatabase(QStringLiteral("initConnection")); return; } DbConfig::configuredDatabase()->stopInternalServer(); } void AkonadiServer::serviceOwnerChanged(const QString &name, const QString &oldOwner, const QString &newOwner) { Q_UNUSED(name); Q_UNUSED(oldOwner); if (newOwner.isEmpty()) { qCCritical(AKONADISERVER_LOG) << "Control process died, committing suicide!"; quit(); } } CacheCleaner *AkonadiServer::cacheCleaner() { return mCacheCleaner; } IntervalCheck *AkonadiServer::intervalChecker() { return mIntervalCheck; } NotificationManager *AkonadiServer::notificationManager() { return mNotificationManager; } QString AkonadiServer::serverPath() const { return StandardDirs::saveDir("config"); } diff --git a/src/server/cachecleaner.cpp b/src/server/cachecleaner.cpp index ca6f5e943..f5d9e932d 100644 --- a/src/server/cachecleaner.cpp +++ b/src/server/cachecleaner.cpp @@ -1,153 +1,153 @@ /* Copyright (c) 2007 Volker Krause Copyright (C) 2014 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "cachecleaner.h" #include "storage/parthelper.h" #include "storage/datastore.h" #include "storage/selectquerybuilder.h" #include "storage/entity.h" #include "akonadi.h" #include "akonadiserver_debug.h" #include #include using namespace Akonadi::Server; QMutex CacheCleanerInhibitor::sLock; int CacheCleanerInhibitor::sInhibitCount = 0; CacheCleanerInhibitor::CacheCleanerInhibitor(bool doInhibit) : mInhibited(false) { if (doInhibit) { inhibit(); } } CacheCleanerInhibitor::~CacheCleanerInhibitor() { if (mInhibited) { uninhibit(); } } void CacheCleanerInhibitor::inhibit() { if (mInhibited) { qCCritical(AKONADISERVER_LOG) << "Cannot recursively inhibit an inhibitor"; return; } sLock.lock(); if (++sInhibitCount == 1) { if (AkonadiServer::instance()->cacheCleaner()) { AkonadiServer::instance()->cacheCleaner()->inhibit(true); } } sLock.unlock(); mInhibited = true; } void CacheCleanerInhibitor::uninhibit() { if (!mInhibited) { qCCritical(AKONADISERVER_LOG) << "Cannot uninhibit an uninhibited inhibitor"; // aaaw yeah return; } mInhibited = false; sLock.lock(); Q_ASSERT(sInhibitCount > 0); if (--sInhibitCount == 0) { if (AkonadiServer::instance()->cacheCleaner()) { AkonadiServer::instance()->cacheCleaner()->inhibit(false); } } sLock.unlock(); } CacheCleaner::CacheCleaner(QObject *parent) : CollectionScheduler(QStringLiteral("CacheCleaner"), QThread::IdlePriority, parent) { setMinimumInterval(5); } CacheCleaner::~CacheCleaner() { quitThread(); } int CacheCleaner::collectionScheduleInterval(const Collection &collection) { return collection.cachePolicyCacheTimeout(); } bool CacheCleaner::hasChanged(const Collection &collection, const Collection &changed) { return collection.cachePolicyLocalParts() != changed.cachePolicyLocalParts() || collection.cachePolicyCacheTimeout() != changed.cachePolicyCacheTimeout() || collection.cachePolicyInherit() != changed.cachePolicyInherit(); } bool CacheCleaner::shouldScheduleCollection(const Collection &collection) { return collection.cachePolicyLocalParts() != QLatin1String("ALL") && collection.cachePolicyCacheTimeout() >= 0 && (collection.enabled() || (collection.displayPref() == Collection::True) || (collection.syncPref() == Collection::True) || (collection.indexPref() == Collection::True)) && collection.resourceId() > 0; } void CacheCleaner::collectionExpired(const Collection &collection) { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdColumn(), PimItem::idFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName()); qb.addValueCondition(PimItem::collectionIdFullColumnName(), Query::Equals, collection.id()); qb.addValueCondition(PimItem::atimeFullColumnName(), Query::Less, QDateTime::currentDateTimeUtc().addSecs(-60 * collection.cachePolicyCacheTimeout())); qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant()); qb.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QLatin1String("PLD")); qb.addValueCondition(PimItem::dirtyFullColumnName(), Query::Equals, false); const QStringList partNames = collection.cachePolicyLocalParts().split(QLatin1Char(' ')); for (QString partName : partNames) { if (partName.startsWith(QLatin1String(AKONADI_PARAM_PLD))) { partName = partName.mid(4); } qb.addValueCondition(PartType::nameFullColumnName(), Query::NotEquals, partName); } if (qb.exec()) { const Part::List parts = qb.result(); if (!parts.isEmpty()) { - qCDebug(AKONADISERVER_LOG) << "found" << parts.count() << "item parts to expire in collection" << collection.name(); + qCInfo(AKONADISERVER_LOG) << "CacheCleaner found" << parts.count() << "item parts to expire in collection" << collection.name(); // clear data field for (Part part : parts) { try { if (!PartHelper::truncate(part)) { - qCDebug(AKONADISERVER_LOG) << "failed to update item part" << part.id(); + qCWarning(AKONADISERVER_LOG) << "CacheCleaner failed to expire item part" << part.id(); } } catch (const PartHelperException &e) { qCCritical(AKONADISERVER_LOG) << e.type() << e.what(); } } } } } diff --git a/src/server/connection.cpp b/src/server/connection.cpp index 602eb7df2..eb292c6cf 100644 --- a/src/server/connection.cpp +++ b/src/server/connection.cpp @@ -1,510 +1,518 @@ /*************************************************************************** * Copyright (C) 2006 by Till Adam * * Copyright (C) 2013 by Volker Krause * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "connection.h" #include "akonadiserver_debug.h" #include #include #include #include #include #include "storage/datastore.h" #include "handler.h" #include "notificationmanager.h" #include "tracer.h" #include "collectionreferencemanager.h" #include #ifndef Q_OS_WIN #include #endif #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; #define IDLE_TIMER_TIMEOUT 180000 // 3 min static QString connectionIdentifier(Connection *c) { QString id; id.sprintf("%p", static_cast(c)); return id; } Connection::Connection(QObject *parent) : AkThread(connectionIdentifier(this), QThread::InheritPriority, parent) { } Connection::Connection(quintptr socketDescriptor, QObject *parent) : AkThread(connectionIdentifier(this), QThread::InheritPriority, parent) { m_socketDescriptor = socketDescriptor; m_identifier = connectionIdentifier(this); // same as objectName() const QSettings settings(Akonadi::StandardDirs::serverConfigFile(), QSettings::IniFormat); m_verifyCacheOnRetrieval = settings.value(QStringLiteral("Cache/VerifyOnRetrieval"), m_verifyCacheOnRetrieval).toBool(); } void Connection::init() { AkThread::init(); QLocalSocket *socket = new QLocalSocket(); if (!socket->setSocketDescriptor(m_socketDescriptor)) { qCWarning(AKONADISERVER_LOG) << "Connection(" << m_identifier << ")::run: failed to set socket descriptor: " << socket->error() << "(" << socket->errorString() << ")"; delete socket; return; } m_socket = socket; connect(socket, &QLocalSocket::disconnected, this, &Connection::slotSocketDisconnected); m_idleTimer = new QTimer(this); connect(m_idleTimer, &QTimer::timeout, this, &Connection::slotConnectionIdle); storageBackend()->notificationCollector()->setConnection(this); if (socket->state() == QLocalSocket::ConnectedState) { QTimer::singleShot(0, this, &Connection::handleIncomingData); } else { connect(socket, &QLocalSocket::connected, this, &Connection::handleIncomingData, Qt::QueuedConnection); } try { slotSendHello(); } catch (const ProtocolException &e) { qCWarning(AKONADISERVER_LOG) << "Protocol Exception sending \"hello\":" << e.what(); m_socket->disconnectFromServer(); } } void Connection::quit() { if (QThread::currentThread()->loopLevel() > 1) { m_connectionClosing = true; Q_EMIT connectionClosing(); return; } Tracer::self()->endConnection(m_identifier, QString()); collectionReferenceManager()->removeSession(m_sessionId); delete m_socket; m_socket = nullptr; if (m_idleTimer) { m_idleTimer->stop(); } delete m_idleTimer; AkThread::quit(); } void Connection::slotSendHello() { SchemaVersion version = SchemaVersion::retrieveAll().first(); Protocol::HelloResponse hello; hello.setServerName(QStringLiteral("Akonadi")); hello.setMessage(QStringLiteral("Not Really IMAP server")); hello.setProtocolVersion(Protocol::version()); hello.setGeneration(version.generation()); sendResponse(0, std::move(hello)); } DataStore *Connection::storageBackend() { if (!m_backend) { m_backend = DataStore::self(); } return m_backend; } CollectionReferenceManager *Connection::collectionReferenceManager() { return CollectionReferenceManager::instance(); } Connection::~Connection() { quitThread(); if (m_reportTime) { reportTime(); } } void Connection::slotConnectionIdle() { Q_ASSERT(m_currentHandler == nullptr); if (m_backend && m_backend->isOpened()) { if (m_backend->inTransaction()) { // This is a programming error, the timer should not have fired. // But it is safer to abort and leave the connection open, until // a later operation causes the idle timer to fire (than crash // the akonadi server). - qCDebug(AKONADISERVER_LOG) << m_sessionId << "NOT Closing idle db connection; we are in transaction"; + qCInfo(AKONADISERVER_LOG) << m_sessionId << "NOT Closing idle db connection; we are in transaction"; return; } m_backend->close(); } } void Connection::slotSocketDisconnected() { // If we have active handler, wait for it to finish, then we emit the signal // from slotNewDate() if (m_currentHandler) { return; } Q_EMIT disconnected(); } void Connection::handleIncomingData() { Q_FOREVER { if (m_connectionClosing || !m_socket || m_socket->state() != QLocalSocket::ConnectedState) { break; } // Blocks with event loop until some data arrive, allows us to still use QTimers // and similar while waiting for some data to arrive if (m_socket->bytesAvailable() < int(sizeof(qint64))) { QEventLoop loop; connect(m_socket, &QLocalSocket::readyRead, &loop, &QEventLoop::quit); connect(m_socket, &QLocalSocket::stateChanged, &loop, &QEventLoop::quit); connect(this, &Connection::connectionClosing, &loop, &QEventLoop::quit); loop.exec(); } if (m_connectionClosing || !m_socket || m_socket->state() != QLocalSocket::ConnectedState) { break; } m_idleTimer->stop(); // will only open() a previously idle backend. // Otherwise, a new backend could lazily be constructed by later calls. if (!storageBackend()->isOpened()) { m_backend->open(); } QString currentCommand; while (m_socket->bytesAvailable() >= int(sizeof(qint64))) { Protocol::DataStream stream(m_socket); qint64 tag = -1; stream >> tag; // TODO: Check tag is incremental sequence Protocol::CommandPtr cmd; try { cmd = Protocol::deserialize(m_socket); } catch (const Akonadi::ProtocolException &e) { - qCWarning(AKONADISERVER_LOG) << "ProtocolException:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "ProtocolException while deserializing incoming data on connection" + << m_identifier << ":" << e.what(); setState(Server::LoggingOut); return; } catch (const std::exception &e) { - qCWarning(AKONADISERVER_LOG) << "Unknown exception:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "Unknown exception while deserializing incoming data on connection" + << m_identifier << ":" << e.what(); setState(Server::LoggingOut); return; } if (cmd->type() == Protocol::Command::Invalid) { - qCWarning(AKONADISERVER_LOG) << "Received an invalid command: resetting connection"; + qCWarning(AKONADISERVER_LOG) << "Received an invalid command on connection" << m_identifier + << ": resetting connection"; setState(Server::LoggingOut); return; } // Tag context and collection context is not persistent. context()->setTag(-1); context()->setCollection(Collection()); if (Tracer::self()->currentTracer() != QLatin1String("null")) { Tracer::self()->connectionInput(m_identifier, tag, cmd); } m_currentHandler = std::unique_ptr(findHandlerForCommand(cmd->type())); if (!m_currentHandler) { - qCWarning(AKONADISERVER_LOG) << "Invalid command: no such handler for" << cmd->type(); + qCWarning(AKONADISERVER_LOG) << "Invalid command: no such handler for" << cmd->type() + << "on connection" << m_identifier; setState(Server::LoggingOut); return; } if (m_reportTime) { startTime(); } m_currentHandler->setConnection(this); m_currentHandler->setTag(tag); m_currentHandler->setCommand(cmd); try { if (!m_currentHandler->parseStream()) { try { - m_currentHandler->failureResponse("Unknown error while handling a command"); + m_currentHandler->failureResponse("Error while handling a command"); } catch (...) { - qCWarning(AKONADISERVER_LOG) << "Unknown error while handling a command"; m_connectionClosing = true; } + qCWarning(AKONADISERVER_LOG) << "Error while handling command" << cmd->type() + << "on connection" << m_identifier; } } catch (const Akonadi::Server::HandlerException &e) { if (m_currentHandler) { try { m_currentHandler->failureResponse(e.what()); } catch (...) { - qCWarning(AKONADISERVER_LOG) << "Handler exception:" << e.what(); m_connectionClosing = true; } + qCWarning(AKONADISERVER_LOG) << "Handler exception when handling command" << cmd->type() + << "on connection" << m_identifier << ":" << e.what(); } } catch (const Akonadi::Server::Exception &e) { if (m_currentHandler) { try { m_currentHandler->failureResponse(QString::fromUtf8(e.type()) + QLatin1String(": ") + QString::fromUtf8(e.what())); } catch (...) { - qCWarning(AKONADISERVER_LOG) << e.type() << "exception:" << e.what(); m_connectionClosing = true; } + qCWarning(AKONADISERVER_LOG) << "General exception when handling command" << cmd->type() + << "on connection" << m_identifier << ":" << e.what(); } } catch (const Akonadi::ProtocolException &e) { // No point trying to send anything back to client, the connection is // already messed up - qCWarning(AKONADISERVER_LOG) << "Protocol exception:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "Protocol exception when handling command" << cmd->type() + << "on connection" << m_identifier << ":" << e.what(); m_connectionClosing = true; #if defined(Q_OS_LINUX) } catch (abi::__forced_unwind&) { // HACK: NPTL throws __forced_unwind during thread cancellation and // we *must* rethrow it otherwise the program aborts. Due to the issue // described in #376385 we might end up destroying (cancelling) the // thread from a nested loop executed inside parseStream() above, // so the exception raised in there gets caught by this try..catch // statement and it must be rethrown at all cost. Remove this hack // once the root problem is fixed. throw; #endif } catch (...) { - qCCritical(AKONADISERVER_LOG) << "Unknown exception caught in Connection for session" << m_sessionId; + qCCritical(AKONADISERVER_LOG) << "Unknown exception while handling command" << cmd->type() + << "on connection" << m_identifier; if (m_currentHandler) { try { m_currentHandler->failureResponse("Unknown exception caught"); } catch (...) { - qCWarning(AKONADISERVER_LOG) << "Unknown exception caught"; m_connectionClosing = true; } } } if (m_reportTime) { stopTime(currentCommand); } m_currentHandler.reset(); if (!m_socket || m_socket->state() != QLocalSocket::ConnectedState) { Q_EMIT disconnected(); return; } if (m_connectionClosing) { break; } } // reset, arm the timer m_idleTimer->start(IDLE_TIMER_TIMEOUT); if (m_connectionClosing) { break; } } if (m_connectionClosing) { m_socket->disconnect(this); m_socket->close(); QTimer::singleShot(0, this, &Connection::quit); } } CommandContext *Connection::context() const { return const_cast(&m_context); } Handler *Connection::findHandlerForCommand(Protocol::Command::Type command) { Handler *handler = Handler::findHandlerForCommandAlwaysAllowed(command); if (handler) { return handler; } switch (m_connectionState) { case NonAuthenticated: handler = Handler::findHandlerForCommandNonAuthenticated(command); break; case Authenticated: handler = Handler::findHandlerForCommandAuthenticated(command); break; case Selected: break; case LoggingOut: break; } return handler; } qint64 Connection::currentTag() const { return m_currentHandler->tag(); } void Connection::setState(ConnectionState state) { if (state == m_connectionState) { return; } m_connectionState = state; switch (m_connectionState) { case NonAuthenticated: assert(0); // can't happen, it's only the initial state, we can't go back to it break; case Authenticated: break; case Selected: break; case LoggingOut: m_socket->disconnectFromServer(); break; } } void Connection::setSessionId(const QByteArray &id) { m_identifier.sprintf("%s (%p)", id.data(), static_cast(this)); Tracer::self()->beginConnection(m_identifier, QString()); //m_streamParser->setTracerIdentifier(m_identifier); m_sessionId = id; setObjectName(QString::fromLatin1(id)); // this races with the use of objectName() in QThreadPrivate::start //thread()->setObjectName(objectName() + QStringLiteral("-Thread")); storageBackend()->setSessionId(id); } QByteArray Connection::sessionId() const { return m_sessionId; } bool Connection::isOwnerResource(const PimItem &item) const { if (context()->resource().isValid() && item.collection().resourceId() == context()->resource().id()) { return true; } // fallback for older resources if (sessionId() == item.collection().resource().name().toUtf8()) { return true; } return false; } bool Connection::isOwnerResource(const Collection &collection) const { if (context()->resource().isValid() && collection.resourceId() == context()->resource().id()) { return true; } if (sessionId() == collection.resource().name().toUtf8()) { return true; } return false; } bool Connection::verifyCacheOnRetrieval() const { return m_verifyCacheOnRetrieval; } void Connection::startTime() { m_time.start(); } void Connection::stopTime(const QString &identifier) { int elapsed = m_time.elapsed(); m_totalTime += elapsed; m_totalTimeByHandler[identifier] += elapsed; m_executionsByHandler[identifier]++; qCDebug(AKONADISERVER_LOG) << identifier << " time : " << elapsed << " total: " << m_totalTime; } void Connection::reportTime() const { qCDebug(AKONADISERVER_LOG) << "===== Time report for " << m_identifier << " ====="; qCDebug(AKONADISERVER_LOG) << " total: " << m_totalTime; for (auto it = m_totalTimeByHandler.cbegin(), end = m_totalTimeByHandler.cend(); it != end; ++it) { const QString &handler = it.key(); qCDebug(AKONADISERVER_LOG) << "handler : " << handler << " time: " << m_totalTimeByHandler.value(handler) << " executions " << m_executionsByHandler.value(handler) << " avg: " << m_totalTimeByHandler.value(handler) / m_executionsByHandler.value(handler); } } void Connection::sendResponse(qint64 tag, const Protocol::CommandPtr &response) { if (Tracer::self()->currentTracer() != QLatin1String("null")) { Tracer::self()->connectionOutput(m_identifier, tag, response); } Protocol::DataStream stream(m_socket); stream << tag; Protocol::serialize(m_socket, response); if (!m_socket->waitForBytesWritten()) { if (m_socket->state() == QLocalSocket::ConnectedState) { throw ProtocolException("Server write timeout"); } else { // The client has disconnected before we managed to send our response, // which is not an error } } } Protocol::CommandPtr Connection::readCommand() { while (m_socket->bytesAvailable() < (int) sizeof(qint64)) { Protocol::DataStream::waitForData(m_socket, 10000); // 10 seconds, just in case client is busy } Protocol::DataStream stream(m_socket); qint64 tag; stream >> tag; // TODO: compare tag with m_currentHandler->tag() ? return Protocol::deserialize(m_socket); } diff --git a/src/server/handler/akappend.cpp b/src/server/handler/akappend.cpp index f942d827b..9863db665 100644 --- a/src/server/handler/akappend.cpp +++ b/src/server/handler/akappend.cpp @@ -1,451 +1,451 @@ /*************************************************************************** * Copyright (C) 2007 by Robert Zwerus * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "akappend.h" #include "fetchhelper.h" #include "connection.h" #include "preprocessormanager.h" #include "handlerhelper.h" #include "storage/datastore.h" #include "storage/transaction.h" #include "storage/parttypehelper.h" #include "storage/dbconfig.h" #include "storage/partstreamer.h" #include "storage/parthelper.h" #include "storage/selectquerybuilder.h" #include #include //std::accumulate using namespace Akonadi; using namespace Akonadi::Server; static QVector localFlagsToPreserve = QVector() << "$ATTACHMENT" << "$INVITATION" << "$ENCRYPTED" << "$SIGNED" << "$WATCHED"; bool AkAppend::buildPimItem(const Protocol::CreateItemCommand &cmd, PimItem &item, Collection &parentCol) { parentCol = HandlerHelper::collectionFromScope(cmd.collection(), connection()); if (!parentCol.isValid()) { return failureResponse(QStringLiteral("Invalid parent collection")); } if (parentCol.isVirtual()) { return failureResponse(QStringLiteral("Cannot append item into virtual collection")); } MimeType mimeType = MimeType::retrieveByNameOrCreate(cmd.mimeType()); if (!mimeType.isValid()) { return failureResponse(QStringLiteral("Unable to create mimetype '") % cmd.mimeType() % QStringLiteral("'.")); } item.setRev(0); item.setSize(cmd.itemSize()); item.setMimeTypeId(mimeType.id()); item.setCollectionId(parentCol.id()); item.setDatetime(cmd.dateTime()); if (cmd.remoteId().isEmpty()) { // from application item.setDirty(true); } else { // from resource item.setRemoteId(cmd.remoteId()); item.setDirty(false); } item.setRemoteRevision(cmd.remoteRevision()); item.setGid(cmd.gid()); item.setAtime(QDateTime::currentDateTimeUtc()); return true; } bool AkAppend::insertItem(const Protocol::CreateItemCommand &cmd, PimItem &item, const Collection &parentCol) { if (!item.datetime().isValid()) { item.setDatetime(QDateTime::currentDateTimeUtc()); } if (!item.insert()) { return failureResponse(QStringLiteral("Failed to append item")); } // set message flags const QSet flags = cmd.mergeModes() == Protocol::CreateItemCommand::None ? cmd.flags() : cmd.addedFlags(); if (!flags.isEmpty()) { // This will hit an entry in cache inserted there in buildPimItem() const Flag::List flagList = HandlerHelper::resolveFlags(flags); bool flagsChanged = false; if (!DataStore::self()->appendItemsFlags(PimItem::List() << item, flagList, &flagsChanged, false, parentCol, true)) { return failureResponse("Unable to append item flags."); } } const Scope tags = cmd.mergeModes() == Protocol::CreateItemCommand::None ? cmd.tags() : cmd.addedTags(); if (!tags.isEmpty()) { const Tag::List tagList = HandlerHelper::tagsFromScope(tags, connection()); bool tagsChanged = false; if (!DataStore::self()->appendItemsTags(PimItem::List() << item, tagList, &tagsChanged, false, parentCol, true)) { return failureResponse(QStringLiteral("Unable to append item tags.")); } } // Handle individual parts qint64 partSizes = 0; PartStreamer streamer(connection(), item); Q_FOREACH (const QByteArray &partName, cmd.parts()) { qint64 partSize = 0; if (!streamer.stream(true, partName, partSize)) { return failureResponse(streamer.error()); } partSizes += partSize; } const Protocol::Attributes attrs = cmd.attributes(); for (auto iter = attrs.cbegin(), end = attrs.cend(); iter != end; ++iter) { if (!streamer.streamAttribute(true, iter.key(), iter.value())) { return failureResponse(streamer.error()); } } // TODO: Try to avoid this addition query if (partSizes > item.size()) { item.setSize(partSizes); item.update(); } // Preprocessing if (PreprocessorManager::instance()->isActive()) { Part hiddenAttribute; hiddenAttribute.setPimItemId(item.id()); hiddenAttribute.setPartType(PartTypeHelper::fromFqName(QStringLiteral(AKONADI_ATTRIBUTE_HIDDEN))); hiddenAttribute.setData(QByteArray()); hiddenAttribute.setDatasize(0); // TODO: Handle errors? Technically, this is not a critical issue as no data are lost PartHelper::insert(&hiddenAttribute); } const bool seen = flags.contains(AKONADI_FLAG_SEEN) || flags.contains(AKONADI_FLAG_IGNORED); notify(item, seen, item.collection()); sendResponse(item, Protocol::CreateItemCommand::None); return true; } bool AkAppend::mergeItem(const Protocol::CreateItemCommand &cmd, PimItem &newItem, PimItem ¤tItem, const Collection &parentCol) { bool needsUpdate = false; QSet changedParts; if (!newItem.remoteId().isEmpty() && currentItem.remoteId() != newItem.remoteId()) { currentItem.setRemoteId(newItem.remoteId()); changedParts.insert(AKONADI_PARAM_REMOTEID); needsUpdate = true; } if (!newItem.remoteRevision().isEmpty() && currentItem.remoteRevision() != newItem.remoteRevision()) { currentItem.setRemoteRevision(newItem.remoteRevision()); changedParts.insert(AKONADI_PARAM_REMOTEREVISION); needsUpdate = true; } if (!newItem.gid().isEmpty() && currentItem.gid() != newItem.gid()) { currentItem.setGid(newItem.gid()); changedParts.insert(AKONADI_PARAM_GID); needsUpdate = true; } if (newItem.datetime().isValid() && newItem.datetime() != currentItem.datetime()) { currentItem.setDatetime(newItem.datetime()); needsUpdate = true; } if (newItem.size() > 0 && newItem.size() != currentItem.size()) { currentItem.setSize(newItem.size()); needsUpdate = true; } const Collection col = Collection::retrieveById(parentCol.id()); if (cmd.flags().isEmpty() && !cmd.flagsOverwritten()) { bool flagsAdded = false, flagsRemoved = false; if (!cmd.addedFlags().isEmpty()) { const Flag::List addedFlags = HandlerHelper::resolveFlags(cmd.addedFlags()); DataStore::self()->appendItemsFlags(PimItem::List() << currentItem, addedFlags, &flagsAdded, true, col, true); } if (!cmd.removedFlags().isEmpty()) { const Flag::List removedFlags = HandlerHelper::resolveFlags(cmd.removedFlags()); DataStore::self()->removeItemsFlags(PimItem::List() << currentItem, removedFlags, &flagsRemoved, col, true); } if (flagsAdded || flagsRemoved) { changedParts.insert(AKONADI_PARAM_FLAGS); needsUpdate = true; } } else { bool flagsChanged = false; QSet flagNames = cmd.flags(); // Make sure we don't overwrite some local-only flags that can't come // through from Resource during ItemSync, like $ATTACHMENT, because the // resource is not aware of them (they are usually assigned by client // upon inspecting the payload) Q_FOREACH (const Flag ¤tFlag, currentItem.flags()) { const QByteArray currentFlagName = currentFlag.name().toLatin1(); if (localFlagsToPreserve.contains(currentFlagName)) { flagNames.insert(currentFlagName); } } const Flag::List flags = HandlerHelper::resolveFlags(flagNames); DataStore::self()->setItemsFlags(PimItem::List() << currentItem, flags, &flagsChanged, col, true); if (flagsChanged) { changedParts.insert(AKONADI_PARAM_FLAGS); needsUpdate = true; } } if (cmd.tags().isEmpty()) { bool tagsAdded = false, tagsRemoved = false; if (!cmd.addedTags().isEmpty()) { const Tag::List addedTags = HandlerHelper::tagsFromScope(cmd.addedTags(), connection()); DataStore::self()->appendItemsTags(PimItem::List() << currentItem, addedTags, &tagsAdded, true, col, true); } if (!cmd.removedTags().isEmpty()) { const Tag::List removedTags = HandlerHelper::tagsFromScope(cmd.removedTags(), connection()); DataStore::self()->removeItemsTags(PimItem::List() << currentItem, removedTags, &tagsRemoved, true); } if (tagsAdded || tagsRemoved) { changedParts.insert(AKONADI_PARAM_TAGS); needsUpdate = true; } } else { bool tagsChanged = false; const Tag::List tags = HandlerHelper::tagsFromScope(cmd.tags(), connection()); DataStore::self()->setItemsTags(PimItem::List() << currentItem, tags, &tagsChanged, true); if (tagsChanged) { changedParts.insert(AKONADI_PARAM_TAGS); needsUpdate = true; } } const Part::List existingParts = Part::retrieveFiltered(Part::pimItemIdColumn(), currentItem.id()); QMap partsSizes; for (const Part &part : existingParts) { partsSizes.insert(PartTypeHelper::fullName(part.partType()).toLatin1(), part.datasize()); } PartStreamer streamer(connection(), currentItem); Q_FOREACH (const QByteArray &partName, cmd.parts()) { bool changed = false; qint64 partSize = 0; if (!streamer.stream(true, partName, partSize, &changed)) { return failureResponse(streamer.error()); } if (changed) { changedParts.insert(partName); partsSizes.insert(partName, partSize); needsUpdate = true; } } const qint64 size = std::accumulate(partsSizes.begin(), partsSizes.end(), 0); if (size > currentItem.size()) { currentItem.setSize(size); needsUpdate = true; } if (needsUpdate) { currentItem.setRev(qMax(newItem.rev(), currentItem.rev()) + 1); currentItem.setAtime(QDateTime::currentDateTimeUtc()); // Only mark dirty when merged from application currentItem.setDirty(!connection()->context()->resource().isValid()); // Store all changes if (!currentItem.update()) { return failureResponse("Failed to store merged item"); } notify(currentItem, currentItem.collection(), changedParts); } sendResponse(currentItem, cmd.mergeModes()); return true; } bool AkAppend::sendResponse(const PimItem &item, Protocol::CreateItemCommand::MergeModes mergeModes) { if (mergeModes & Protocol::CreateItemCommand::Silent || mergeModes & Protocol::CreateItemCommand::None) { Protocol::FetchItemsResponse resp; resp.setId(item.id()); resp.setMTime(item.datetime()); Handler::sendResponse(std::move(resp)); return true; } Protocol::ItemFetchScope fetchScope; fetchScope.setAncestorDepth(Protocol::ItemFetchScope::ParentAncestor); fetchScope.setFetch(Protocol::ItemFetchScope::AllAttributes | Protocol::ItemFetchScope::FullPayload | Protocol::ItemFetchScope::CacheOnly | Protocol::ItemFetchScope::Flags | Protocol::ItemFetchScope::GID | Protocol::ItemFetchScope::MTime | Protocol::ItemFetchScope::RemoteID | Protocol::ItemFetchScope::RemoteRevision | Protocol::ItemFetchScope::Size | Protocol::ItemFetchScope::Tags); ImapSet set; set.add(QVector() << item.id()); Scope scope; scope.setUidSet(set); FetchHelper fetchHelper(connection(), scope, fetchScope, Protocol::TagFetchScope{}); if (!fetchHelper.fetchItems()) { return failureResponse("Failed to retrieve item"); } return true; } bool AkAppend::notify(const PimItem &item, bool seen, const Collection &collection) { DataStore::self()->notificationCollector()->itemAdded(item, seen, collection); if (PreprocessorManager::instance()->isActive()) { // enqueue the item for preprocessing PreprocessorManager::instance()->beginHandleItem(item, DataStore::self()); } return true; } bool AkAppend::notify(const PimItem &item, const Collection &collection, const QSet &changedParts) { if (!changedParts.isEmpty()) { DataStore::self()->notificationCollector()->itemChanged(item, changedParts, collection); } return true; } bool AkAppend::parseStream() { const auto &cmd = Protocol::cmdCast(m_command); // FIXME: The streaming/reading of all item parts can hold the transaction for // unnecessary long time -> should we wrap the PimItem into one transaction // and try to insert Parts independently? In case we fail to insert a part, // it's not a problem as it can be re-fetched at any time, except for attributes. DataStore *db = DataStore::self(); Transaction transaction(db, QStringLiteral("AKAPPEND")); ExternalPartStorageTransaction storageTrx; PimItem item; Collection parentCol; if (!buildPimItem(cmd, item, parentCol)) { return false; } if (cmd.mergeModes() == Protocol::CreateItemCommand::None) { if (!insertItem(cmd, item, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse(QStringLiteral("Failed to commit transaction")); } storageTrx.commit(); } else { // Merging is always restricted to the same collection SelectQueryBuilder qb; qb.setForUpdate(); qb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, parentCol.id()); Query::Condition rootCondition(Query::Or); Query::Condition mergeCondition(Query::And); if (cmd.mergeModes() & Protocol::CreateItemCommand::GID) { mergeCondition.addValueCondition(PimItem::gidColumn(), Query::Equals, item.gid()); } if (cmd.mergeModes() & Protocol::CreateItemCommand::RemoteID) { mergeCondition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, item.remoteId()); } rootCondition.addCondition(mergeCondition); // If an Item with matching RID but empty GID exists during GID merge, // merge into this item instead of creating a new one if (cmd.mergeModes() & Protocol::CreateItemCommand::GID && !item.remoteId().isEmpty()) { mergeCondition = Query::Condition(Query::And); mergeCondition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, item.remoteId()); mergeCondition.addValueCondition(PimItem::gidColumn(), Query::Equals, QStringLiteral("")); rootCondition.addCondition(mergeCondition); } qb.addCondition(rootCondition); if (!qb.exec()) { return failureResponse("Failed to query database for item"); } const QVector result = qb.result(); if (result.isEmpty()) { // No item with such GID/RID exists, so call AkAppend::insert() and behave // like if this was a new item if (!insertItem(cmd, item, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse("Failed to commit transaction"); } storageTrx.commit(); } else if (result.count() == 1) { // Item with matching GID/RID combination exists, so merge this item into it // and send itemChanged() PimItem existingItem = result.at(0); if (!mergeItem(cmd, item, existingItem, parentCol)) { return false; } if (!transaction.commit()) { return failureResponse("Failed to commit transaction"); } storageTrx.commit(); } else { - qCDebug(AKONADISERVER_LOG) << "Multiple merge candidates:"; + qCWarning(AKONADISERVER_LOG) << "Multiple merge candidates:"; for (const PimItem &item : result) { - qCDebug(AKONADISERVER_LOG) << "\tID:" << item.id() << ", RID:" << item.remoteId() + qCWarning(AKONADISERVER_LOG) << "\tID:" << item.id() << ", RID:" << item.remoteId() << ", GID:" << item.gid() << ", Collection:" << item.collection().name() << "(" << item.collectionId() << ")" << ", Resource:" << item.collection().resource().name() << "(" << item.collection().resourceId() << ")"; } // Nor GID or RID are guaranteed to be unique, so make sure we don't merge // something we don't want return failureResponse(QStringLiteral("Multiple merge candidates, aborting")); } } return successResponse(); } diff --git a/src/server/handler/list.cpp b/src/server/handler/list.cpp index e39f391d7..c714d7f8d 100644 --- a/src/server/handler/list.cpp +++ b/src/server/handler/list.cpp @@ -1,609 +1,604 @@ /* Copyright (c) 2007 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "list.h" #include "akonadiserver_debug.h" #include "connection.h" #include "handlerhelper.h" #include "collectionreferencemanager.h" #include "storage/datastore.h" #include "storage/selectquerybuilder.h" #include "storage/collectionqueryhelper.h" #include using namespace Akonadi; using namespace Akonadi::Server; template static bool intersect(const QVector &l1, const QVector &l2) { for (const T &e2 : l2) { if (l1.contains(e2.id())) { return true; } } return false; } QStack List::ancestorsForCollection(const Collection &col) { if (mAncestorDepth <= 0) { return QStack(); } QStack ancestors; Collection parent = col; for (int i = 0; i < mAncestorDepth; ++i) { if (parent.parentId() == 0) { break; } if (mAncestors.contains(parent.parentId())) { parent = mAncestors.value(parent.parentId()); } else { parent = mCollections.value(parent.parentId()); } if (!parent.isValid()) { - qCWarning(AKONADISERVER_LOG) << col.id(); + qCWarning(AKONADISERVER_LOG) << "Found an invalid parent in ancestors of Collection" << col.name() + << "(ID:" << col.id() << ")"; throw HandlerException("Found invalid parent in ancestors"); } ancestors.prepend(parent); } return ancestors; } CollectionAttribute::List List::getAttributes(const Collection &col, const QSet &filter) { CollectionAttribute::List attributes; auto it = mCollectionAttributes.find(col.id()); while (it != mCollectionAttributes.end() && it.key() == col.id()) { if (filter.isEmpty() || filter.contains(it.value().type())) { attributes << it.value(); } ++it; } // We need the reference and enabled status, to not need to request the server multiple times. // Mostly interesting for ancestors for i.e. updates provided by the monitor. const bool isReferenced = connection()->collectionReferenceManager()->isReferenced(col.id(), connection()->sessionId()); if (isReferenced) { CollectionAttribute attr; attr.setType(AKONADI_PARAM_REFERENCED); attr.setValue("TRUE"); attributes << attr; } { CollectionAttribute attr; attr.setType(AKONADI_PARAM_ENABLED); attr.setValue(col.enabled() ? "TRUE" : "FALSE"); attributes << attr; } return attributes; } void List::listCollection(const Collection &root, const QStack &ancestors, const QStringList &mimeTypes, const CollectionAttribute::List &attributes) { const bool isReferencedFromSession = connection()->collectionReferenceManager()->isReferenced(root.id(), connection()->sessionId()); //We always expose referenced collections to the resource as referenced (although it's a different session) //Otherwise syncing wouldn't work. const bool resourceIsSynchronizing = root.referenced() && mCollectionsToSynchronize && connection()->context()->resource().isValid(); QStack ancestorAttributes; //backwards compatibility, collectionToByteArray will automatically fall-back to id + remoteid if (!mAncestorAttributes.isEmpty()) { ancestorAttributes.reserve(ancestors.size()); for (const Collection &col : ancestors) { ancestorAttributes.push(getAttributes(col, mAncestorAttributes)); } } // write out collection details Collection dummy = root; DataStore *db = connection()->storageBackend(); db->activeCachePolicy(dummy); sendResponse(HandlerHelper::fetchCollectionsResponse(dummy, attributes, mIncludeStatistics, mAncestorDepth, ancestors, ancestorAttributes, isReferencedFromSession || resourceIsSynchronizing, mimeTypes)); } static Query::Condition filterCondition(const QString &column) { Query::Condition orCondition(Query::Or); orCondition.addValueCondition(column, Query::Equals, (int)Collection::True); Query::Condition andCondition(Query::And); andCondition.addValueCondition(column, Query::Equals, (int)Collection::Undefined); andCondition.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true); orCondition.addCondition(andCondition); orCondition.addValueCondition(Collection::referencedFullColumnName(), Query::Equals, true); return orCondition; } bool List::checkFilterCondition(const Collection &col) const { //Don't include the collection when only looking for enabled collections if (mEnabledCollections && !col.enabled()) { return false; } //Don't include the collection when only looking for collections to display/index/sync if (mCollectionsToDisplay && (((col.displayPref() == Collection::Undefined) && !col.enabled()) || (col.displayPref() == Collection::False))) { return false; } if (mCollectionsToIndex && (((col.indexPref() == Collection::Undefined) && !col.enabled()) || (col.indexPref() == Collection::False))) { return false; } //Single collection sync will still work since that is using a base fetch if (mCollectionsToSynchronize && (((col.syncPref() == Collection::Undefined) && !col.enabled()) || (col.syncPref() == Collection::False))) { return false; } return true; } static QSqlQuery getAttributeQuery(const QVariantList &ids, const QSet &requestedAttributes) { QueryBuilder qb(CollectionAttribute::tableName()); qb.addValueCondition(CollectionAttribute::collectionIdFullColumnName(), Query::In, ids); qb.addColumn(CollectionAttribute::collectionIdFullColumnName()); qb.addColumn(CollectionAttribute::typeFullColumnName()); qb.addColumn(CollectionAttribute::valueFullColumnName()); if (!requestedAttributes.isEmpty()) { QVariantList attributes; attributes.reserve(requestedAttributes.size()); for (const QByteArray &type : requestedAttributes) { attributes << type; } qb.addValueCondition(CollectionAttribute::typeFullColumnName(), Query::In, attributes); } qb.addSortColumn(CollectionAttribute::collectionIdFullColumnName(), Query::Ascending); if (!qb.exec()) { throw HandlerException("Unable to retrieve attributes for listing"); } return qb.query(); } void List::retrieveAttributes(const QVariantList &collectionIds) { //We are querying for the attributes in batches because something can't handle WHERE IN queries with sets larger than 999 int start = 0; const int size = 999; while (start < collectionIds.size()) { const QVariantList ids = collectionIds.mid(start, size); QSqlQuery attributeQuery = getAttributeQuery(ids, mAncestorAttributes); while (attributeQuery.next()) { CollectionAttribute attr; attr.setType(attributeQuery.value(1).toByteArray()); attr.setValue(attributeQuery.value(2).toByteArray()); // qCDebug(AKONADISERVER_LOG) << "found attribute " << attr.type() << attr.value(); mCollectionAttributes.insert(attributeQuery.value(0).toLongLong(), attr); } start += size; } } static QSqlQuery getMimeTypeQuery(const QVariantList &ids) { QueryBuilder qb(CollectionMimeTypeRelation::tableName()); qb.addJoin(QueryBuilder::LeftJoin, MimeType::tableName(), MimeType::idFullColumnName(), CollectionMimeTypeRelation::rightFullColumnName()); qb.addValueCondition(CollectionMimeTypeRelation::leftFullColumnName(), Query::In, ids); qb.addColumn(CollectionMimeTypeRelation::leftFullColumnName()); qb.addColumn(CollectionMimeTypeRelation::rightFullColumnName()); qb.addColumn(MimeType::nameFullColumnName()); qb.addSortColumn(CollectionMimeTypeRelation::leftFullColumnName(), Query::Ascending); if (!qb.exec()) { throw HandlerException("Unable to retrieve mimetypes for listing"); } return qb.query(); } void List::retrieveCollections(const Collection &topParent, int depth) { /* * Retrieval of collections: * The aim is to reduce the amount of queries as much as possible, as this has the largest performance impact for large queries. * * First all collections that match the given criteria are queried * * We then filter the false positives: * ** all collections out that are not part of the tree we asked for are filtered * ** all collections that are referenced but not by this session or by the owning resource are filtered * * Finally we complete the tree by adding missing collections * * Mimetypes and attributes are also retrieved in single queries to avoid spawning two queries per collection (the N+1 problem). * Note that we're not querying attributes and mimetypes for the collections that are only included to complete the tree, * this results in no items being queried for those collections. */ const qint64 parentId = topParent.isValid() ? topParent.id() : 0; { SelectQueryBuilder qb; if (depth == 0) { qb.addValueCondition(Collection::idFullColumnName(), Query::Equals, parentId); } else if (depth == 1) { if (topParent.isValid()) { qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Equals, parentId); } else { qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant()); } } else { if (topParent.isValid()) { qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, topParent.resourceId()); } else { // Gimme gimme gimme...everything! } } //Base listings should succeed always if (depth != 0) { if (mCollectionsToSynchronize) { qb.addCondition(filterCondition(Collection::syncPrefFullColumnName())); } else if (mCollectionsToDisplay) { - qCDebug(AKONADISERVER_LOG) << "only display"; qb.addCondition(filterCondition(Collection::displayPrefFullColumnName())); } else if (mCollectionsToIndex) { qb.addCondition(filterCondition(Collection::indexPrefFullColumnName())); } else if (mEnabledCollections) { Query::Condition orCondition(Query::Or); orCondition.addValueCondition(Collection::enabledFullColumnName(), Query::Equals, true); orCondition.addValueCondition(Collection::referencedFullColumnName(), Query::Equals, true); qb.addCondition(orCondition); } if (mResource.isValid()) { qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, mResource.id()); } if (!mMimeTypes.isEmpty()) { qb.addJoin(QueryBuilder::LeftJoin, CollectionMimeTypeRelation::tableName(), CollectionMimeTypeRelation::leftColumn(), Collection::idFullColumnName()); QVariantList mimeTypeFilter; mimeTypeFilter.reserve(mMimeTypes.size()); for (MimeType::Id mtId : qAsConst(mMimeTypes)) { mimeTypeFilter << mtId; } qb.addValueCondition(CollectionMimeTypeRelation::rightColumn(), Query::In, mimeTypeFilter); qb.addGroupColumn(Collection::idFullColumnName()); } } if (!qb.exec()) { throw HandlerException("Unable to retrieve collection for listing"); } Q_FOREACH (const Collection &col, qb.result()) { mCollections.insert(col.id(), col); } } //Post filtering that we couldn't do as part of the sql query if (depth > 0) { auto it = mCollections.begin(); while (it != mCollections.end()) { if (topParent.isValid()) { //Check that each collection is linked to the root collection bool foundParent = false; //We iterate over parents to link it to topParent if possible Collection::Id id = it->parentId(); while (id > 0) { if (id == parentId) { foundParent = true; break; } Collection col = mCollections.value(id); if (!col.isValid()) { col = Collection::retrieveById(id); } id = col.parentId(); } if (!foundParent) { it = mCollections.erase(it); continue; } } //If we matched referenced collections we need to ensure the collection was referenced from this session const bool isReferencedFromSession = connection()->collectionReferenceManager()->isReferenced(it->id(), connection()->sessionId()); //The collection is referenced, but not from this session. We need to reevaluate the filter condition if (it->referenced() && !isReferencedFromSession) { //Don't include the collection when only looking for enabled collections. //However, a referenced collection should be still synchronized by the resource, so we exclude this case. if (!checkFilterCondition(*it) && !(mCollectionsToSynchronize && connection()->context()->resource().isValid())) { it = mCollections.erase(it); continue; } } ++it; } } QVariantList mimeTypeIds; QVariantList attributeIds; QVariantList ancestorIds; const int collectionSize{mCollections.size()}; mimeTypeIds.reserve(collectionSize); attributeIds.reserve(collectionSize); //We'd only require the non-leaf collections, but we don't know which those are, so we take all. ancestorIds.reserve(collectionSize); for (auto it = mCollections.cbegin(), end = mCollections.cend(); it != end; ++it) { mimeTypeIds << it.key(); attributeIds << it.key(); ancestorIds << it.key(); } if (mAncestorDepth > 0 && topParent.isValid()) { //unless depth is 0 the base collection is not part of the listing mAncestors.insert(topParent.id(), topParent); ancestorIds << topParent.id(); //We need to retrieve additional ancestors to what we already have in the tree Collection parent = topParent; for (int i = 0; i < mAncestorDepth; ++i) { if (parent.parentId() == 0) { break; } parent = parent.parent(); mAncestors.insert(parent.id(), parent); //We also require the attributes ancestorIds << parent.id(); } } QSet missingCollections; if (depth > 0) { for (const Collection &col : qAsConst(mCollections)) { if (col.parentId() != parentId && !mCollections.contains(col.parentId())) { missingCollections.insert(col.parentId()); } } } /* QSet knownIds; for (const Collection &col : mCollections) { knownIds.insert(col.id()); } qCDebug(AKONADISERVER_LOG) << "HAS:" << knownIds; qCDebug(AKONADISERVER_LOG) << "MISSING:" << missingCollections; */ //Fetch missing collections that are part of the tree while (!missingCollections.isEmpty()) { SelectQueryBuilder qb; QVariantList ids; ids.reserve(missingCollections.size()); for (qint64 id : qAsConst(missingCollections)) { ids << id; } qb.addValueCondition(Collection::idFullColumnName(), Query::In, ids); if (!qb.exec()) { throw HandlerException("Unable to retrieve collections for listing"); } missingCollections.clear(); Q_FOREACH (const Collection &missingCol, qb.result()) { mCollections.insert(missingCol.id(), missingCol); ancestorIds << missingCol.id(); attributeIds << missingCol.id(); mimeTypeIds << missingCol.id(); //We have to do another round if the parents parent is missing if (missingCol.parentId() != parentId && !mCollections.contains(missingCol.parentId())) { missingCollections.insert(missingCol.parentId()); } } } //Since we don't know when we'll need the ancestor attributes, we have to fetch them all together. //The alternative would be to query for each collection which would reintroduce the N+1 query performance problem. if (!mAncestorAttributes.isEmpty()) { retrieveAttributes(ancestorIds); } //We are querying in batches because something can't handle WHERE IN queries with sets larger than 999 const int querySizeLimit = 999; int mimetypeQueryStart = 0; int attributeQueryStart = 0; QSqlQuery mimeTypeQuery(DataStore::self()->database()); QSqlQuery attributeQuery(DataStore::self()->database()); auto it = mCollections.begin(); while (it != mCollections.end()) { const Collection col = it.value(); - // qCDebug(AKONADISERVER_LOG) << "col " << col.id(); QStringList mimeTypes; { //Get new query if necessary if (!mimeTypeQuery.isValid() && mimetypeQueryStart < mimeTypeIds.size()) { const QVariantList ids = mimeTypeIds.mid(mimetypeQueryStart, querySizeLimit); mimetypeQueryStart += querySizeLimit; mimeTypeQuery = getMimeTypeQuery(ids); mimeTypeQuery.next(); //place at first record } - // qCDebug(AKONADISERVER_LOG) << mimeTypeQuery.isValid() << mimeTypeQuery.value(0).toLongLong(); while (mimeTypeQuery.isValid() && mimeTypeQuery.value(0).toLongLong() < col.id()) { - qCDebug(AKONADISERVER_LOG) << "skipped: " << mimeTypeQuery.value(0).toLongLong() << mimeTypeQuery.value(2).toString(); if (!mimeTypeQuery.next()) { break; } } //Advance query while a mimetype for this collection is returned while (mimeTypeQuery.isValid() && mimeTypeQuery.value(0).toLongLong() == col.id()) { mimeTypes << mimeTypeQuery.value(2).toString(); if (!mimeTypeQuery.next()) { break; } } } CollectionAttribute::List attributes; { //Get new query if necessary if (!attributeQuery.isValid() && attributeQueryStart < attributeIds.size()) { const QVariantList ids = attributeIds.mid(attributeQueryStart, querySizeLimit); attributeQueryStart += querySizeLimit; attributeQuery = getAttributeQuery(ids, QSet()); attributeQuery.next(); //place at first record } - // qCDebug(AKONADISERVER_LOG) << attributeQuery.isValid() << attributeQuery.value(0).toLongLong(); while (attributeQuery.isValid() && attributeQuery.value(0).toLongLong() < col.id()) { - qCDebug(AKONADISERVER_LOG) << "skipped: " << attributeQuery.value(0).toLongLong() << attributeQuery.value(1).toByteArray(); if (!attributeQuery.next()) { break; } } //Advance query while a mimetype for this collection is returned while (attributeQuery.isValid() && attributeQuery.value(0).toLongLong() == col.id()) { CollectionAttribute attr; attr.setType(attributeQuery.value(1).toByteArray()); attr.setValue(attributeQuery.value(2).toByteArray()); attributes << attr; if (!attributeQuery.next()) { break; } } } listCollection(col, ancestorsForCollection(col), mimeTypes, attributes); it++; } } bool List::parseStream() { const auto &cmd = Protocol::cmdCast(m_command); if (!cmd.resource().isEmpty()) { mResource = Resource::retrieveByName(cmd.resource()); if (!mResource.isValid()) { return failureResponse("Unknown resource"); } } const QStringList lstMimeTypes = cmd.mimeTypes(); for (const QString &mtName : lstMimeTypes) { const MimeType mt = MimeType::retrieveByNameOrCreate(mtName); if (!mt.isValid()) { return failureResponse("Failed to create mimetype record"); } mMimeTypes.append(mt.id()); } mEnabledCollections = cmd.enabled(); mCollectionsToSynchronize = cmd.syncPref(); mCollectionsToDisplay = cmd.displayPref(); mCollectionsToIndex = cmd.indexPref(); mIncludeStatistics = cmd.fetchStats(); int depth = 0; switch (cmd.depth()) { case Protocol::FetchCollectionsCommand::BaseCollection: depth = 0; break; case Protocol::FetchCollectionsCommand::ParentCollection: depth = 1; break; case Protocol::FetchCollectionsCommand::AllCollections: depth = INT_MAX; break; } switch (cmd.ancestorsDepth()) { case Protocol::Ancestor::NoAncestor: mAncestorDepth = 0; break; case Protocol::Ancestor::ParentAncestor: mAncestorDepth = 1; break; case Protocol::Ancestor::AllAncestors: mAncestorDepth = INT_MAX; break; } mAncestorAttributes = cmd.ancestorsAttributes(); Scope scope = cmd.collections(); if (!scope.isEmpty()) { // not root Collection col; if (scope.scope() == Scope::Uid) { col = Collection::retrieveById(scope.uid()); } else if (scope.scope() == Scope::Rid) { SelectQueryBuilder qb; qb.addValueCondition(Collection::remoteIdFullColumnName(), Query::Equals, scope.rid()); qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName()); if (mCollectionsToSynchronize) { qb.addCondition(filterCondition(Collection::syncPrefFullColumnName())); } else if (mCollectionsToDisplay) { qb.addCondition(filterCondition(Collection::displayPrefFullColumnName())); } else if (mCollectionsToIndex) { qb.addCondition(filterCondition(Collection::indexPrefFullColumnName())); } if (mResource.isValid()) { qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, mResource.id()); } else if (connection()->context()->resource().isValid()) { qb.addValueCondition(Resource::idFullColumnName(), Query::Equals, connection()->context()->resource().id()); } else { return failureResponse("Cannot retrieve collection based on remote identifier without a resource context"); } if (!qb.exec()) { return failureResponse("Unable to retrieve collection for listing"); } Collection::List results = qb.result(); if (results.count() != 1) { return failureResponse(QString::number(results.count()) + QStringLiteral(" collections found")); } col = results.first(); } else if (scope.scope() == Scope::HierarchicalRid) { if (!connection()->context()->resource().isValid()) { return failureResponse("Cannot retrieve collection based on hierarchical remote identifier without a resource context"); } col = CollectionQueryHelper::resolveHierarchicalRID(scope.hridChain(), connection()->context()->resource().id()); } else { return failureResponse("Unexpected error"); } if (!col.isValid()) { return failureResponse("Collection does not exist"); } retrieveCollections(col, depth); } else { //Root folder listing if (depth != 0) { retrieveCollections(Collection(), depth); } } return successResponse(); } diff --git a/src/server/handler/modify.cpp b/src/server/handler/modify.cpp index 5e5637fa4..f826d428c 100644 --- a/src/server/handler/modify.cpp +++ b/src/server/handler/modify.cpp @@ -1,312 +1,313 @@ /* Copyright (c) 2006 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "modify.h" #include "akonadi.h" #include "connection.h" #include "handlerhelper.h" #include "cachecleaner.h" #include "collectionreferencemanager.h" #include "intervalcheck.h" #include "storage/datastore.h" #include "storage/transaction.h" #include "storage/itemretriever.h" #include "storage/selectquerybuilder.h" #include "storage/collectionqueryhelper.h" #include "search/searchmanager.h" #include "akonadiserver_debug.h" using namespace Akonadi; using namespace Akonadi::Server; bool Modify::parseStream() { const auto &cmd = Protocol::cmdCast(m_command); Collection collection = HandlerHelper::collectionFromScope(cmd.collection(), connection()); if (!collection.isValid()) { return failureResponse("No such collection"); } CacheCleanerInhibitor inhibitor(false); if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::ParentID) { const Collection newParent = Collection::retrieveById(cmd.parentId()); if (newParent.isValid() && collection.parentId() != newParent.id() && collection.resourceId() != newParent.resourceId()) { inhibitor.inhibit(); ItemRetriever retriever(connection()); retriever.setCollection(collection, true); retriever.setRetrieveFullPayload(true); if (!retriever.exec()) { throw HandlerException(retriever.lastError()); } } } DataStore *db = connection()->storageBackend(); Transaction transaction(db, QStringLiteral("MODIFY")); QList changes; bool referencedChanged = false; if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::MimeTypes) { QStringList mts = cmd.mimeTypes(); const MimeType::List currentMts = collection.mimeTypes(); bool equal = true; for (const MimeType ¤tMt : currentMts) { const int removeMts = mts.removeAll(currentMt.name()); if (removeMts > 0) { continue; } equal = false; if (!collection.removeMimeType(currentMt)) { return failureResponse("Unable to remove collection mimetype"); } } if (!db->appendMimeTypeForCollection(collection.id(), mts)) { return failureResponse("Unable to add collection mimetypes"); } if (!equal || !mts.isEmpty()) { changes.append(AKONADI_PARAM_MIMETYPE); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::CachePolicy) { bool changed = false; const Protocol::CachePolicy newCp = cmd.cachePolicy(); if (collection.cachePolicyCacheTimeout() != newCp.cacheTimeout()) { collection.setCachePolicyCacheTimeout(newCp.cacheTimeout()); changed = true; } if (collection.cachePolicyCheckInterval() != newCp.checkInterval()) { collection.setCachePolicyCheckInterval(newCp.checkInterval()); changed = true; } if (collection.cachePolicyInherit() != newCp.inherit()) { collection.setCachePolicyInherit(newCp.inherit()); changed = true; } QStringList parts = newCp.localParts(); std::sort(parts.begin(), parts.end()); const QString localParts = parts.join(QLatin1Char(' ')); if (collection.cachePolicyLocalParts() != localParts) { collection.setCachePolicyLocalParts(localParts); changed = true; } if (collection.cachePolicySyncOnDemand() != newCp.syncOnDemand()) { collection.setCachePolicySyncOnDemand(newCp.syncOnDemand()); changed = true; } if (changed) { changes.append(AKONADI_PARAM_CACHEPOLICY); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::Name) { if (cmd.name() != collection.name()) { if (!CollectionQueryHelper::hasAllowedName(collection, cmd.name(), collection.parentId())) { return failureResponse("Collection with the same name exists already"); } collection.setName(cmd.name()); changes.append(AKONADI_PARAM_NAME); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::ParentID) { if (collection.parentId() != cmd.parentId()) { if (!db->moveCollection(collection, Collection::retrieveById(cmd.parentId()))) { return failureResponse("Unable to reparent collection"); } changes.append(AKONADI_PARAM_PARENT); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::RemoteID) { if (cmd.remoteId() != collection.remoteId() && !cmd.remoteId().isEmpty()) { if (!connection()->isOwnerResource(collection)) { - qCWarning(AKONADISERVER_LOG) << "Invalid attempt to modify the collection remoteID from" << collection.remoteId() << "to" << cmd.remoteId(); + qCWarning(AKONADISERVER_LOG) << "Invalid attempt to modify the collection remoteID from" + << collection.remoteId() << "to" << cmd.remoteId(); return failureResponse("Only resources can modify remote identifiers"); } collection.setRemoteId(cmd.remoteId()); changes.append(AKONADI_PARAM_REMOTEID); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::RemoteRevision) { if (cmd.remoteRevision() != collection.remoteRevision()) { collection.setRemoteRevision(cmd.remoteRevision()); changes.append(AKONADI_PARAM_REMOTEREVISION); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::PersistentSearch) { bool changed = false; if (cmd.persistentSearchQuery() != collection.queryString()) { collection.setQueryString(cmd.persistentSearchQuery()); changed = true; } QList queryAttributes = collection.queryAttributes().toUtf8().split(' '); if (cmd.persistentSearchRemote() != queryAttributes.contains(AKONADI_PARAM_REMOTE)) { if (cmd.persistentSearchRemote()) { queryAttributes.append(AKONADI_PARAM_REMOTE); } else { queryAttributes.removeOne(AKONADI_PARAM_REMOTE); } changed = true; } if (cmd.persistentSearchRecursive() != queryAttributes.contains(AKONADI_PARAM_RECURSIVE)) { if (cmd.persistentSearchRecursive()) { queryAttributes.append(AKONADI_PARAM_RECURSIVE); } else { queryAttributes.removeOne(AKONADI_PARAM_RECURSIVE); } changed = true; } if (changed) { collection.setQueryAttributes(QString::fromLatin1(queryAttributes.join(' '))); } QStringList cols; cols.reserve(cmd.persistentSearchCollections().size()); QVector inCols = cmd.persistentSearchCollections(); std::sort(inCols.begin(), inCols.end()); for (qint64 col : qAsConst(inCols)) { cols.append(QString::number(col)); } const QString colStr = cols.join(QLatin1Char(' ')); if (colStr != collection.queryCollections()) { collection.setQueryCollections(colStr); changed = true; } if (changed || cmd.modifiedParts() & Protocol::ModifyCollectionCommand::MimeTypes) { changes.append(AKONADI_PARAM_PERSISTENTSEARCH); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::ListPreferences) { if (cmd.enabled() != collection.enabled()) { collection.setEnabled(cmd.enabled()); changes.append(AKONADI_PARAM_ENABLED); } if (cmd.syncPref() != static_cast(collection.syncPref())) { collection.setSyncPref(static_cast(cmd.syncPref())); changes.append(AKONADI_PARAM_SYNC); } if (cmd.displayPref() != static_cast(collection.displayPref())) { collection.setDisplayPref(static_cast(cmd.displayPref())); changes.append(AKONADI_PARAM_DISPLAY); } if (cmd.indexPref() != static_cast(collection.indexPref())) { collection.setIndexPref(static_cast(cmd.indexPref())); changes.append(AKONADI_PARAM_INDEX); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::Referenced) { const bool wasReferencedFromSession = connection()->collectionReferenceManager()->isReferenced(collection.id(), connection()->sessionId()); connection()->collectionReferenceManager()->referenceCollection(connection()->sessionId(), collection, cmd.referenced()); const bool referenced = connection()->collectionReferenceManager()->isReferenced(collection.id()); if (cmd.referenced() != wasReferencedFromSession) { changes.append(AKONADI_PARAM_REFERENCED); } if (referenced != collection.referenced()) { referencedChanged = true; collection.setReferenced(referenced); } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::RemovedAttributes) { Q_FOREACH (const QByteArray &attr, cmd.removedAttributes()) { if (db->removeCollectionAttribute(collection, attr)) { changes.append(attr); } } } if (cmd.modifiedParts() & Protocol::ModifyCollectionCommand::Attributes) { const QMap attrs = cmd.attributes(); for (auto iter = attrs.cbegin(), end = attrs.cend(); iter != end; ++iter) { SelectQueryBuilder qb; qb.addValueCondition(CollectionAttribute::collectionIdColumn(), Query::Equals, collection.id()); qb.addValueCondition(CollectionAttribute::typeColumn(), Query::Equals, iter.key()); if (!qb.exec()) { return failureResponse("Unable to retrieve collection attribute"); } const CollectionAttribute::List attrs = qb.result(); if (attrs.isEmpty()) { CollectionAttribute newAttr; newAttr.setCollectionId(collection.id()); newAttr.setType(iter.key()); newAttr.setValue(iter.value()); if (!newAttr.insert()) { return failureResponse("Unable to add collection attribute"); } changes.append(iter.key()); } else if (attrs.size() == 1) { CollectionAttribute currAttr = attrs.first(); if (currAttr.value() == iter.value()) { continue; } currAttr.setValue(iter.value()); if (!currAttr.update()) { return failureResponse("Unable to update collection attribute"); } changes.append(iter.key()); } else { return failureResponse("WTF: more than one attribute with the same name"); } } } if (!changes.isEmpty()) { if (collection.hasPendingChanges() && !collection.update()) { return failureResponse("Unable to update collection"); } //This must be after the collection was updated in the db. The resource will immediately request a copy of the collection. if (AkonadiServer::instance()->intervalChecker() && collection.referenced() && referencedChanged) { AkonadiServer::instance()->intervalChecker()->requestCollectionSync(collection); } db->notificationCollector()->collectionChanged(collection, changes); //For backwards compatibility. Must be after the changed notification (otherwise the compression removes it). if (changes.contains(AKONADI_PARAM_ENABLED)) { if (collection.enabled()) { db->notificationCollector()->collectionSubscribed(collection); } else { db->notificationCollector()->collectionUnsubscribed(collection); } } if (!transaction.commit()) { return failureResponse("Unable to commit transaction"); } // Only request Search update AFTER committing the transaction to avoid // transaction deadlock with SQLite if (changes.contains(AKONADI_PARAM_PERSISTENTSEARCH)) { SearchManager::instance()->updateSearch(collection); } } return successResponse(); } diff --git a/src/server/handler/store.cpp b/src/server/handler/store.cpp index aa75cd20c..c8d2ed625 100644 --- a/src/server/handler/store.cpp +++ b/src/server/handler/store.cpp @@ -1,387 +1,387 @@ /*************************************************************************** * Copyright (C) 2006 by Tobias Koenig * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "store.h" #include "connection.h" #include "handlerhelper.h" #include "storage/datastore.h" #include "storage/transaction.h" #include "storage/itemqueryhelper.h" #include "storage/selectquerybuilder.h" #include "storage/parthelper.h" #include "storage/dbconfig.h" #include "storage/itemretriever.h" #include "storage/parttypehelper.h" #include "storage/partstreamer.h" #include #include "akonadiserver_debug.h" #include #include using namespace Akonadi; using namespace Akonadi::Server; static bool payloadChanged(const QSet &changes) { for (const QByteArray &change : changes) { if (change.startsWith(AKONADI_PARAM_PLD)) { return true; } } return false; } bool Store::replaceFlags(const PimItem::List &item, const QSet &flags, bool &flagsChanged) { Flag::List flagList = HandlerHelper::resolveFlags(flags); DataStore *store = connection()->storageBackend(); if (!store->setItemsFlags(item, flagList, &flagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::replaceFlags: Unable to replace flags"; + qCWarning(AKONADISERVER_LOG) << "Store::replaceFlags: Unable to replace flags"; return false; } return true; } bool Store::addFlags(const PimItem::List &items, const QSet &flags, bool &flagsChanged) { const Flag::List flagList = HandlerHelper::resolveFlags(flags); DataStore *store = connection()->storageBackend(); if (!store->appendItemsFlags(items, flagList, &flagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::addFlags: Unable to add new item flags"; + qCWarning(AKONADISERVER_LOG) << "Store::addFlags: Unable to add new item flags"; return false; } return true; } bool Store::deleteFlags(const PimItem::List &items, const QSet &flags, bool &flagsChanged) { DataStore *store = connection()->storageBackend(); QVector flagList; flagList.reserve(flags.size()); for (auto iter = flags.cbegin(), end = flags.cend(); iter != end; ++iter) { Flag flag = Flag::retrieveByName(QString::fromUtf8(*iter)); if (!flag.isValid()) { continue; } flagList.append(flag); } if (!store->removeItemsFlags(items, flagList, &flagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::deleteFlags: Unable to remove item flags"; + qCWarning(AKONADISERVER_LOG) << "Store::deleteFlags: Unable to remove item flags"; return false; } return true; } bool Store::replaceTags(const PimItem::List &item, const Scope &tags, bool &tagsChanged) { const Tag::List tagList = HandlerHelper::tagsFromScope(tags, connection()); if (!connection()->storageBackend()->setItemsTags(item, tagList, &tagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::replaceTags: unable to replace tags"; + qCWarning(AKONADISERVER_LOG) << "Store::replaceTags: unable to replace tags"; return false; } return true; } bool Store::addTags(const PimItem::List &items, const Scope &tags, bool &tagsChanged) { const Tag::List tagList = HandlerHelper::tagsFromScope(tags, connection()); if (!connection()->storageBackend()->appendItemsTags(items, tagList, &tagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::addTags: Unable to add new item tags"; + qCWarning(AKONADISERVER_LOG) << "Store::addTags: Unable to add new item tags"; return false; } return true; } bool Store::deleteTags(const PimItem::List &items, const Scope &tags, bool &tagsChanged) { const Tag::List tagList = HandlerHelper::tagsFromScope(tags, connection()); if (!connection()->storageBackend()->removeItemsTags(items, tagList, &tagsChanged)) { - qCDebug(AKONADISERVER_LOG) << "Store::deleteTags: Unable to remove item tags"; + qCWarning(AKONADISERVER_LOG) << "Store::deleteTags: Unable to remove item tags"; return false; } return true; } bool Store::parseStream() { const auto &cmd = Protocol::cmdCast(m_command); //parseCommand(); DataStore *store = connection()->storageBackend(); Transaction transaction(store, QStringLiteral("STORE")); ExternalPartStorageTransaction storageTrx; // Set the same modification time for each item. QDateTime modificationtime = QDateTime::currentDateTimeUtc(); if (DbType::type(store->database()) != DbType::Sqlite) { // Remove milliseconds from the modificationtime. PSQL and MySQL don't // support milliseconds in DATETIME column, so FETCHed Items will report // time without milliseconds, while this command would return answer // with milliseconds modificationtime = modificationtime.addMSecs(-modificationtime.time().msec()); } // retrieve selected items SelectQueryBuilder qb; qb.setForUpdate(); ItemQueryHelper::scopeToQuery(cmd.items(), connection()->context(), qb); if (!qb.exec()) { return failureResponse("Unable to retrieve items"); } PimItem::List pimItems = qb.result(); if (pimItems.isEmpty()) { return failureResponse("No items found"); } for (int i = 0; i < pimItems.size(); ++i) { if (cmd.oldRevision() > -1) { // check for conflicts if a resources tries to overwrite an item with dirty payload const PimItem &pimItem = pimItems.at(i); if (connection()->isOwnerResource(pimItem)) { if (pimItem.dirty()) { const QString error = QStringLiteral("[LRCONFLICT] Resource %1 tries to modify item %2 (%3) (in collection %4) with dirty payload, aborting STORE."); return failureResponse( error.arg(pimItem.collection().resource().name()) .arg(pimItem.id()) .arg(pimItem.remoteId()).arg(pimItem.collectionId())); } } // check and update revisions if (pimItems.at(i).rev() != (int) cmd.oldRevision()) { const QString error = QStringLiteral("[LLCONFLICT] Resource %1 tries to modify item %2 (%3) (in collection %4) with revision %5; the item was modified elsewhere and has revision %6, aborting STORE."); return failureResponse(error.arg(pimItem.collection().resource().name()) .arg(pimItem.id()) .arg(pimItem.remoteId()).arg(pimItem.collectionId()) .arg(cmd.oldRevision()).arg(pimItems.at(i).rev())); } } } PimItem &item = pimItems.first(); QSet changes; qint64 partSizes = 0; qint64 size = 0; bool flagsChanged = false; bool tagsChanged = false; if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::AddedFlags) { if (!addFlags(pimItems, cmd.addedFlags(), flagsChanged)) { return failureResponse("Unable to add item flags"); } } if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::RemovedFlags) { if (!deleteFlags(pimItems, cmd.removedFlags(), flagsChanged)) { return failureResponse("Unable to remove item flags"); } } if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::Flags) { if (!replaceFlags(pimItems, cmd.flags(), flagsChanged)) { return failureResponse("Unable to reset flags"); } } if (flagsChanged) { changes << AKONADI_PARAM_FLAGS; } if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::AddedTags) { if (!addTags(pimItems, cmd.addedTags(), tagsChanged)) { return failureResponse("Unable to add item tags"); } } if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::RemovedTags) { if (!deleteTags(pimItems, cmd.removedTags(), tagsChanged)) { return failureResponse("Unable to remove item tags"); } } if (cmd.modifiedParts() & Protocol::ModifyItemsCommand::Tags) { if (!replaceTags(pimItems, cmd.tags(), tagsChanged)) { return failureResponse("Unable to reset item tags"); } } if (tagsChanged) { changes << AKONADI_PARAM_TAGS; } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::RemoteID) { if (item.remoteId() != cmd.remoteId() && !cmd.remoteId().isEmpty()) { if (!connection()->isOwnerResource(item)) { qCWarning(AKONADISERVER_LOG) << "Invalid attempt to modify the remoteID for item" << item.id() << "from" << item.remoteId() << "to" << cmd.remoteId(); return failureResponse("Only resources can modify remote identifiers"); } item.setRemoteId(cmd.remoteId()); changes << AKONADI_PARAM_REMOTEID; } } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::GID) { if (item.gid() != cmd.gid()) { item.setGid(cmd.gid()); } changes << AKONADI_PARAM_GID; } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::RemoteRevision) { if (item.remoteRevision() != cmd.remoteRevision()) { if (!connection()->isOwnerResource(item)) { return failureResponse("Only resources can modify remote revisions"); } item.setRemoteRevision(cmd.remoteRevision()); changes << AKONADI_PARAM_REMOTEREVISION; } } if (item.isValid() && !cmd.dirty()) { item.setDirty(false); } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::Size) { size = cmd.itemSize(); changes << AKONADI_PARAM_SIZE; } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::RemovedParts) { if (!cmd.removedParts().isEmpty()) { if (!store->removeItemParts(item, cmd.removedParts())) { return failureResponse("Unable to remove item parts"); } Q_FOREACH (const QByteArray &part, cmd.removedParts()) { changes.insert(part); } } } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::Parts) { PartStreamer streamer(connection(), item); Q_FOREACH (const QByteArray &partName, cmd.parts()) { qint64 partSize = 0; if (!streamer.stream(true, partName, partSize)) { return failureResponse(streamer.error()); } changes.insert(partName); partSizes += partSize; } } if (item.isValid() && cmd.modifiedParts() & Protocol::ModifyItemsCommand::Attributes) { PartStreamer streamer(connection(), item); const Protocol::Attributes attrs = cmd.attributes(); for (auto iter = attrs.cbegin(), end = attrs.cend(); iter != end; ++iter) { bool changed = false; if (!streamer.streamAttribute(true, iter.key(), iter.value(), &changed)) { return failureResponse(streamer.error()); } if (changed) { changes.insert(iter.key()); } } } QDateTime datetime; if (!changes.isEmpty() || cmd.invalidateCache() || !cmd.dirty()) { // update item size if (pimItems.size() == 1 && (size > 0 || partSizes > 0)) { pimItems.first().setSize(qMax(size, partSizes)); } const bool onlyRemoteIdChanged = (changes.size() == 1 && changes.contains(AKONADI_PARAM_REMOTEID)); const bool onlyRemoteRevisionChanged = (changes.size() == 1 && changes.contains(AKONADI_PARAM_REMOTEREVISION)); const bool onlyRemoteIdAndRevisionChanged = (changes.size() == 2 && changes.contains(AKONADI_PARAM_REMOTEID) && changes.contains(AKONADI_PARAM_REMOTEREVISION)); const bool onlyFlagsChanged = (changes.size() == 1 && changes.contains(AKONADI_PARAM_FLAGS)); const bool onlyGIDChanged = (changes.size() == 1 && changes.contains(AKONADI_PARAM_GID)); // If only the remote id and/or the remote revision changed, we don't have to increase the REV, // because these updates do not change the payload and can only be done by the owning resource -> no conflicts possible const bool revisionNeedsUpdate = (!changes.isEmpty() && !onlyRemoteIdChanged && !onlyRemoteRevisionChanged && !onlyRemoteIdAndRevisionChanged && !onlyGIDChanged); // run update query and prepare change notifications for (int i = 0; i < pimItems.count(); ++i) { if (revisionNeedsUpdate) { pimItems[i].setRev(pimItems[i].rev() + 1); } PimItem &item = pimItems[i]; item.setDatetime(modificationtime); item.setAtime(modificationtime); if (!connection()->isOwnerResource(item) && payloadChanged(changes)) { item.setDirty(true); } if (!item.update()) { return failureResponse("Unable to write item changes into the database"); } if (cmd.invalidateCache()) { if (!store->invalidateItemCache(item)) { return failureResponse("Unable to invalidate item cache in the database"); } } // flags change notification went separately during command parsing // GID-only changes are ignored to prevent resources from updating their storage when no actual change happened if (cmd.notify() && !changes.isEmpty() && !onlyFlagsChanged && !onlyGIDChanged) { // Don't send FLAGS notification in itemChanged changes.remove(AKONADI_PARAM_FLAGS); store->notificationCollector()->itemChanged(item, changes); } if (!cmd.noResponse()) { Protocol::ModifyItemsResponse resp; resp.setId(item.id()); resp.setNewRevision(item.rev()); sendResponse(std::move(resp)); } } if (!transaction.commit()) { return failureResponse("Cannot commit transaction."); } // Always commit storage changes (deletion) after DB transaction storageTrx.commit(); datetime = modificationtime; } else { datetime = pimItems.first().datetime(); } Protocol::ModifyItemsResponse resp; resp.setModificationDateTime(datetime); return successResponse(std::move(resp)); } diff --git a/src/server/handlerhelper.cpp b/src/server/handlerhelper.cpp index 6610fba61..f3f9235ef 100644 --- a/src/server/handlerhelper.cpp +++ b/src/server/handlerhelper.cpp @@ -1,443 +1,440 @@ /*************************************************************************** * Copyright (C) 2006 by Tobias Koenig * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "handlerhelper.h" #include "storage/countquerybuilder.h" #include "storage/datastore.h" #include "storage/selectquerybuilder.h" #include "storage/collectionstatistics.h" #include "storage/queryhelper.h" #include "storage/collectionqueryhelper.h" #include "commandcontext.h" #include "handler.h" #include "connection.h" #include "utils.h" #include #include #include using namespace Akonadi; using namespace Akonadi::Server; Collection HandlerHelper::collectionFromIdOrName(const QByteArray &id) { // id is a number bool ok = false; qint64 collectionId = id.toLongLong(&ok); if (ok) { return Collection::retrieveById(collectionId); } // id is a path QString path = QString::fromUtf8(id); // ### should be UTF-7 for real IMAP compatibility const QStringList pathParts = path.split(QLatin1Char('/'), QString::SkipEmptyParts); Collection col; for (const QString &part : pathParts) { SelectQueryBuilder qb; qb.addValueCondition(Collection::nameColumn(), Query::Equals, part); if (col.isValid()) { qb.addValueCondition(Collection::parentIdColumn(), Query::Equals, col.id()); } else { qb.addValueCondition(Collection::parentIdColumn(), Query::Is, QVariant()); } if (!qb.exec()) { return Collection(); } Collection::List list = qb.result(); if (list.count() != 1) { return Collection(); } col = list.first(); } return col; } QString HandlerHelper::pathForCollection(const Collection &col) { QStringList parts; Collection current = col; while (current.isValid()) { parts.prepend(current.name()); current = current.parent(); } return parts.join(QLatin1Char('/')); } Protocol::CachePolicy HandlerHelper::cachePolicyResponse(const Collection &col) { Protocol::CachePolicy cachePolicy; cachePolicy.setInherit(col.cachePolicyInherit()); cachePolicy.setCacheTimeout(col.cachePolicyCacheTimeout()); cachePolicy.setCheckInterval(col.cachePolicyCheckInterval()); if (!col.cachePolicyLocalParts().isEmpty()) { cachePolicy.setLocalParts(col.cachePolicyLocalParts().split(QLatin1Char(' '))); } cachePolicy.setSyncOnDemand(col.cachePolicySyncOnDemand()); return cachePolicy; } Protocol::FetchCollectionsResponse HandlerHelper::fetchCollectionsResponse(const Collection &col) { QStringList mimeTypes; mimeTypes.reserve(col.mimeTypes().size()); Q_FOREACH (const MimeType &mt, col.mimeTypes()) { mimeTypes << mt.name(); } return fetchCollectionsResponse(col, col.attributes(), false, 0, QStack(), QStack(), false, mimeTypes); } Protocol::FetchCollectionsResponse HandlerHelper::fetchCollectionsResponse(const Collection &col, const CollectionAttribute::List &attrs, bool includeStatistics, int ancestorDepth, const QStack &ancestors, const QStack &ancestorAttributes, bool isReferenced, const QStringList &mimeTypes) { Protocol::FetchCollectionsResponse response; response.setId(col.id()); response.setParentId(col.parentId()); response.setName(col.name()); response.setMimeTypes(mimeTypes); response.setRemoteId(col.remoteId()); response.setRemoteRevision(col.remoteRevision()); response.setResource(col.resource().name()); response.setIsVirtual(col.isVirtual()); if (includeStatistics) { const CollectionStatistics::Statistics stats = CollectionStatistics::self()->statistics(col); if (stats.count > -1) { Protocol::FetchCollectionStatsResponse statsResponse; statsResponse.setCount(stats.count); statsResponse.setUnseen(stats.count - stats.read); statsResponse.setSize(stats.size); response.setStatistics(statsResponse); } } if (!col.queryString().isEmpty()) { response.setSearchQuery(col.queryString()); QVector searchCols; const QStringList searchColIds = col.queryCollections().split(QLatin1Char(' ')); searchCols.reserve(searchColIds.size()); for (const QString &searchColId : searchColIds) { searchCols << searchColId.toLongLong(); } response.setSearchCollections(searchCols); } Protocol::CachePolicy cachePolicy = cachePolicyResponse(col); response.setCachePolicy(cachePolicy); if (ancestorDepth) { QVector ancestorList = HandlerHelper::ancestorsResponse(ancestorDepth, ancestors, ancestorAttributes); response.setAncestors(ancestorList); } response.setReferenced(isReferenced); response.setEnabled(col.enabled()); response.setDisplayPref(static_cast(col.displayPref())); response.setSyncPref(static_cast(col.syncPref())); response.setIndexPref(static_cast(col.indexPref())); QMap ra; for (const CollectionAttribute &attr : attrs) { ra.insert(attr.type(), attr.value()); } response.setAttributes(ra); return response; } QVector HandlerHelper::ancestorsResponse(int ancestorDepth, const QStack &_ancestors, const QStack &_ancestorsAttributes) { QVector rv; if (ancestorDepth > 0) { QStack ancestors(_ancestors); QStack ancestorAttributes(_ancestorsAttributes); for (int i = 0; i < ancestorDepth; ++i) { if (ancestors.isEmpty()) { Protocol::Ancestor ancestor; ancestor.setId(0); rv << ancestor; break; } const Collection c = ancestors.pop(); Protocol::Ancestor a; a.setId(c.id()); a.setRemoteId(c.remoteId()); a.setName(c.name()); if (!ancestorAttributes.isEmpty()) { QMap attrs; Q_FOREACH (const CollectionAttribute &attr, ancestorAttributes.pop()) { attrs.insert(attr.type(), attr.value()); } a.setAttributes(attrs); } rv << a; } } return rv; } Protocol::FetchTagsResponse HandlerHelper::fetchTagsResponse(const Tag &tag, const Protocol::TagFetchScope &tagFetchScope, Connection *connection) { Protocol::FetchTagsResponse response; response.setId(tag.id()); - qCDebug(AKONADISERVER_LOG) << "TAGFETCH IDONLY" << tagFetchScope.fetchIdOnly(); if (tagFetchScope.fetchIdOnly()) { return response; } response.setType(tag.tagType().name().toUtf8()); response.setParentId(tag.parentId()); response.setGid(tag.gid().toUtf8()); - qCDebug(AKONADISERVER_LOG) << "TAGFETCH" << tagFetchScope.fetchRemoteID() << connection; if (tagFetchScope.fetchRemoteID() && connection) { - qCDebug(AKONADISERVER_LOG) << connection->context()->resource().name(); // Fail silently if retrieving tag RID is not allowed in current context if (connection->context()->resource().isValid()) { QueryBuilder qb(TagRemoteIdResourceRelation::tableName()); qb.addColumn(TagRemoteIdResourceRelation::remoteIdColumn()); qb.addValueCondition(TagRemoteIdResourceRelation::resourceIdColumn(), Query::Equals, connection->context()->resource().id()); qb.addValueCondition(TagRemoteIdResourceRelation::tagIdColumn(), Query::Equals, tag.id()); if (!qb.exec()) { throw HandlerException("Unable to query Tag Remote ID"); } QSqlQuery query = qb.query(); // RID may not be available if (query.next()) { response.setRemoteId(Utils::variantToByteArray(query.value(0))); } } } if (tagFetchScope.fetchAllAttributes() || !tagFetchScope.attributes().isEmpty()) { QueryBuilder qb(TagAttribute::tableName()); qb.addColumns({ TagAttribute::typeFullColumnName(), TagAttribute::valueFullColumnName() }); Query::Condition cond(Query::And); cond.addValueCondition(TagAttribute::tagIdFullColumnName(), Query::Equals, tag.id()); if (!tagFetchScope.fetchAllAttributes() && !tagFetchScope.attributes().isEmpty()) { QVariantList types; const auto scope = tagFetchScope.attributes(); std::transform(scope.cbegin(), scope.cend(), std::back_inserter(types), [](const QByteArray &ba) { return QVariant(ba); }); cond.addValueCondition(TagAttribute::typeFullColumnName(), Query::In, types); } qb.addCondition(cond); if (!qb.exec()) { throw HandlerException("Unable to query Tag Attributes"); } QSqlQuery query = qb.query(); Protocol::Attributes attributes; while (query.next()) { attributes.insert(Utils::variantToByteArray(query.value(0)), Utils::variantToByteArray(query.value(1))); } response.setAttributes(attributes); } return response; } Protocol::FetchRelationsResponse HandlerHelper::fetchRelationsResponse(const Relation &relation) { Protocol::FetchRelationsResponse resp; resp.setLeft(relation.leftId()); resp.setLeftMimeType(relation.left().mimeType().name().toUtf8()); resp.setRight(relation.rightId()); resp.setRightMimeType(relation.right().mimeType().name().toUtf8()); resp.setType(relation.relationType().name().toUtf8()); return resp; } Flag::List HandlerHelper::resolveFlags(const QSet &flagNames) { Flag::List flagList; flagList.reserve(flagNames.size()); for (const QByteArray &flagName : flagNames) { const Flag flag = Flag::retrieveByNameOrCreate(QString::fromUtf8(flagName)); if (!flag.isValid()) { throw HandlerException("Unable to create flag"); } flagList.append(flag); } return flagList; } Tag::List HandlerHelper::resolveTagsByUID(const ImapSet &tags) { if (tags.isEmpty()) { return Tag::List(); } SelectQueryBuilder qb; QueryHelper::setToQuery(tags, Tag::idFullColumnName(), qb); if (!qb.exec()) { throw HandlerException("Unable to resolve tags"); } const Tag::List result = qb.result(); if (result.isEmpty()) { throw HandlerException("No tags found"); } return result; } Tag::List HandlerHelper::resolveTagsByGID(const QStringList &tagsGIDs) { Tag::List tagList; if (tagsGIDs.isEmpty()) { return tagList; } for (const QString &tagGID : tagsGIDs) { Tag::List tags = Tag::retrieveFiltered(Tag::gidColumn(), tagGID); Tag tag; if (tags.isEmpty()) { tag.setGid(tagGID); tag.setParentId(0); const TagType type = TagType::retrieveByNameOrCreate(QStringLiteral("PLAIN")); if (!type.isValid()) { throw HandlerException("Unable to create tag type"); } tag.setTagType(type); if (!tag.insert()) { throw HandlerException("Unable to create tag"); } } else if (tags.count() == 1) { tag = tags[0]; } else { // Should not happen throw HandlerException("Tag GID is not unique"); } tagList.append(tag); } return tagList; } Tag::List HandlerHelper::resolveTagsByRID(const QStringList &tagsRIDs, CommandContext *context) { Tag::List tags; if (tagsRIDs.isEmpty()) { return tags; } if (!context->resource().isValid()) { throw HandlerException("Tags can be resolved by their RID only in resource context"); } tags.reserve(tagsRIDs.size()); for (const QString &tagRID : tagsRIDs) { SelectQueryBuilder qb; Query::Condition cond; cond.addColumnCondition(Tag::idFullColumnName(), Query::Equals, TagRemoteIdResourceRelation::tagIdFullColumnName()); cond.addValueCondition(TagRemoteIdResourceRelation::resourceIdFullColumnName(), Query::Equals, context->resource().id()); qb.addJoin(QueryBuilder::LeftJoin, TagRemoteIdResourceRelation::tableName(), cond); qb.addValueCondition(TagRemoteIdResourceRelation::remoteIdFullColumnName(), Query::Equals, tagRID); if (!qb.exec()) { throw HandlerException("Unable to resolve tags"); } Tag tag; Tag::List results = qb.result(); if (results.isEmpty()) { // If the tag does not exist, we create a new one with GID matching RID Tag::List tags = resolveTagsByGID(QStringList() << tagRID); if (tags.count() != 1) { throw HandlerException("Unable to resolve tag"); } tag = tags[0]; TagRemoteIdResourceRelation rel; rel.setRemoteId(tagRID); rel.setTagId(tag.id()); rel.setResourceId(context->resource().id()); if (!rel.insert()) { throw HandlerException("Unable to create tag"); } } else if (results.count() == 1) { tag = results[0]; } else { throw HandlerException("Tag RID is not unique within this resource context"); } tags.append(tag); } return tags; } Collection HandlerHelper::collectionFromScope(const Scope &scope, Connection *connection) { if (scope.scope() == Scope::Invalid || scope.scope() == Scope::Gid) { throw HandlerException("Invalid collection scope"); } SelectQueryBuilder qb; CollectionQueryHelper::scopeToQuery(scope, connection, qb); if (!qb.exec()) { throw HandlerException("Failed to execute SQL query"); } const Collection::List c = qb.result(); if (c.isEmpty()) { return Collection(); } else if (c.count() == 1) { return c.at(0); } else { throw HandlerException("Query returned more than one reslut"); } } Tag::List HandlerHelper::tagsFromScope(const Scope &scope, Connection *connection) { if (scope.scope() == Scope::Invalid || scope.scope() == Scope::HierarchicalRid) { throw HandlerException("Invalid tag scope"); } if (scope.scope() == Scope::Uid) { return resolveTagsByUID(scope.uidSet()); } else if (scope.scope() == Scope::Gid) { return resolveTagsByGID(scope.gidSet()); } else if (scope.scope() == Scope::Rid) { return resolveTagsByRID(scope.ridSet(), connection->context()); } Q_ASSERT(false); return Tag::List(); } diff --git a/src/server/main.cpp b/src/server/main.cpp index 05140da42..76bab467d 100644 --- a/src/server/main.cpp +++ b/src/server/main.cpp @@ -1,86 +1,86 @@ /*************************************************************************** * Copyright (C) 2006 by Till Adam * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "akonadi.h" #include "akonadi_version.h" #include "akonadiserver_debug.h" #include #include #include #include #include #include #include #ifdef QT_STATICPLUGIN Q_IMPORT_PLUGIN(qsqlite3) #endif int main(int argc, char **argv) { Q_INIT_RESOURCE(akonadidb); AkCoreApplication app(argc, argv, AKONADISERVER_LOG()); app.setDescription(QStringLiteral("Akonadi Server\nDo not run manually, use 'akonadictl' instead to start/stop Akonadi.")); // Set KAboutData so that DrKonqi can report bugs KAboutData aboutData(QStringLiteral("akonadiserver"), QStringLiteral("Akonadi Server"), // we don't have any localization in the server QStringLiteral(AKONADI_VERSION_STRING), QStringLiteral("Akonadi Server"), // we don't have any localization in the server KAboutLicense::LGPL_V2); KAboutData::setApplicationData(aboutData); #if !defined(NDEBUG) const QCommandLineOption startWithoutControlOption( QStringLiteral("start-without-control"), QStringLiteral("Allow to start the Akonadi server without the Akonadi control process being available")); app.addCommandLineOptions(startWithoutControlOption); #endif app.parseCommandLine(); #if !defined(NDEBUG) if (!app.commandLineArguments().isSet(QStringLiteral("start-without-control")) && #else if (true && #endif !QDBusConnection::sessionBus().interface()->isServiceRegistered(Akonadi::DBus::serviceName(Akonadi::DBus::ControlLock))) { qCCritical(AKONADISERVER_LOG) << "Akonadi control process not found - aborting."; qCCritical(AKONADISERVER_LOG) << "If you started akonadiserver manually, try 'akonadictl start' instead."; } // Make sure we do initialization from eventloop, otherwise // org.freedesktop.Akonadi.upgrading service won't be registered to DBus at all QTimer::singleShot(0, Akonadi::Server::AkonadiServer::instance(), &Akonadi::Server::AkonadiServer::init); const int result = app.exec(); - qCDebug(AKONADISERVER_LOG) << "Shutting down AkonadiServer..."; + qCInfo(AKONADISERVER_LOG) << "Shutting down AkonadiServer..."; Akonadi::Server::AkonadiServer::instance()->quit(); Q_CLEANUP_RESOURCE(akonadidb); return result; } diff --git a/src/server/notificationmanager.cpp b/src/server/notificationmanager.cpp index 46d5aabf3..92eee5daa 100644 --- a/src/server/notificationmanager.cpp +++ b/src/server/notificationmanager.cpp @@ -1,230 +1,230 @@ /* Copyright (c) 2006 - 2007 Volker Krause Copyright (c) 2010 Michael Jansen This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "notificationmanager.h" #include "notificationsubscriber.h" #include "storage/notificationcollector.h" #include "tracer.h" #include "akonadiserver_debug.h" #include "aggregatedfetchscope.h" #include "storage/collectionstatistics.h" #include "storage/selectquerybuilder.h" #include "handler/fetchhelper.h" #include "handlerhelper.h" #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; NotificationManager::NotificationManager(StartMode startMode) : AkThread(QStringLiteral("NotificationManager"), startMode) , mTimer(nullptr) , mNotifyThreadPool(nullptr) , mDebugNotifications(0) { } NotificationManager::~NotificationManager() { quitThread(); } void NotificationManager::init() { AkThread::init(); const QString serverConfigFile = StandardDirs::serverConfigFile(StandardDirs::ReadWrite); QSettings settings(serverConfigFile, QSettings::IniFormat); mTimer = new QTimer(this); mTimer->setInterval(settings.value(QStringLiteral("NotificationManager/Interval"), 50).toInt()); mTimer->setSingleShot(true); connect(mTimer, &QTimer::timeout, this, &NotificationManager::emitPendingNotifications); mNotifyThreadPool = new QThreadPool(this); mNotifyThreadPool->setMaxThreadCount(5); mCollectionFetchScope = new AggregatedCollectionFetchScope(); mItemFetchScope = new AggregatedItemFetchScope(); mTagFetchScope = new AggregatedTagFetchScope(); } void NotificationManager::quit() { mQuitting = true; if (mEventLoop) { mEventLoop->quit(); return; } mTimer->stop(); delete mTimer; mNotifyThreadPool->clear(); mNotifyThreadPool->waitForDone(); delete mNotifyThreadPool; qDeleteAll(mSubscribers); delete mCollectionFetchScope; delete mItemFetchScope; delete mTagFetchScope; AkThread::quit(); } void NotificationManager::registerConnection(quintptr socketDescriptor) { Q_ASSERT(thread() == QThread::currentThread()); NotificationSubscriber *subscriber = new NotificationSubscriber(this, socketDescriptor); - qCDebug(AKONADISERVER_LOG) << "New notification connection (registered as" << subscriber << ")"; + qCInfo(AKONADISERVER_LOG) << "New notification connection (registered as" << subscriber << ")"; connect(subscriber, &NotificationSubscriber::notificationDebuggingChanged, this, [this](bool enabled) { if (enabled) { ++mDebugNotifications; } else { --mDebugNotifications; } Q_ASSERT(mDebugNotifications >= 0); Q_ASSERT(mDebugNotifications <= mSubscribers.count()); }); mSubscribers.push_back(subscriber); } void NotificationManager::forgetSubscriber(NotificationSubscriber *subscriber) { Q_ASSERT(QThread::currentThread() == thread()); mSubscribers.removeAll(subscriber); } void NotificationManager::slotNotify(const Protocol::ChangeNotificationList &msgs) { Q_ASSERT(QThread::currentThread() == thread()); for (const auto &msg : msgs) { switch (msg->type()) { case Protocol::Command::CollectionChangeNotification: Protocol::CollectionChangeNotification::appendAndCompress(mNotifications, msg); continue; case Protocol::Command::ItemChangeNotification: case Protocol::Command::TagChangeNotification: case Protocol::Command::RelationChangeNotification: case Protocol::Command::SubscriptionChangeNotification: case Protocol::Command::DebugChangeNotification: mNotifications.push_back(msg); continue; default: Q_ASSERT_X(false, "slotNotify", "Invalid notification type!"); continue; } } if (!mTimer->isActive()) { mTimer->start(); } } class NotifyRunnable : public QRunnable { public: explicit NotifyRunnable(NotificationSubscriber *subscriber, const Protocol::ChangeNotificationList ¬ifications) : mSubscriber(subscriber) , mNotifications(notifications) { } ~NotifyRunnable() { } void run() override { for (const auto &ntf : qAsConst(mNotifications)) { if (mSubscriber) { mSubscriber->notify(ntf); } else { break; } } } private: QPointer mSubscriber; Protocol::ChangeNotificationList mNotifications; }; void NotificationManager::emitPendingNotifications() { Q_ASSERT(QThread::currentThread() == thread()); if (mNotifications.isEmpty()) { return; } if (mDebugNotifications == 0) { for (NotificationSubscriber *subscriber : qAsConst(mSubscribers)) { if (subscriber) { mNotifyThreadPool->start(new NotifyRunnable(subscriber, mNotifications)); } } } else { // When debugging notification we have to use a non-threaded approach // so that we can work with return value of notify() for (const auto ¬ification : qAsConst(mNotifications)) { QVector listeners; for (NotificationSubscriber *subscriber : qAsConst(mSubscribers)) { if (subscriber && subscriber->notify(notification)) { listeners.push_back(subscriber->subscriber()); } } emitDebugNotification(notification, listeners); } } mNotifications.clear(); } void NotificationManager::emitDebugNotification(const Protocol::ChangeNotificationPtr &ntf, const QVector &listeners) { auto debugNtf = Protocol::DebugChangeNotificationPtr::create(); debugNtf->setNotification(ntf); debugNtf->setListeners(listeners); debugNtf->setTimestamp(QDateTime::currentMSecsSinceEpoch()); for (NotificationSubscriber *subscriber : qAsConst(mSubscribers)) { if (subscriber) { mNotifyThreadPool->start(new NotifyRunnable(subscriber, { debugNtf })); } } } diff --git a/src/server/notificationsubscriber.cpp b/src/server/notificationsubscriber.cpp index 8c1f0bcda..8d2ccb241 100644 --- a/src/server/notificationsubscriber.cpp +++ b/src/server/notificationsubscriber.cpp @@ -1,744 +1,744 @@ /* Copyright (c) 2015 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "notificationsubscriber.h" #include "akonadiserver_debug.h" #include "notificationmanager.h" #include "collectionreferencemanager.h" #include "aggregatedfetchscope.h" #include "storage/querybuilder.h" #include "utils.h" #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; QMimeDatabase NotificationSubscriber::sMimeDatabase; #define TRACE_NTF(x) //#define TRACE_NTF(x) qCDebug(AKONADISERVER_LOG) << mSubscriber << x NotificationSubscriber::NotificationSubscriber(NotificationManager *manager) : QObject() , mManager(manager) , mSocket(nullptr) , mAllMonitored(false) , mExclusive(false) , mNotificationDebugging(false) { if (mManager) { mManager->itemFetchScope()->addSubscriber(); mManager->collectionFetchScope()->addSubscriber(); mManager->tagFetchScope()->addSubscriber(); } } NotificationSubscriber::NotificationSubscriber(NotificationManager *manager, quintptr socketDescriptor) : NotificationSubscriber(manager) { mSocket = new QLocalSocket(this); connect(mSocket, &QLocalSocket::readyRead, this, &NotificationSubscriber::handleIncomingData); connect(mSocket, &QLocalSocket::disconnected, this, &NotificationSubscriber::socketDisconnected); mSocket->setSocketDescriptor(socketDescriptor); const SchemaVersion schema = SchemaVersion::retrieveAll().first(); auto hello = Protocol::HelloResponsePtr::create(); hello->setServerName(QStringLiteral("Akonadi")); hello->setMessage(QStringLiteral("Not really IMAP server")); hello->setProtocolVersion(Protocol::version()); hello->setGeneration(schema.generation()); writeCommand(0, hello); } NotificationSubscriber::~NotificationSubscriber() { QMutexLocker locker(&mLock); if (mNotificationDebugging) { Q_EMIT notificationDebuggingChanged(false); } } void NotificationSubscriber::handleIncomingData() { while (mSocket->bytesAvailable() > (int) sizeof(qint64)) { Protocol::DataStream stream(mSocket); // Ignored atm qint64 tag = -1; stream >> tag; Protocol::CommandPtr cmd; try { cmd = Protocol::deserialize(mSocket); } catch (const Akonadi::ProtocolException &e) { - qCWarning(AKONADISERVER_LOG) << "ProtocolException:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "ProtocolException while reading from notification bus for" << mSubscriber << ":" << e.what(); disconnectSubscriber(); return; } catch (const std::exception &e) { - qCWarning(AKONADISERVER_LOG) << "Unknown exception:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "Unknown exception while reading from notification bus for" << mSubscriber << ":" << e.what(); disconnectSubscriber(); return; } if (cmd->type() == Protocol::Command::Invalid) { - qCWarning(AKONADISERVER_LOG) << "Received an invalid command: resetting connection"; + qCWarning(AKONADISERVER_LOG) << "Invalid command while reading from notification bus for " << mSubscriber << ", resetting connection"; disconnectSubscriber(); return; } switch (cmd->type()) { case Protocol::Command::CreateSubscription: registerSubscriber(Protocol::cmdCast(cmd)); writeCommand(tag, Protocol::CreateSubscriptionResponsePtr::create()); break; case Protocol::Command::ModifySubscription: if (mSubscriber.isEmpty()) { - qCWarning(AKONADISERVER_LOG) << "Received ModifySubscription command before RegisterSubscriber"; + qCWarning(AKONADISERVER_LOG) << "Notification subscriber received ModifySubscription command before RegisterSubscriber"; disconnectSubscriber(); return; } modifySubscription(Protocol::cmdCast(cmd)); writeCommand(tag, Protocol::ModifySubscriptionResponsePtr::create()); break; case Protocol::Command::Logout: disconnectSubscriber(); break; default: - qCWarning(AKONADISERVER_LOG) << "Invalid command" << cmd->type() << "received by NotificationSubscriber" << mSubscriber; + qCWarning(AKONADISERVER_LOG) << "Notification subscriber for" << mSubscriber << "received an invalid command" << cmd->type(); disconnectSubscriber(); break; } } } void NotificationSubscriber::socketDisconnected() { - qCDebug(AKONADISERVER_LOG) << "Subscriber" << mSubscriber << "disconnected"; + qCInfo(AKONADISERVER_LOG) << "Subscriber" << mSubscriber << "disconnected"; disconnectSubscriber(); } void NotificationSubscriber::disconnectSubscriber() { QMutexLocker locker(&mLock); auto changeNtf = Protocol::SubscriptionChangeNotificationPtr::create(); changeNtf->setSubscriber(mSubscriber); changeNtf->setSessionId(mSession); changeNtf->setOperation(Protocol::SubscriptionChangeNotification::Remove); mManager->slotNotify({ changeNtf }); if (mSocket) { disconnect(mSocket, &QLocalSocket::disconnected, this, &NotificationSubscriber::socketDisconnected); mSocket->close(); } // Unregister ourselves from the aggregated collection fetch scope auto cfs = mManager->collectionFetchScope(); cfs->apply(mCollectionFetchScope, Protocol::CollectionFetchScope()); cfs->removeSubscriber(); auto tfs = mManager->tagFetchScope(); tfs->apply(mTagFetchScope, Protocol::TagFetchScope()); tfs->removeSubscriber(); auto ifs = mManager->itemFetchScope(); ifs->apply(mItemFetchScope, Protocol::ItemFetchScope()); ifs->removeSubscriber(); mManager->forgetSubscriber(this); deleteLater(); } void NotificationSubscriber::registerSubscriber(const Protocol::CreateSubscriptionCommand &command) { QMutexLocker locker(&mLock); - qCDebug(AKONADISERVER_LOG) << "Subscriber" << this << "identified as" << command.subscriberName(); + qCInfo(AKONADISERVER_LOG) << "Subscriber" << this << "identified as" << command.subscriberName(); mSubscriber = command.subscriberName(); mSession = command.session(); auto changeNtf = Protocol::SubscriptionChangeNotificationPtr::create(); changeNtf->setSubscriber(mSubscriber); changeNtf->setSessionId(mSession); changeNtf->setOperation(Protocol::SubscriptionChangeNotification::Add); mManager->slotNotify({ changeNtf }); } void NotificationSubscriber::modifySubscription(const Protocol::ModifySubscriptionCommand &command) { QMutexLocker locker(&mLock); const auto modifiedParts = command.modifiedParts(); #define START_MONITORING(type) \ (modifiedParts & Protocol::ModifySubscriptionCommand::ModifiedParts( \ Protocol::ModifySubscriptionCommand::type | Protocol::ModifySubscriptionCommand::Add)) #define STOP_MONITORING(type) \ (modifiedParts & Protocol::ModifySubscriptionCommand::ModifiedParts( \ Protocol::ModifySubscriptionCommand::type | Protocol::ModifySubscriptionCommand::Remove)) #define APPEND(set, newItems) \ Q_FOREACH (const auto &entity, newItems) { \ set.insert(entity); \ } #define REMOVE(set, items) \ Q_FOREACH (const auto &entity, items) { \ set.remove(entity); \ } if (START_MONITORING(Types)) { APPEND(mMonitoredTypes, command.startMonitoringTypes()) } if (STOP_MONITORING(Types)) { REMOVE(mMonitoredTypes, command.stopMonitoringTypes()) } if (START_MONITORING(Collections)) { APPEND(mMonitoredCollections, command.startMonitoringCollections()) } if (STOP_MONITORING(Collections)) { REMOVE(mMonitoredCollections, command.stopMonitoringCollections()) } if (START_MONITORING(Items)) { APPEND(mMonitoredItems, command.startMonitoringItems()) } if (STOP_MONITORING(Items)) { REMOVE(mMonitoredItems, command.stopMonitoringItems()) } if (START_MONITORING(Tags)) { APPEND(mMonitoredTags, command.startMonitoringTags()) } if (STOP_MONITORING(Tags)) { REMOVE(mMonitoredTags, command.stopMonitoringTags()) } if (START_MONITORING(Resources)) { APPEND(mMonitoredResources, command.startMonitoringResources()) } if (STOP_MONITORING(Resources)) { REMOVE(mMonitoredResources, command.stopMonitoringResources()) } if (START_MONITORING(MimeTypes)) { APPEND(mMonitoredMimeTypes, command.startMonitoringMimeTypes()) } if (STOP_MONITORING(MimeTypes)) { REMOVE(mMonitoredMimeTypes, command.stopMonitoringMimeTypes()) } if (START_MONITORING(Sessions)) { APPEND(mIgnoredSessions, command.startIgnoringSessions()) } if (STOP_MONITORING(Sessions)) { REMOVE(mIgnoredSessions, command.stopIgnoringSessions()) } if (modifiedParts & Protocol::ModifySubscriptionCommand::AllFlag) { mAllMonitored = command.allMonitored(); } if (modifiedParts & Protocol::ModifySubscriptionCommand::ExclusiveFlag) { mExclusive = command.isExclusive(); } if (modifiedParts & Protocol::ModifySubscriptionCommand::ItemFetchScope) { const auto newScope = command.itemFetchScope(); mManager->itemFetchScope()->apply(mItemFetchScope, newScope); mItemFetchScope = newScope; } if (modifiedParts & Protocol::ModifySubscriptionCommand::CollectionFetchScope) { const auto newScope = command.collectionFetchScope(); mManager->collectionFetchScope()->apply(mCollectionFetchScope, newScope); mCollectionFetchScope = newScope; } if (modifiedParts & Protocol::ModifySubscriptionCommand::TagFetchScope) { const auto newScope = command.tagFetchScope(); mManager->tagFetchScope()->apply(mTagFetchScope, newScope); mTagFetchScope = newScope; if (!newScope.fetchIdOnly()) Q_ASSERT(!mManager->tagFetchScope()->fetchIdOnly()); } if (mManager) { if (modifiedParts & Protocol::ModifySubscriptionCommand::Types) { // Did the caller just subscribed to subscription changes? if (command.startMonitoringTypes().contains(Protocol::ModifySubscriptionCommand::SubscriptionChanges)) { // If yes, then send them list of all existing subscribers Q_FOREACH (const NotificationSubscriber *subscriber, mManager->mSubscribers) { // Send them back to caller if (subscriber) { QMetaObject::invokeMethod(this, "notify", Qt::QueuedConnection, Q_ARG(Akonadi::Protocol::ChangeNotificationPtr, subscriber->toChangeNotification())); } } } if (command.startMonitoringTypes().contains(Protocol::ModifySubscriptionCommand::ChangeNotifications)) { if (!mNotificationDebugging) { mNotificationDebugging = true; Q_EMIT notificationDebuggingChanged(true); } } else if (command.stopMonitoringTypes().contains(Protocol::ModifySubscriptionCommand::ChangeNotifications)) { if (mNotificationDebugging) { mNotificationDebugging = false; Q_EMIT notificationDebuggingChanged(false); } } } // Emit subscription change notification auto changeNtf = toChangeNotification(); changeNtf->setOperation(Protocol::SubscriptionChangeNotification::Modify); mManager->slotNotify({ changeNtf }); } #undef START_MONITORING #undef STOP_MONITORING #undef APPEND #undef REMOVE } Protocol::SubscriptionChangeNotificationPtr NotificationSubscriber::toChangeNotification() const { // Assumes mLock being locked by caller auto ntf = Protocol::SubscriptionChangeNotificationPtr::create(); ntf->setSessionId(mSession); ntf->setSubscriber(mSubscriber); ntf->setOperation(Protocol::SubscriptionChangeNotification::Add); ntf->setCollections(mMonitoredCollections); ntf->setItems(mMonitoredItems); ntf->setTags(mMonitoredTags); ntf->setTypes(mMonitoredTypes); ntf->setMimeTypes(mMonitoredMimeTypes); ntf->setResources(mMonitoredResources); ntf->setIgnoredSessions(mIgnoredSessions); ntf->setAllMonitored(mAllMonitored); ntf->setExclusive(mExclusive); ntf->setItemFetchScope(mItemFetchScope); ntf->setTagFetchScope(mTagFetchScope); ntf->setCollectionFetchScope(mCollectionFetchScope); return ntf; } bool NotificationSubscriber::isCollectionMonitored(Entity::Id id) const { // Assumes mLock being locked by caller if (id < 0) { return false; } else if (mMonitoredCollections.contains(id)) { return true; } else if (mMonitoredCollections.contains(0)) { return true; } return false; } bool NotificationSubscriber::isMimeTypeMonitored(const QString &mimeType) const { // Assumes mLock being locked by caller const QMimeType mt = sMimeDatabase.mimeTypeForName(mimeType); if (mMonitoredMimeTypes.contains(mimeType)) { return true; } const QStringList lst = mt.aliases(); for (const QString &alias : lst) { if (mMonitoredMimeTypes.contains(alias)) { return true; } } return false; } bool NotificationSubscriber::isMoveDestinationResourceMonitored(const Protocol::ItemChangeNotification &msg) const { // Assumes mLock being locked by caller if (msg.operation() != Protocol::ItemChangeNotification::Move) { return false; } return mMonitoredResources.contains(msg.destinationResource()); } bool NotificationSubscriber::isMoveDestinationResourceMonitored(const Protocol::CollectionChangeNotification &msg) const { // Assumes mLock being locked by caller if (msg.operation() != Protocol::CollectionChangeNotification::Move) { return false; } return mMonitoredResources.contains(msg.destinationResource()); } bool NotificationSubscriber::acceptsItemNotification(const Protocol::ItemChangeNotification &msg) const { // Assumes mLock being locked by caller if (msg.items().isEmpty()) { return false; } if (CollectionReferenceManager::instance()->isReferenced(msg.parentCollection())) { //We always want notifications that affect the parent resource (like an item added to a referenced collection) const bool notificationForParentResource = (mSession == msg.resource()); const bool accepts = mExclusive || isCollectionMonitored(msg.parentCollection()) || isMoveDestinationResourceMonitored(msg) || notificationForParentResource; TRACE_NTF("ACCEPTS ITEM: parent col referenced" << "exclusive:" << mExclusive << "," << "parent monitored:" << isCollectionMonitored(msg.parentCollection()) << "," << "destination monitored:" << isMoveDestinationResourceMonitored(msg) << "," << "ntf for parent resource:" << notificationForParentResource << ":" << "ACCEPTED:" << accepts); return accepts; } if (mAllMonitored) { TRACE_NTF("ACCEPTS ITEM: all monitored"); return true; } if (!mMonitoredTypes.isEmpty() && !mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::ItemChanges)) { TRACE_NTF("ACCEPTS ITEM: REJECTED - Item changes not monitored"); return false; } // we have a resource or mimetype filter if (!mMonitoredResources.isEmpty() || !mMonitoredMimeTypes.isEmpty()) { if (mMonitoredResources.contains(msg.resource())) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED - resource monitored"); return true; } if (isMoveDestinationResourceMonitored(msg)) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED: move destination monitored"); return true; } Q_FOREACH (const auto &item, msg.items()) { if (isMimeTypeMonitored(item.mimeType())) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED - mimetype monitored"); return true; } } TRACE_NTF("ACCEPTS ITEM: REJECTED: resource nor mimetype monitored"); return false; } // we explicitly monitor that item or the collections it's in Q_FOREACH (const auto &item, msg.items()) { if (mMonitoredItems.contains(item.id())) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED: item explicitly monitored"); return true; } } if (isCollectionMonitored(msg.parentCollection())) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED: parent collection monitored"); return true; } if (isCollectionMonitored(msg.parentDestCollection())) { TRACE_NTF("ACCEPTS ITEM: ACCEPTED: destination collection monitored"); return true; } TRACE_NTF("ACCEPTS ITEM: REJECTED"); return false; } bool NotificationSubscriber::acceptsCollectionNotification(const Protocol::CollectionChangeNotification &msg) const { // Assumes mLock being locked by caller const auto &collection = msg.collection(); if (collection.id() < 0) { return false; } // HACK: We need to dispatch notifications about disabled collections to SOME // agents (that's what we have the exclusive subscription for) - but because // querying each Collection from database would be expensive, we use the // metadata hack to transfer this information from NotificationCollector if (msg.metadata().contains("DISABLED") && (msg.operation() != Protocol::CollectionChangeNotification::Unsubscribe) && !msg.changedParts().contains("ENABLED")) { // Exclusive subscriber always gets it if (mExclusive) { return true; } //Deliver the notification if referenced from this session if (CollectionReferenceManager::instance()->isReferenced(collection.id(), mSession)) { return true; } //Exclusive subscribers still want the notification if (mExclusive && CollectionReferenceManager::instance()->isReferenced(collection.id())) { return true; } //The session belonging to this monitor referenced or dereferenced the collection. We always want this notification. //The referencemanager no longer holds a reference, so we have to check this way. if (msg.changedParts().contains(AKONADI_PARAM_REFERENCED) && mSession == msg.sessionId()) { return true; } // If the collection is not referenced, monitored or the subscriber is not // exclusive (i.e. if we got here), then the subscriber does not care about // this one, so drop it return false; } if (mAllMonitored) { return true; } if (!mMonitoredTypes.isEmpty() && !mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::CollectionChanges)) { return false; } // we have a resource filter if (!mMonitoredResources.isEmpty()) { const bool resourceMatches = mMonitoredResources.contains(msg.resource()) || isMoveDestinationResourceMonitored(msg); // a bit hacky, but match the behaviour from the item case, // if resource is the only thing we are filtering on, stop here, and if the resource filter matched, of course if (mMonitoredMimeTypes.isEmpty() || resourceMatches) { return resourceMatches; } // else continue } // we explicitly monitor that collection, or all of them if (isCollectionMonitored(collection.id())) { return true; } return isCollectionMonitored(msg.parentCollection()) || isCollectionMonitored(msg.parentDestCollection()); } bool NotificationSubscriber::acceptsTagNotification(const Protocol::TagChangeNotification &msg) const { // Assumes mLock being locked by caller if (msg.tag().id() < 0) { return false; } // Special handling for Tag removal notifications: When a Tag is removed, // a notification is emitted for each Resource that owns the tag (i.e. // each resource that owns a Tag RID - Tag RIDs are resource-specific). // Additionally then we send one more notification without any RID that is // destined for regular applications (which don't know anything about Tag RIDs) if (msg.operation() == Protocol::TagChangeNotification::Remove) { // HACK: Since have no way to determine which resource this NotificationSource // belongs to, we are abusing the fact that each resource ignores it's own // main session, which is called the same name as the resource. // If there are any ignored sessions, but this notification does not have // a specific resource set, then we ignore it, as this notification is // for clients, not resources (does not have tag RID) if (!mIgnoredSessions.isEmpty() && msg.resource().isEmpty()) { return false; } // If this source ignores a session (i.e. we assume it is a resource), // but this notification is for another resource, then we ignore it if (!msg.resource().isEmpty() && !mIgnoredSessions.contains(msg.resource())) { return false; } // Now we got here, which means that this notification either has empty // resource, i.e. it is destined for a client applications, or it's // destined for resource that we *think* (see the hack above) this // NotificationSource belongs too. Which means we approve this notification, // but it can still be discarded in the generic Tag notification filter // below } if (mAllMonitored) { return true; } if (!mMonitoredTypes.isEmpty() && !mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::TagChanges)) { return false; } if (mMonitoredTags.isEmpty()) { return true; } if (mMonitoredTags.contains(msg.tag().id())) { return true; } return true; } bool NotificationSubscriber::acceptsRelationNotification(const Protocol::RelationChangeNotification &msg) const { // Assumes mLock being locked by caller Q_UNUSED(msg); if (mAllMonitored) { return true; } if (!mMonitoredTypes.isEmpty() && !mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::RelationChanges)) { return false; } return true; } bool NotificationSubscriber::acceptsSubscriptionNotification(const Protocol::SubscriptionChangeNotification &msg) const { // Assumes mLock being locked by caller Q_UNUSED(msg); // Unlike other types, subscription notifications must be explicitly enabled // by caller and are excluded from "monitor all" as well return mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::SubscriptionChanges); } bool NotificationSubscriber::acceptsDebugChangeNotification(const Protocol::DebugChangeNotification &msg) const { // Assumes mLock being locked by caller // We should never end up sending debug notification about a debug notification. // This could get very messy very quickly... Q_ASSERT(msg.notification()->type() != Protocol::Command::DebugChangeNotification); if (msg.notification()->type() == Protocol::Command::DebugChangeNotification) { return false; } // Unlike other types, debug change notifications must be explicitly enabled // by caller and are excluded from "monitor all" as well return mMonitoredTypes.contains(Protocol::ModifySubscriptionCommand::ChangeNotifications); } bool NotificationSubscriber::acceptsNotification(const Protocol::ChangeNotification &msg) const { // Assumes mLock being locked // Uninitialized subscriber gets nothing if (mSubscriber.isEmpty()) { return false; } // session is ignored // TODO: Should this affect SubscriptionChangeNotification and DebugChangeNotification? if (mIgnoredSessions.contains(msg.sessionId())) { return false; } switch (msg.type()) { case Protocol::Command::ItemChangeNotification: return acceptsItemNotification(static_cast(msg)); case Protocol::Command::CollectionChangeNotification: return acceptsCollectionNotification(static_cast(msg)); case Protocol::Command::TagChangeNotification: return acceptsTagNotification(static_cast(msg)); case Protocol::Command::RelationChangeNotification: return acceptsRelationNotification(static_cast(msg)); case Protocol::Command::SubscriptionChangeNotification: return acceptsSubscriptionNotification(static_cast(msg)); case Protocol::Command::DebugChangeNotification: return acceptsDebugChangeNotification(static_cast(msg)); default: - qCDebug(AKONADISERVER_LOG) << "Received invalid change notification!"; + qCWarning(AKONADISERVER_LOG) << "NotificationSubscriber" << mSubscriber << "received an invalid notification type" << msg.type(); return false; } } Protocol::CollectionChangeNotificationPtr NotificationSubscriber::customizeCollection(const Protocol::CollectionChangeNotificationPtr &ntf) { const bool isReferencedFromSession = CollectionReferenceManager::instance()->isReferenced(ntf->collection().id(), mSession); if (isReferencedFromSession != ntf->collection().referenced()) { auto copy = Protocol::CollectionChangeNotificationPtr::create(*ntf); auto copyCol = ntf->collection(); copyCol.setReferenced(isReferencedFromSession); copy->setCollection(std::move(copyCol)); return copy; } return ntf; } bool NotificationSubscriber::notify(const Protocol::ChangeNotificationPtr ¬ification) { // Guard against this object being deleted while we are waiting for the lock QPointer ptr(this); QMutexLocker locker(&mLock); if (!ptr) { return false; } if (acceptsNotification(*notification)) { auto ntf = notification; if (ntf->type() == Protocol::Command::CollectionChangeNotification) { ntf = customizeCollection(notification.staticCast()); } QMetaObject::invokeMethod(this, "writeNotification", Qt::QueuedConnection, Q_ARG(Akonadi::Protocol::ChangeNotificationPtr, ntf)); return true; } return false; } void NotificationSubscriber::writeNotification(const Protocol::ChangeNotificationPtr ¬ification) { // tag chosen by fair dice roll writeCommand(4, notification); } void NotificationSubscriber::writeCommand(qint64 tag, const Protocol::CommandPtr &cmd) { Q_ASSERT(QThread::currentThread() == thread()); Protocol::DataStream stream(mSocket); stream << tag; try { Protocol::serialize(mSocket, cmd); if (!mSocket->waitForBytesWritten()) { if (mSocket->state() == QLocalSocket::ConnectedState) { - qCWarning(AKONADISERVER_LOG) << "Notification socket write timeout!"; + qCWarning(AKONADISERVER_LOG) << "NotificationSubscriber for" << mSubscriber << ": timeout writing into stream"; } else { // client has disconnected, just discard the message } } } catch (const ProtocolException &e) { - qCWarning(AKONADISERVER_LOG) << "Notification protocol exception:" << e.what(); + qCWarning(AKONADISERVER_LOG) << "ProtocolException while writing into stream for subscriber" << mSubscriber << ":" << e.what(); } } diff --git a/src/server/search/searchmanager.cpp b/src/server/search/searchmanager.cpp index e97f914f1..8b9f4c72f 100644 --- a/src/server/search/searchmanager.cpp +++ b/src/server/search/searchmanager.cpp @@ -1,456 +1,455 @@ /* Copyright (c) 2010 Volker Krause Copyright (c) 2013 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "searchmanager.h" #include "abstractsearchplugin.h" #include "akonadiserver_search_debug.h" #include "agentsearchengine.h" #include "notificationmanager.h" #include "dbusconnectionpool.h" #include "searchrequest.h" #include "searchtaskmanager.h" #include "storage/datastore.h" #include "storage/querybuilder.h" #include "storage/transaction.h" #include "storage/selectquerybuilder.h" #include "handler/searchhelper.h" #include #include #include #include #include #include Q_DECLARE_METATYPE(Akonadi::Server::NotificationCollector *) using namespace Akonadi; using namespace Akonadi::Server; SearchManager *SearchManager::sInstance = nullptr; Q_DECLARE_METATYPE(Collection) SearchManager::SearchManager(const QStringList &searchEngines, QObject *parent) : AkThread(QStringLiteral("SearchManager"), AkThread::ManualStart, QThread::InheritPriority, parent) , mEngineNames(searchEngines), mSearchUpdateTimer(nullptr) { qRegisterMetaType(); Q_ASSERT(sInstance == nullptr); sInstance = this; // We load search plugins (as in QLibrary::load()) in the main thread so that // static initialization happens in the QApplication thread loadSearchPlugins(); // Register to DBus on the main thread connection - otherwise we don't appear // on the service. QDBusConnection conn = QDBusConnection::sessionBus(); conn.registerObject(QStringLiteral("/SearchManager"), this, QDBusConnection::ExportAllSlots); // Delay-call init() startThread(); } void SearchManager::init() { AkThread::init(); mEngines.reserve(mEngineNames.size()); for (const QString &engineName : qAsConst(mEngineNames)) { if (engineName == QLatin1String("Agent")) { mEngines.append(new AgentSearchEngine); } else { qCCritical(AKONADISERVER_SEARCH_LOG) << "Unknown search engine type: " << engineName; } } initSearchPlugins(); // The timer will tick 15 seconds after last change notification. If a new notification // is delivered in the meantime, the timer is reset mSearchUpdateTimer = new QTimer(this); mSearchUpdateTimer->setInterval(15 * 1000); mSearchUpdateTimer->setSingleShot(true); connect(mSearchUpdateTimer, &QTimer::timeout, this, &SearchManager::searchUpdateTimeout); } void SearchManager::quit() { QDBusConnection conn = DBusConnectionPool::threadConnection(); conn.unregisterObject(QStringLiteral("/SearchManager"), QDBusConnection::UnregisterTree); conn.disconnectFromBus(conn.name()); // Make sure all children are deleted within context of this thread qDeleteAll(children()); qDeleteAll(mEngines); qDeleteAll(mPlugins); /* * FIXME: Unloading plugin messes up some global statics from client libs * and causes crash on Akonadi shutdown (below main). Keeping the plugins * loaded is not really a big issue as this is only invoked on server shutdown * anyway, so we are not leaking any memory. Q_FOREACH (QPluginLoader *loader, mPluginLoaders) { loader->unload(); delete loader; } */ AkThread::quit(); } SearchManager::~SearchManager() { quitThread(); sInstance = nullptr; } SearchManager *SearchManager::instance() { Q_ASSERT(sInstance); return sInstance; } void SearchManager::registerInstance(const QString &id) { SearchTaskManager::instance()->registerInstance(id); } void SearchManager::unregisterInstance(const QString &id) { SearchTaskManager::instance()->unregisterInstance(id); } QVector SearchManager::searchPlugins() const { return mPlugins; } void SearchManager::loadSearchPlugins() { QStringList loadedPlugins; const QString pluginOverride = QString::fromLatin1(qgetenv("AKONADI_OVERRIDE_SEARCHPLUGIN")); if (!pluginOverride.isEmpty()) { - qCDebug(AKONADISERVER_SEARCH_LOG) << "Overriding the search plugins with: " << pluginOverride; + qCInfo(AKONADISERVER_SEARCH_LOG) << "Overriding the search plugins with: " << pluginOverride; } const QStringList dirs = QCoreApplication::libraryPaths(); for (const QString &pluginDir : dirs) { QDir dir(pluginDir + QLatin1String("/akonadi")); const QStringList fileNames = dir.entryList(QDir::Files); qCDebug(AKONADISERVER_SEARCH_LOG) << "SEARCH MANAGER: searching in " << pluginDir + QLatin1String("/akonadi") << ":" << fileNames; for (const QString &fileName : fileNames) { const QString filePath = pluginDir % QLatin1String("/akonadi/") % fileName; std::unique_ptr loader(new QPluginLoader(filePath)); const QVariantMap metadata = loader->metaData().value(QStringLiteral("MetaData")).toVariant().toMap(); if (metadata.value(QStringLiteral("X-Akonadi-PluginType")).toString() != QLatin1String("SearchPlugin")) { - qCDebug(AKONADISERVER_SEARCH_LOG) << "===>" << fileName << metadata.value(QStringLiteral("X-Akonadi-PluginType")).toString(); continue; } const QString libraryName = metadata.value(QStringLiteral("X-Akonadi-Library")).toString(); if (loadedPlugins.contains(libraryName)) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Already loaded one version of this plugin, skipping: " << libraryName; continue; } // When search plugin override is active, ignore all plugins except for the override if (!pluginOverride.isEmpty()) { if (libraryName != pluginOverride) { qCDebug(AKONADISERVER_SEARCH_LOG) << libraryName << "skipped because of AKONADI_OVERRIDE_SEARCHPLUGIN"; continue; } // When there's no override, only load plugins enabled by default } else if (metadata.value(QStringLiteral("X-Akonadi-LoadByDefault"), true).toBool() == false) { continue; } if (!loader->load()) { qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << libraryName << ":" << loader->errorString(); continue; } mPluginLoaders << loader.release(); loadedPlugins << libraryName; } } } void SearchManager::initSearchPlugins() { for (QPluginLoader *loader : qAsConst(mPluginLoaders)) { if (!loader->load()) { qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << loader->fileName() << ":" << loader->errorString(); continue; } AbstractSearchPlugin *plugin = qobject_cast(loader->instance()); if (!plugin) { qCCritical(AKONADISERVER_SEARCH_LOG) << loader->fileName() << "is not a valid Akonadi search plugin"; continue; } qCDebug(AKONADISERVER_SEARCH_LOG) << "SearchManager: loaded search plugin" << loader->fileName(); mPlugins << plugin; } } void SearchManager::scheduleSearchUpdate() { // Reset if the timer is active (use QueuedConnection to invoke start() from // the thread the QTimer lives in instead of caller's thread, otherwise crashes // and weird things can happen. #if QT_VERSION >= QT_VERSION_CHECK(5, 10, 0) QMetaObject::invokeMethod(mSearchUpdateTimer, QOverload<>::of(&QTimer::start), Qt::QueuedConnection); #else QMetaObject::invokeMethod(mSearchUpdateTimer, "start", Qt::QueuedConnection); #endif } void SearchManager::searchUpdateTimeout() { // Get all search collections, that is subcollections of "Search", which always has ID 1 const Collection::List collections = Collection::retrieveFiltered(Collection::parentIdFullColumnName(), 1); for (const Collection &collection : collections) { updateSearchAsync(collection); } } void SearchManager::updateSearchAsync(const Collection &collection) { #if QT_VERSION >= QT_VERSION_CHECK(5, 10, 0) QMetaObject::invokeMethod(this, [this, collection]() { updateSearchImpl(collection); }, Qt::QueuedConnection); #else QMetaObject::invokeMethod(this, "updateSearchImpl", Qt::QueuedConnection, Q_ARG(Collection, collection)); #endif } void SearchManager::updateSearch(const Collection &collection) { mLock.lock(); if (mUpdatingCollections.contains(collection.id())) { mLock.unlock(); return; // FIXME: If another thread already requested an update, we return to the caller before the // search update is performed; this contradicts the docs } mUpdatingCollections.insert(collection.id()); mLock.unlock(); #if QT_VERSION >= QT_VERSION_CHECK(5, 10, 0) QMetaObject::invokeMethod(this, [this, collection]() { updateSearchImpl(collection); }, Qt::BlockingQueuedConnection); #else QMetaObject::invokeMethod(this, "updateSearchImpl", Qt::BlockingQueuedConnection, Q_ARG(Collection, collection)); #endif mLock.lock(); mUpdatingCollections.remove(collection.id()); mLock.unlock(); } void SearchManager::updateSearchImpl(const Collection &collection) { if (collection.queryString().size() >= 32768) { qCWarning(AKONADISERVER_SEARCH_LOG) << "The query is at least 32768 chars long, which is the maximum size supported by the akonadi db schema. The query is therefore most likely truncated and will not be executed."; return; } if (collection.queryString().isEmpty()) { return; } const QStringList queryAttributes = collection.queryAttributes().split(QLatin1Char(' ')); const bool remoteSearch = queryAttributes.contains(QStringLiteral(AKONADI_PARAM_REMOTE)); bool recursive = queryAttributes.contains(QStringLiteral(AKONADI_PARAM_RECURSIVE)); QStringList queryMimeTypes; const QVector mimeTypes = collection.mimeTypes(); queryMimeTypes.reserve(mimeTypes.count()); for (const MimeType &mt : mimeTypes) { queryMimeTypes << mt.name(); } QVector queryAncestors; if (collection.queryCollections().isEmpty()) { queryAncestors << 0; recursive = true; } else { const QStringList collectionIds = collection.queryCollections().split(QLatin1Char(' ')); queryAncestors.reserve(collectionIds.count()); for (const QString &colId : collectionIds) { queryAncestors << colId.toLongLong(); } } // Always query the given collections QVector queryCollections = queryAncestors; if (recursive) { // Resolve subcollections if necessary queryCollections += SearchHelper::matchSubcollectionsByMimeType(queryAncestors, queryMimeTypes); } //This happens if we try to search a virtual collection in recursive mode (because virtual collections are excluded from listCollectionsRecursive) if (queryCollections.isEmpty()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "No collections to search, you're probably trying to search a virtual collection."; return; } // Query all plugins for search results SearchRequest request("searchUpdate-" + QByteArray::number(QDateTime::currentDateTimeUtc().toTime_t())); request.setCollections(queryCollections); request.setMimeTypes(queryMimeTypes); request.setQuery(collection.queryString()); request.setRemoteSearch(remoteSearch); request.setStoreResults(true); request.setProperty("SearchCollection", QVariant::fromValue(collection)); connect(&request, &SearchRequest::resultsAvailable, this, &SearchManager::searchUpdateResultsAvailable); request.exec(); // blocks until all searches are done const QSet results = request.results(); // Get all items in the collection QueryBuilder qb(CollectionPimItemRelation::tableName()); qb.addColumn(CollectionPimItemRelation::rightColumn()); qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id()); if (!qb.exec()) { return; } Transaction transaction(DataStore::self(), QStringLiteral("UPDATE SEARCH")); // Unlink all items that were not in search results from the collection QVariantList toRemove; while (qb.query().next()) { const qint64 id = qb.query().value(0).toLongLong(); if (!results.contains(id)) { toRemove << id; Collection::removePimItem(collection.id(), id); } } if (!transaction.commit()) { return; } if (!toRemove.isEmpty()) { SelectQueryBuilder qb; qb.addValueCondition(PimItem::idFullColumnName(), Query::In, toRemove); if (!qb.exec()) { return; } const QVector removedItems = qb.result(); DataStore::self()->notificationCollector()->itemsUnlinked(removedItems, collection); } - qCDebug(AKONADISERVER_SEARCH_LOG) << "Search update finished"; - qCDebug(AKONADISERVER_SEARCH_LOG) << "All results:" << results.count(); - qCDebug(AKONADISERVER_SEARCH_LOG) << "Removed results:" << toRemove.count(); + qCInfo(AKONADISERVER_SEARCH_LOG) << "Search update for collection" << collection.name() + << "(" << collection.id() << ") finished:" + << "all results: " << results.count() << ", removed results:" << toRemove.count(); } void SearchManager::searchUpdateResultsAvailable(const QSet &results) { const Collection collection = sender()->property("SearchCollection").value(); qCDebug(AKONADISERVER_SEARCH_LOG) << "searchUpdateResultsAvailable" << collection.id() << results.count() << "results"; QSet newMatches = results; QSet existingMatches; { QueryBuilder qb(CollectionPimItemRelation::tableName()); qb.addColumn(CollectionPimItemRelation::rightColumn()); qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id()); if (!qb.exec()) { return; } while (qb.query().next()) { const qint64 id = qb.query().value(0).toLongLong(); if (newMatches.contains(id)) { existingMatches << id; } } } qCDebug(AKONADISERVER_SEARCH_LOG) << "Got" << newMatches.count() << "results, out of which" << existingMatches.count() << "are already in the collection"; newMatches = newMatches - existingMatches; if (newMatches.isEmpty()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (fast path)"; return; } Transaction transaction(DataStore::self(), QStringLiteral("PUSH SEARCH RESULTS"), !DataStore::self()->inTransaction()); // First query all the IDs we got from search plugin/agent against the DB. // This will remove IDs that no longer exist in the DB. QVariantList newMatchesVariant; newMatchesVariant.reserve(newMatches.count()); for (qint64 id : qAsConst(newMatches)) { newMatchesVariant << id; } SelectQueryBuilder qb; qb.addValueCondition(PimItem::idFullColumnName(), Query::In, newMatchesVariant); if (!qb.exec()) { return; } const auto items = qb.result(); if (items.count() != newMatches.count()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Search backend returned" << (newMatches.count() - items.count()) << "results that no longer exist in Akonadi."; qCDebug(AKONADISERVER_SEARCH_LOG) << "Please reindex collection" << collection.id(); // TODO: Request the reindexing directly from here } if (items.isEmpty()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (no existing result)"; return; } for (const auto &item : items) { Collection::addPimItem(collection.id(), item.id()); } if (!transaction.commit()) { - qCDebug(AKONADISERVER_SEARCH_LOG) << "Failed to commit transaction"; + qCWarning(AKONADISERVER_SEARCH_LOG) << "Failed to commit search results transaction"; return; } DataStore::self()->notificationCollector()->itemsLinked(items, collection); // Force collector to dispatch the notification now DataStore::self()->notificationCollector()->dispatchNotifications(); qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results:" << items.count(); } diff --git a/src/server/search/searchrequest.cpp b/src/server/search/searchrequest.cpp index 32e32a884..8d2671cc9 100644 --- a/src/server/search/searchrequest.cpp +++ b/src/server/search/searchrequest.cpp @@ -1,159 +1,158 @@ /* Copyright (c) 2013, 2014 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "searchrequest.h" #include "searchtaskmanager.h" #include "abstractsearchplugin.h" #include "searchmanager.h" #include "connection.h" #include "akonadiserver_search_debug.h" using namespace Akonadi::Server; SearchRequest::SearchRequest(const QByteArray &connectionId) : mConnectionId(connectionId) , mRemoteSearch(true) , mStoreResults(false) { } SearchRequest::~SearchRequest() { } QByteArray SearchRequest::connectionId() const { return mConnectionId; } void SearchRequest::setQuery(const QString &query) { mQuery = query; } QString SearchRequest::query() const { return mQuery; } void SearchRequest::setCollections(const QVector &collectionsIds) { mCollections = collectionsIds; } QVector SearchRequest::collections() const { return mCollections; } void SearchRequest::setMimeTypes(const QStringList &mimeTypes) { mMimeTypes = mimeTypes; } QStringList SearchRequest::mimeTypes() const { return mMimeTypes; } void SearchRequest::setRemoteSearch(bool remote) { mRemoteSearch = remote; } bool SearchRequest::remoteSearch() const { return mRemoteSearch; } void SearchRequest::setStoreResults(bool storeResults) { mStoreResults = storeResults; } QSet SearchRequest::results() const { return mResults; } void SearchRequest::emitResults(const QSet &results) { Q_EMIT resultsAvailable(results); if (mStoreResults) { mResults.unite(results); } } void SearchRequest::searchPlugins() { const QVector plugins = SearchManager::instance()->searchPlugins(); for (AbstractSearchPlugin *plugin : plugins) { const QSet result = plugin->search(mQuery, mCollections, mMimeTypes); emitResults(result); } } void SearchRequest::exec() { - qCDebug(AKONADISERVER_SEARCH_LOG) << "Executing search" << mConnectionId; + qCInfo(AKONADISERVER_SEARCH_LOG) << "Executing search" << mConnectionId; //TODO should we move this to the AgentSearchManager as well? If we keep it here the agents can be searched in parallel //since the plugin search is executed in this thread directly. searchPlugins(); // If remote search is disabled, just finish here after searching the plugins if (!mRemoteSearch) { - qCDebug(AKONADISERVER_SEARCH_LOG) << "Search done" << mConnectionId << "(without remote search)"; + qCInfo(AKONADISERVER_SEARCH_LOG) << "Search " << mConnectionId << "done (without remote search)"; return; } SearchTask task; task.id = mConnectionId; task.query = mQuery; task.mimeTypes = mMimeTypes; task.collections = mCollections; task.complete = false; SearchTaskManager::instance()->addTask(&task); task.sharedLock.lock(); Q_FOREVER { if (task.complete) { - qCDebug(AKONADISERVER_SEARCH_LOG) << "All queries processed!"; break; - } else { - task.notifier.wait(&task.sharedLock); - - qCDebug(AKONADISERVER_SEARCH_LOG) << task.pendingResults.count() << "search results available in search" << task.id; - if (!task.pendingResults.isEmpty()) { - emitResults(task.pendingResults); - } - task.pendingResults.clear(); } + + task.notifier.wait(&task.sharedLock); + + qCDebug(AKONADISERVER_SEARCH_LOG) << task.pendingResults.count() << "search results available in search" << task.id; + if (!task.pendingResults.isEmpty()) { + emitResults(task.pendingResults); + } + task.pendingResults.clear(); } if (!task.pendingResults.isEmpty()) { emitResults(task.pendingResults); } task.sharedLock.unlock(); - qCDebug(AKONADISERVER_SEARCH_LOG) << "Search done" << mConnectionId; + qCInfo(AKONADISERVER_SEARCH_LOG) << "Search" << mConnectionId << "done (with remote search)"; } diff --git a/src/server/storage/datastore.cpp b/src/server/storage/datastore.cpp index 4c0397dd8..293f0ed85 100644 --- a/src/server/storage/datastore.cpp +++ b/src/server/storage/datastore.cpp @@ -1,1496 +1,1553 @@ /*************************************************************************** * Copyright (C) 2006 by Andreas Gungl * * Copyright (C) 2007 by Robert Zwerus * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "datastore.h" #include "akonadi.h" #include "dbconfig.h" #include "dbinitializer.h" #include "dbupdater.h" #include "notificationmanager.h" #include "tracer.h" #include "transaction.h" #include "selectquerybuilder.h" #include "handlerhelper.h" #include "countquerybuilder.h" #include "parthelper.h" #include "handler.h" #include "collectionqueryhelper.h" #include "akonadischema.h" #include "parttypehelper.h" #include "querycache.h" #include "queryhelper.h" #include "akonadiserver_debug.h" #include "storagedebugger.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; static QMutex sTransactionMutex; bool DataStore::s_hasForeignKeyConstraints = false; QThreadStorage DataStore::sInstances; #define TRANSACTION_MUTEX_LOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.lock() #define TRANSACTION_MUTEX_UNLOCK if ( DbType::isSystemSQLite( m_database ) ) sTransactionMutex.unlock() #define setBoolPtr(ptr, val) \ { \ if ((ptr)) { \ *(ptr) = (val); \ } \ } DataStore *DataStoreFactory::createStore() { return new DataStore(); } std::unique_ptr DataStore::sFactory = std::make_unique(); /*************************************************************************** * DataStore * ***************************************************************************/ DataStore::DataStore() : QObject() , m_dbOpened(false) , m_transactionLevel(0) , m_keepAliveTimer(nullptr) { if (DbConfig::configuredDatabase()->driverName() == QLatin1String("QMYSQL")) { // Send a dummy query to MySQL every 1 hour to keep the connection alive, // otherwise MySQL just drops the connection and our subsequent queries fail // without properly reporting the error m_keepAliveTimer = new QTimer(this); m_keepAliveTimer->setInterval(3600 * 1000); QObject::connect(m_keepAliveTimer, &QTimer::timeout, this, &DataStore::sendKeepAliveQuery); } } DataStore::~DataStore() { if (m_dbOpened) { close(); } } void DataStore::open() { m_connectionName = QUuid::createUuid().toString() + QString::number(reinterpret_cast(QThread::currentThread())); Q_ASSERT(!QSqlDatabase::contains(m_connectionName)); m_database = QSqlDatabase::addDatabase(DbConfig::configuredDatabase()->driverName(), m_connectionName); DbConfig::configuredDatabase()->apply(m_database); if (!m_database.isValid()) { m_dbOpened = false; return; } m_dbOpened = m_database.open(); if (!m_dbOpened) { debugLastDbError("Cannot open database."); } else { qCDebug(AKONADISERVER_LOG) << "Database" << m_database.databaseName() << "opened using driver" << m_database.driverName(); } StorageDebugger::instance()->addConnection(reinterpret_cast(this), QThread::currentThread()->objectName()); connect(QThread::currentThread(), &QThread::objectNameChanged, this, [this](const QString &name) { if (!name.isEmpty()) { StorageDebugger::instance()->changeConnection(reinterpret_cast(this), name); } }); DbConfig::configuredDatabase()->initSession(m_database); if (m_keepAliveTimer) { m_keepAliveTimer->start(); } } QSqlDatabase DataStore::database() { if (!m_dbOpened) { open(); } return m_database; } void DataStore::close() { if (m_keepAliveTimer) { m_keepAliveTimer->stop(); } if (!m_dbOpened) { return; } if (inTransaction()) { // By setting m_transactionLevel to '1' here, we skip all nested transactions // and rollback the outermost transaction. m_transactionLevel = 1; rollbackTransaction(); } QueryCache::clear(); m_database.close(); m_database = QSqlDatabase(); m_transactionQueries.clear(); QSqlDatabase::removeDatabase(m_connectionName); StorageDebugger::instance()->removeConnection(reinterpret_cast(this)); m_dbOpened = false; } bool DataStore::init() { Q_ASSERT(QThread::currentThread() == QCoreApplication::instance()->thread()); AkonadiSchema schema; DbInitializer::Ptr initializer = DbInitializer::createInstance(database(), &schema); if (!initializer->run()) { qCCritical(AKONADISERVER_LOG) << initializer->errorMsg(); return false; } s_hasForeignKeyConstraints = initializer->hasForeignKeyConstraints(); if (QFile::exists(QStringLiteral(":dbupdate.xml"))) { DbUpdater updater(database(), QStringLiteral(":dbupdate.xml")); if (!updater.run()) { return false; } } else { qCWarning(AKONADISERVER_LOG) << "Warning: dbupdate.xml not found, skipping updates"; } if (!initializer->updateIndexesAndConstraints()) { qCCritical(AKONADISERVER_LOG) << initializer->errorMsg(); return false; } // enable caching for some tables MimeType::enableCache(true); Flag::enableCache(true); Resource::enableCache(true); Collection::enableCache(true); PartType::enableCache(true); return true; } NotificationCollector *DataStore::notificationCollector() { if (!mNotificationCollector) { mNotificationCollector = std::make_unique(this); } return mNotificationCollector.get(); } DataStore *DataStore::self() { if (!sInstances.hasLocalData()) { sInstances.setLocalData(sFactory->createStore()); } return sInstances.localData(); } bool DataStore::hasDataStore() { return sInstances.hasLocalData(); } /* --- ItemFlags ----------------------------------------------------- */ bool DataStore::setItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col_, bool silent) { QSet removedFlags; QSet addedFlags; QVariantList insIds; QVariantList insFlags; Query::Condition delConds(Query::Or); Collection col = col_; setBoolPtr(flagsChanged, false); for (const PimItem &item : items) { const Flag::List itemFlags = item.flags(); for (const Flag &flag : itemFlags) { if (!flags.contains(flag)) { removedFlags << flag.name().toLatin1(); Query::Condition cond; cond.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::Equals, item.id()); cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), Query::Equals, flag.id()); delConds.addCondition(cond); } } for (const Flag &flag : flags) { if (!itemFlags.contains(flag)) { addedFlags << flag.name().toLatin1(); insIds << item.id(); insFlags << flag.id(); } } if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } } if (!removedFlags.empty()) { QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); qb.addCondition(delConds); if (!qb.exec()) { return false; } } if (!addedFlags.empty()) { QueryBuilder qb2(PimItemFlagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemFlagRelation::leftColumn(), insIds); qb2.setColumnValue(PimItemFlagRelation::rightColumn(), insFlags); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { return false; } } if (!silent && (!addedFlags.isEmpty() || !removedFlags.isEmpty())) { notificationCollector()->itemsFlagsChanged(items, addedFlags, removedFlags, col); } setBoolPtr(flagsChanged, (addedFlags != removedFlags)); return true; } bool DataStore::doAppendItemsFlag(const PimItem::List &items, const Flag &flag, const QSet &existing, const Collection &col_, bool silent) { Collection col = col_; QVariantList flagIds; QVariantList appendIds; PimItem::List appendItems; for (const PimItem &item : items) { if (existing.contains(item.id())) { continue; } flagIds << flag.id(); appendIds << item.id(); appendItems << item; if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } } if (appendItems.isEmpty()) { return true; // all items have the desired flags already } QueryBuilder qb2(PimItemFlagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemFlagRelation::leftColumn(), appendIds); qb2.setColumnValue(PimItemFlagRelation::rightColumn(), flagIds); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query:" << qb2.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Failed to append flag" << flag.name() << "to Items" << appendIds; return false; } if (!silent) { notificationCollector()->itemsFlagsChanged(appendItems, QSet() << flag.name().toLatin1(), QSet(), col); } return true; } bool DataStore::appendItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, bool checkIfExists, const Collection &col, bool silent) { QVariantList itemsIds; itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds.append(item.id()); } setBoolPtr(flagsChanged, false); for (const Flag &flag : flags) { QSet existing; if (checkIfExists) { QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Select); Query::Condition cond; cond.addValueCondition(PimItemFlagRelation::rightColumn(), Query::Equals, flag.id()); cond.addValueCondition(PimItemFlagRelation::leftColumn(), Query::In, itemsIds); qb.addColumn(PimItemFlagRelation::leftColumn()); qb.addCondition(cond); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query:" << qb.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Failed to retrieve existing flags for Items " << itemsIds; return false; } QSqlQuery query = qb.query(); if (query.driver()->hasFeature(QSqlDriver::QuerySize)) { //The query size feature is not supported by the sqllite driver if (query.size() == items.count()) { continue; } setBoolPtr(flagsChanged, true); } while (query.next()) { existing << query.value(0).value(); } if (!query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (existing.size() != items.count()) { setBoolPtr(flagsChanged, true); } } } if (!doAppendItemsFlag(items, flag, existing, col, silent)) { return false; } } return true; } bool DataStore::removeItemsFlags(const PimItem::List &items, const QVector &flags, bool *flagsChanged, const Collection &col_, bool silent) { Collection col = col_; QSet removedFlags; QVariantList itemsIds; QVariantList flagsIds; setBoolPtr(flagsChanged, false); itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds << item.id(); if (col.id() == -1) { col.setId(item.collectionId()); } else if (col.id() != item.collectionId()) { col.setId(-2); } for (int i = 0; i < flags.count(); ++i) { const QByteArray flagName = flags[i].name().toLatin1(); if (!removedFlags.contains(flagName)) { flagsIds << flags[i].id(); removedFlags << flagName; } } } // Delete all given flags from all given items in one go QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); Query::Condition cond(Query::And); cond.addValueCondition(PimItemFlagRelation::rightFullColumnName(), Query::In, flagsIds); cond.addValueCondition(PimItemFlagRelation::leftFullColumnName(), Query::In, itemsIds); qb.addCondition(cond); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove flags" << flags << "from Items" << itemsIds; return false; } if (qb.query().numRowsAffected() != 0) { setBoolPtr(flagsChanged, true); if (!silent) { notificationCollector()->itemsFlagsChanged(items, QSet(), removedFlags, col); } } return true; } /* --- ItemTags ----------------------------------------------------- */ bool DataStore::setItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { QSet removedTags; QSet addedTags; QVariantList insIds; QVariantList insTags; Query::Condition delConds(Query::Or); setBoolPtr(tagsChanged, false); for (const PimItem &item : items) { const Tag::List itemTags = item.tags(); for (const Tag &tag : itemTags) { if (!tags.contains(tag)) { // Remove tags from items that had it set removedTags << tag.id(); Query::Condition cond; cond.addValueCondition(PimItemTagRelation::leftFullColumnName(), Query::Equals, item.id()); cond.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::Equals, tag.id()); delConds.addCondition(cond); } } for (const Tag &tag : tags) { if (!itemTags.contains(tag)) { // Add tags to items that did not have the tag addedTags << tag.id(); insIds << item.id(); insTags << tag.id(); } } } if (!removedTags.empty()) { QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Delete); qb.addCondition(delConds); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << removedTags << "from Items"; return false; } } if (!addedTags.empty()) { QueryBuilder qb2(PimItemTagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemTagRelation::leftColumn(), insIds); qb2.setColumnValue(PimItemTagRelation::rightColumn(), insTags); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to add tags" << addedTags << "to Items"; return false; } } if (!silent && (!addedTags.empty() || !removedTags.empty())) { notificationCollector()->itemsTagsChanged(items, addedTags, removedTags); } setBoolPtr(tagsChanged, (addedTags != removedTags)); return true; } bool DataStore::doAppendItemsTag(const PimItem::List &items, const Tag &tag, const QSet &existing, const Collection &col, bool silent) { QVariantList tagIds; QVariantList appendIds; PimItem::List appendItems; for (const PimItem &item : items) { if (existing.contains(item.id())) { continue; } tagIds << tag.id(); appendIds << item.id(); appendItems << item; } if (appendItems.isEmpty()) { return true; // all items have the desired tags already } QueryBuilder qb2(PimItemTagRelation::tableName(), QueryBuilder::Insert); qb2.setColumnValue(PimItemTagRelation::leftColumn(), appendIds); qb2.setColumnValue(PimItemTagRelation::rightColumn(), tagIds); qb2.setIdentificationColumn(QString()); if (!qb2.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query:" << qb2.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Failed to append tag" << tag << "to Items" << appendItems; return false; } if (!silent) { - notificationCollector()->itemsTagsChanged(appendItems, QSet() << tag.id(), - QSet(), col); + notificationCollector()->itemsTagsChanged(appendItems, {tag.id()}, {}, col); } return true; } bool DataStore::appendItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool checkIfExists, const Collection &col, bool silent) { QVariantList itemsIds; itemsIds.reserve(items.count()); for (const PimItem &item : items) { itemsIds.append(item.id()); } setBoolPtr(tagsChanged, false); for (const Tag &tag : tags) { QSet existing; if (checkIfExists) { QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Select); Query::Condition cond; cond.addValueCondition(PimItemTagRelation::rightColumn(), Query::Equals, tag.id()); cond.addValueCondition(PimItemTagRelation::leftColumn(), Query::In, itemsIds); qb.addColumn(PimItemTagRelation::leftColumn()); qb.addCondition(cond); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query:" << qb.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Failed to retrieve existing tag" << tag << "for Items" << itemsIds; return false; } QSqlQuery query = qb.query(); if (query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (query.size() == items.count()) { continue; } setBoolPtr(tagsChanged, true); } while (query.next()) { existing << query.value(0).value(); } if (!query.driver()->hasFeature(QSqlDriver::QuerySize)) { if (existing.size() != items.count()) { setBoolPtr(tagsChanged, true); } } } if (!doAppendItemsTag(items, tag, existing, col, silent)) { return false; } } return true; } bool DataStore::removeItemsTags(const PimItem::List &items, const Tag::List &tags, bool *tagsChanged, bool silent) { QSet removedTags; QVariantList itemsIds; QVariantList tagsIds; setBoolPtr(tagsChanged, false); itemsIds.reserve(items.count()); Q_FOREACH (const PimItem &item, items) { itemsIds << item.id(); for (int i = 0; i < tags.count(); ++i) { const qint64 tagId = tags[i].id(); if (!removedTags.contains(tagId)) { tagsIds << tagId; removedTags << tagId; } } } // Delete all given tags from all given items in one go QueryBuilder qb(PimItemTagRelation::tableName(), QueryBuilder::Delete); Query::Condition cond(Query::And); cond.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::In, tagsIds); cond.addValueCondition(PimItemTagRelation::leftFullColumnName(), Query::In, itemsIds); qb.addCondition(cond); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << tagsIds << "from Items" << itemsIds; return false; } if (qb.query().numRowsAffected() != 0) { setBoolPtr(tagsChanged, true); if (!silent) { notificationCollector()->itemsTagsChanged(items, QSet(), removedTags); } } return true; } bool DataStore::removeTags(const Tag::List &tags, bool silent) { // Currently the "silent" argument is only for API symmetry Q_UNUSED(silent); QVariantList removedTagsIds; QSet removedTags; removedTagsIds.reserve(tags.count()); removedTags.reserve(tags.count()); for (const Tag &tag : tags) { removedTagsIds << tag.id(); removedTags << tag.id(); } // Get all PIM items that we will untag SelectQueryBuilder itemsQuery; itemsQuery.addJoin(QueryBuilder::LeftJoin, PimItemTagRelation::tableName(), PimItemTagRelation::leftFullColumnName(), PimItem::idFullColumnName()); itemsQuery.addValueCondition(PimItemTagRelation::rightFullColumnName(), Query::In, removedTagsIds); if (!itemsQuery.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query: " << itemsQuery.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Removing tags failed: failed to query Items for given tags" << removedTagsIds; return false; } const PimItem::List items = itemsQuery.result(); if (!items.isEmpty()) { notificationCollector()->itemsTagsChanged(items, QSet(), removedTags); } Q_FOREACH (const Tag &tag, tags) { // Emit special tagRemoved notification for each resource that owns the tag QueryBuilder qb(TagRemoteIdResourceRelation::tableName(), QueryBuilder::Select); qb.addColumn(TagRemoteIdResourceRelation::remoteIdFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), TagRemoteIdResourceRelation::resourceIdFullColumnName(), Resource::idFullColumnName()); qb.addColumn(Resource::nameFullColumnName()); qb.addValueCondition(TagRemoteIdResourceRelation::tagIdFullColumnName(), Query::Equals, tag.id()); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query: " << qb.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Removing tags failed: failed to retrieve RIDs for tag" << tag.id(); return false; } // Emit specialized notifications for each resource QSqlQuery query = qb.query(); while (query.next()) { const QString rid = query.value(0).toString(); const QByteArray resource = query.value(1).toByteArray(); notificationCollector()->tagRemoved(tag, resource, rid); } // And one for clients - without RID notificationCollector()->tagRemoved(tag, QByteArray(), QString()); } // Just remove the tags, table constraints will take care of the rest QueryBuilder qb(Tag::tableName(), QueryBuilder::Delete); qb.addValueCondition(Tag::idColumn(), Query::In, removedTagsIds); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Failed to execute query: " << itemsQuery.query().lastError(); + qCWarning(AKONADISERVER_LOG) << "Failed to remove tags" << removedTagsIds; return false; } return true; } /* --- ItemParts ----------------------------------------------------- */ bool DataStore::removeItemParts(const PimItem &item, const QSet &parts) { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName()); qb.addValueCondition(Part::pimItemIdFullColumnName(), Query::Equals, item.id()); qb.addCondition(PartTypeHelper::conditionFromFqNames(parts)); - qb.exec(); + if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Removing item parts failed: failed to query parts" << parts << "from Item " << item.id(); + return false; + } + const Part::List existingParts = qb.result(); for (Part part : qAsConst(existingParts)) { //krazy:exclude=foreach if (!PartHelper::remove(&part)) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove part" << part.id() << "(" << part.partType().ns() + << ":" << part.partType().name() << ") from Item" << item.id(); return false; } } notificationCollector()->itemChanged(item, parts); return true; } bool DataStore::invalidateItemCache(const PimItem &item) { // find all payload item parts SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName()); qb.addValueCondition(Part::pimItemIdFullColumnName(), Query::Equals, item.id()); qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant()); qb.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QLatin1String("PLD")); qb.addValueCondition(PimItem::dirtyFullColumnName(), Query::Equals, false); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to invalidate cache for Item" << item.id(); return false; } const Part::List parts = qb.result(); // clear data field for (Part part : parts) { if (!PartHelper::truncate(part)) { + qCWarning(AKONADISERVER_LOG) << "Failed to truncate payload part" << part.id() << "(" << part.partType().ns() + << ":" << part.partType().name() << ") of Item" << item.id(); return false; } } return true; } /* --- Collection ------------------------------------------------------ */ bool DataStore::appendCollection(Collection &collection, const QStringList &mimeTypes, const QMap &attributes) { // no need to check for already existing collection with the same name, // a unique index on parent + name prevents that in the database if (!collection.insert()) { + qCWarning(AKONADISERVER_LOG) << "Failed to append Collection" << collection.name() << "in resource" + << collection.resource().name(); return false; } if (!appendMimeTypeForCollection(collection.id(), mimeTypes)) { + qCWarning(AKONADISERVER_LOG) << "Failed to append mimetypes" << mimeTypes << "to new collection" << collection.name() + << "(ID" << collection.id() << ") in resource" << collection.resource().name(); return false; } for (auto it = attributes.cbegin(), end = attributes.cend(); it != end; ++it) { if (!addCollectionAttribute(collection, it.key(), it.value(), true)) { + qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << it.key() << "to new collection" << collection.name() + << "(ID" << collection.id() << ") in resource" << collection.resource().name(); return false; } } notificationCollector()->collectionAdded(collection); return true; } bool DataStore::cleanupCollection(Collection &collection) { if (!s_hasForeignKeyConstraints) { return cleanupCollection_slow(collection); } // db will do most of the work for us, we just deal with notifications and external payload parts here Q_ASSERT(s_hasForeignKeyConstraints); // collect item deletion notifications const PimItem::List items = collection.items(); const QByteArray resource = collection.resource().name().toLatin1(); // generate the notification before actually removing the data // TODO: we should try to get rid of this, requires client side changes to resources and Monitor though notificationCollector()->itemsRemoved(items, collection, resource); // remove all external payload parts QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName()); qb.addValueCondition(Collection::idFullColumnName(), Query::Equals, collection.id()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External); qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant()); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to cleanup collection" << collection.name() << "(ID" << collection.id() << "):" + << "Failed to query existing payload parts"; return false; } try { while (qb.query().next()) { ExternalPartStorage::self()->removePartFile( ExternalPartStorage::resolveAbsolutePath(qb.query().value(0).toByteArray())); } } catch (const PartHelperException &e) { - qCDebug(AKONADISERVER_LOG) << e.what(); + qb.query().finish(); + qCWarning(AKONADISERVER_LOG) << "PartHelperException while cleaning up collection" << collection.name() + << "(ID" << collection.id() << "):" << e.what(); return false; } // delete the collection itself, referential actions will do the rest notificationCollector()->collectionRemoved(collection); return collection.remove(); } bool DataStore::cleanupCollection_slow(Collection &collection) { Q_ASSERT(!s_hasForeignKeyConstraints); // delete the content const PimItem::List items = collection.items(); const QByteArray resource = collection.resource().name().toLatin1(); notificationCollector()->itemsRemoved(items, collection, resource); for (const PimItem &item : items) { if (!item.clearFlags()) { // TODO: move out of loop and use only a single query + qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" + << "failed: error clearing items flags"; return false; } if (!PartHelper::remove(Part::pimItemIdColumn(), item.id())) { // TODO: reduce to single query + qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" + << "failed: error clearing item payload parts"; + return false; } - if (!PimItem::remove(PimItem::idColumn(), item.id())) { // TODO: move into single query + if (!PimItem::remove(PimItem::idColumn(), item.id())) { // TODO: move into single querya + qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" + << "failed: error clearing items"; return false; } if (!Entity::clearRelation(item.id(), Entity::Right)) { // TODO: move into single query + qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() <<")" + << "failed: error clearing linked items"; return false; } } // delete collection mimetypes collection.clearMimeTypes(); Collection::clearPimItems(collection.id()); // delete attributes Q_FOREACH (CollectionAttribute attr, collection.attributes()) { //krazy:exclude=foreach if (!attr.remove()) { + qCWarning(AKONADISERVER_LOG) << "Slow cleanup of collection" << collection.name() << "(ID" << collection.id() << ")" + << "failed: error clearing attribute" << attr.type(); return false; } } // delete the collection itself notificationCollector()->collectionRemoved(collection); return collection.remove(); } static bool recursiveSetResourceId(const Collection &collection, qint64 resourceId) { Transaction transaction(DataStore::self(), QStringLiteral("RECURSIVE SET RESOURCEID")); QueryBuilder qb(Collection::tableName(), QueryBuilder::Update); qb.addValueCondition(Collection::parentIdColumn(), Query::Equals, collection.id()); qb.setColumnValue(Collection::resourceIdColumn(), resourceId); qb.setColumnValue(Collection::remoteIdColumn(), QVariant()); qb.setColumnValue(Collection::remoteRevisionColumn(), QVariant()); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to set resource ID" << resourceId << "to collection" + << collection.name() << "(ID" << collection.id() << ")"; return false; } // this is a cross-resource move, so also reset any resource-specific data (RID, RREV, etc) // as well as mark the items dirty to prevent cache purging before they have been written back qb = QueryBuilder(PimItem::tableName(), QueryBuilder::Update); qb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, collection.id()); qb.setColumnValue(PimItem::remoteIdColumn(), QVariant()); qb.setColumnValue(PimItem::remoteRevisionColumn(), QVariant()); const QDateTime now = QDateTime::currentDateTimeUtc(); qb.setColumnValue(PimItem::datetimeColumn(), now); qb.setColumnValue(PimItem::atimeColumn(), now); qb.setColumnValue(PimItem::dirtyColumn(), true); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed reset RID/RREV for PimItems in Collection" << collection.name() + << "(ID" << collection.id() << ")"; return false; } transaction.commit(); Q_FOREACH (const Collection &col, collection.children()) { if (!recursiveSetResourceId(col, resourceId)) { return false; } } return true; } bool DataStore::moveCollection(Collection &collection, const Collection &newParent) { if (collection.parentId() == newParent.id()) { return true; } - if (!m_dbOpened || !newParent.isValid()) { + if (!m_dbOpened) { + return false; + } + + if (!newParent.isValid()) { + qCWarning(AKONADISERVER_LOG) << "Failed to move collection" << collection.name() << "(ID" + << collection.id() << "): invalid destination"; return false; } const QByteArray oldResource = collection.resource().name().toLatin1(); int resourceId = collection.resourceId(); const Collection source = collection.parent(); if (newParent.id() > 0) { // not root resourceId = newParent.resourceId(); } if (!CollectionQueryHelper::canBeMovedTo(collection, newParent)) { return false; } collection.setParentId(newParent.id()); if (collection.resourceId() != resourceId) { collection.setResourceId(resourceId); collection.setRemoteId(QString()); collection.setRemoteRevision(QString()); if (!recursiveSetResourceId(collection, resourceId)) { return false; } } if (!collection.update()) { + qCWarning(AKONADISERVER_LOG) << "Failed to move Collection" << collection.name() << "(ID" << collection.id() << ")" + << "into Collection" << collection.name() << "(ID" << collection.id() << ")"; return false; } notificationCollector()->collectionMoved(collection, source, oldResource, newParent.resource().name().toLatin1()); return true; } bool DataStore::appendMimeTypeForCollection(qint64 collectionId, const QStringList &mimeTypes) { if (mimeTypes.isEmpty()) { return true; } for (const QString &mimeType : mimeTypes) { const auto &mt = MimeType::retrieveByNameOrCreate(mimeType); if (!mt.isValid()) { return false; } if (!Collection::addMimeType(collectionId, mt.id())) { + qCWarning(AKONADISERVER_LOG) << "Failed to append mimetype" << mt.name() << "to Collection" << collectionId; return false; } } return true; } void DataStore::activeCachePolicy(Collection &col) { if (!col.cachePolicyInherit()) { return; } Collection parent = col; while (parent.parentId() != 0) { parent = parent.parent(); if (!parent.cachePolicyInherit()) { col.setCachePolicyCheckInterval(parent.cachePolicyCheckInterval()); col.setCachePolicyCacheTimeout(parent.cachePolicyCacheTimeout()); col.setCachePolicySyncOnDemand(parent.cachePolicySyncOnDemand()); col.setCachePolicyLocalParts(parent.cachePolicyLocalParts()); return; } } // ### system default col.setCachePolicyCheckInterval(-1); col.setCachePolicyCacheTimeout(-1); col.setCachePolicySyncOnDemand(false); col.setCachePolicyLocalParts(QStringLiteral("ALL")); } QVector DataStore::virtualCollections(const PimItem &item) { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), Collection::idFullColumnName(), CollectionPimItemRelation::leftFullColumnName()); qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::Equals, item.id()); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during selection of records from table CollectionPimItemRelation" - << qb.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Failed to query virtual collections which PimItem" << item.id() << "belongs into"; return QVector(); } return qb.result(); } QMap > DataStore::virtualCollections(const PimItem::List &items) { QueryBuilder qb(CollectionPimItemRelation::tableName(), QueryBuilder::Select); qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), Collection::idFullColumnName(), CollectionPimItemRelation::leftFullColumnName()); qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), PimItem::idFullColumnName(), CollectionPimItemRelation::rightFullColumnName()); qb.addColumn(Collection::idFullColumnName()); qb.addColumns(QStringList() << PimItem::idFullColumnName() << PimItem::remoteIdFullColumnName() << PimItem::remoteRevisionFullColumnName() << PimItem::mimeTypeIdFullColumnName()); qb.addSortColumn(Collection::idFullColumnName(), Query::Ascending); if (items.count() == 1) { qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::Equals, items.first().id()); } else { QVariantList ids; ids.reserve(items.count()); for (const PimItem &item : items) { ids << item.id(); } qb.addValueCondition(CollectionPimItemRelation::rightFullColumnName(), Query::In, ids); } if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during selection of records from table CollectionPimItemRelation" - << qb.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Failed to query virtual Collections which PimItems" << items << "belong into"; return QMap >(); } QSqlQuery query = qb.query(); QMap > map; query.next(); while (query.isValid()) { const qlonglong collectionId = query.value(0).toLongLong(); QList &pimItems = map[collectionId]; do { PimItem item; item.setId(query.value(1).toLongLong()); item.setRemoteId(query.value(2).toString()); item.setRemoteRevision(query.value(3).toString()); item.setMimeTypeId(query.value(4).toLongLong()); pimItems << item; } while (query.next() && query.value(0).toLongLong() == collectionId); } return map; } /* --- PimItem ------------------------------------------------------- */ bool DataStore::appendPimItem(QVector &parts, const QVector &flags, const MimeType &mimetype, const Collection &collection, const QDateTime &dateTime, const QString &remote_id, const QString &remoteRevision, const QString &gid, PimItem &pimItem) { pimItem.setMimeTypeId(mimetype.id()); pimItem.setCollectionId(collection.id()); if (dateTime.isValid()) { pimItem.setDatetime(dateTime); } if (remote_id.isEmpty()) { // from application pimItem.setDirty(true); } else { // from resource pimItem.setRemoteId(remote_id); pimItem.setDirty(false); } pimItem.setRemoteRevision(remoteRevision); pimItem.setGid(gid); pimItem.setAtime(QDateTime::currentDateTimeUtc()); if (!pimItem.insert()) { + qCWarning(AKONADISERVER_LOG) << "Failed to append new PimItem into Collection" << collection.name() + << "(ID" << collection.id() << ")"; return false; } // insert every part if (!parts.isEmpty()) { //don't use foreach, the caller depends on knowing the part has changed, see the Append handler for (QVector::iterator it = parts.begin(); it != parts.end(); ++it) { (*it).setPimItemId(pimItem.id()); if ((*it).datasize() < (*it).data().size()) { (*it).setDatasize((*it).data().size()); } // qCDebug(AKONADISERVER_LOG) << "Insert from DataStore::appendPimItem"; if (!PartHelper::insert(&(*it))) { + qCWarning(AKONADISERVER_LOG) << "Failed to add part" << it->partType().name() << "to new PimItem" << pimItem.id(); return false; } } } bool seen = false; Q_FOREACH (const Flag &flag, flags) { seen |= (flag.name() == QLatin1String(AKONADI_FLAG_SEEN) || flag.name() == QLatin1String(AKONADI_FLAG_IGNORED)); if (!pimItem.addFlag(flag)) { + qCWarning(AKONADISERVER_LOG) << "Failed to add flag" << flag.name() << "to new PimItem" << pimItem.id(); return false; } } // qCDebug(AKONADISERVER_LOG) << "appendPimItem: " << pimItem; notificationCollector()->itemAdded(pimItem, seen, collection); return true; } bool DataStore::unhidePimItem(PimItem &pimItem) { if (!m_dbOpened) { return false; } qCDebug(AKONADISERVER_LOG) << "DataStore::unhidePimItem(" << pimItem << ")"; // FIXME: This is inefficient. Using a bit on the PimItemTable record would probably be some orders of magnitude faster... return removeItemParts(pimItem, { AKONADI_ATTRIBUTE_HIDDEN }); } bool DataStore::unhideAllPimItems() { if (!m_dbOpened) { return false; } qCDebug(AKONADISERVER_LOG) << "DataStore::unhideAllPimItems()"; try { return PartHelper::remove(Part::partTypeIdFullColumnName(), PartTypeHelper::fromFqName(QStringLiteral("ATR"), QStringLiteral("HIDDEN")).id()); } catch (...) { } // we can live with this failing return false; } bool DataStore::cleanupPimItems(const PimItem::List &items) { // generate relation removed notifications for (const PimItem &item : items) { SelectQueryBuilder relationQuery; relationQuery.addValueCondition(Relation::leftIdFullColumnName(), Query::Equals, item.id()); relationQuery.addValueCondition(Relation::rightIdFullColumnName(), Query::Equals, item.id()); relationQuery.setSubQueryMode(Query::Or); if (!relationQuery.exec()) { throw HandlerException("Failed to obtain relations"); } const Relation::List relations = relationQuery.result(); for (const Relation &relation : relations) { notificationCollector()->relationRemoved(relation); } } // generate the notification before actually removing the data notificationCollector()->itemsRemoved(items); // FIXME: Create a single query to do this Q_FOREACH (const PimItem &item, items) { if (!item.clearFlags()) { + qCWarning(AKONADISERVER_LOG) << "Failed to clean up flags from PimItem" << item.id(); return false; } if (!PartHelper::remove(Part::pimItemIdColumn(), item.id())) { + qCWarning(AKONADISERVER_LOG) << "Failed to clean up parts from PimItem" << item.id(); return false; } if (!PimItem::remove(PimItem::idColumn(), item.id())) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove PimItem" << item.id(); return false; } if (!Entity::clearRelation(item.id(), Entity::Right)) { + qCWarning(AKONADISERVER_LOG) << "Failed to remove PimItem" << item.id() << "from linked collections"; return false; } } return true; } bool DataStore::addCollectionAttribute(const Collection &col, const QByteArray &key, const QByteArray &value, bool silent) { SelectQueryBuilder qb; qb.addValueCondition(CollectionAttribute::collectionIdColumn(), Query::Equals, col.id()); qb.addValueCondition(CollectionAttribute::typeColumn(), Query::Equals, key); if (!qb.exec()) { + qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() + << "(ID" << col.id() << "): Failed to query existing attribute"; return false; } if (!qb.result().isEmpty()) { - qCDebug(AKONADISERVER_LOG) << "Attribute" << key << "already exists for collection" << col.id(); + qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() + << "(ID" << col.id() << "): Attribute already exists"; return false; } CollectionAttribute attr; attr.setCollectionId(col.id()); attr.setType(key); attr.setValue(value); if (!attr.insert()) { + qCWarning(AKONADISERVER_LOG) << "Failed to append attribute" << key << "to Collection" << col.name() + << "(ID" << col.id() << ")"; return false; } if (!silent) { notificationCollector()->collectionChanged(col, QList() << key); } return true; } bool DataStore::removeCollectionAttribute(const Collection &col, const QByteArray &key) { SelectQueryBuilder qb; qb.addValueCondition(CollectionAttribute::collectionIdColumn(), Query::Equals, col.id()); qb.addValueCondition(CollectionAttribute::typeColumn(), Query::Equals, key); if (!qb.exec()) { throw HandlerException("Unable to query for collection attribute"); } const QVector result = qb.result(); for (CollectionAttribute attr : result) { if (!attr.remove()) { throw HandlerException("Unable to remove collection attribute"); } } if (!result.isEmpty()) { notificationCollector()->collectionChanged(col, QList() << key); return true; } return false; } void DataStore::debugLastDbError(const char *actionDescription) const { qCCritical(AKONADISERVER_LOG) << "Database error:" << actionDescription; qCCritical(AKONADISERVER_LOG) << " Last driver error:" << m_database.lastError().driverText(); qCCritical(AKONADISERVER_LOG) << " Last database error:" << m_database.lastError().databaseText(); Tracer::self()->error("DataStore (Database Error)", QStringLiteral("%1\nDriver said: %2\nDatabase said:%3") .arg(QString::fromLatin1(actionDescription), m_database.lastError().driverText(), m_database.lastError().databaseText())); } void DataStore::debugLastQueryError(const QSqlQuery &query, const char *actionDescription) const { qCCritical(AKONADISERVER_LOG) << "Query error:" << actionDescription; qCCritical(AKONADISERVER_LOG) << " Last error message:" << query.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Last driver error:" << m_database.lastError().driverText(); qCCritical(AKONADISERVER_LOG) << " Last database error:" << m_database.lastError().databaseText(); Tracer::self()->error("DataStore (Database Query Error)", QStringLiteral("%1: %2") .arg(QString::fromLatin1(actionDescription), query.lastError().text())); } // static QString DataStore::dateTimeFromQDateTime(const QDateTime &dateTime) { QDateTime utcDateTime = dateTime; if (utcDateTime.timeSpec() != Qt::UTC) { utcDateTime.toUTC(); } return utcDateTime.toString(QStringLiteral("yyyy-MM-dd hh:mm:ss")); } // static QDateTime DataStore::dateTimeToQDateTime(const QByteArray &dateTime) { return QDateTime::fromString(QString::fromLatin1(dateTime), QStringLiteral("yyyy-MM-dd hh:mm:ss")); } void DataStore::addQueryToTransaction(const QString &statement, const QVector &bindValues, bool isBatch) { // This is used for replaying deadlocked transactions, so only record queries // for backends that support concurrent transactions. if (!inTransaction() || DbType::isSystemSQLite(m_database)) { return; } m_transactionQueries.append({ statement, bindValues, isBatch }); } QSqlQuery DataStore::retryLastTransaction(bool rollbackFirst) { if (!inTransaction() || DbType::isSystemSQLite(m_database)) { return QSqlQuery(); } if (rollbackFirst) { // In some cases the SQL database won't rollback the failed transaction, so // we need to do it manually QElapsedTimer timer; timer.start(); m_database.driver()->rollbackTransaction(); StorageDebugger::instance()->removeTransaction(reinterpret_cast(this), false, timer.elapsed(), m_database.lastError().text()); } // The database has rolled back the actual transaction, so reset the counter // to 0 and start a new one in beginTransaction(). Then restore the level // because this has to be completely transparent to the original caller const int oldTransactionLevel = m_transactionLevel; m_transactionLevel = 0; if (!beginTransaction(QStringLiteral("RETRY LAST TRX"))) { m_transactionLevel = oldTransactionLevel; return QSqlQuery(); } m_transactionLevel = oldTransactionLevel; QSqlQuery lastQuery; for (auto q = m_transactionQueries.begin(), qEnd = m_transactionQueries.end(); q != qEnd; ++q) { QSqlQuery query(database()); query.prepare(q->query); for (int i = 0, total = q->boundValues.count(); i < total; ++i) { query.bindValue(QLatin1Char(':') + QString::number(i), q->boundValues.at(i)); } bool res = false; QElapsedTimer t; t.start(); if (q->isBatch) { res = query.execBatch(); } else { res = query.exec(); } if (StorageDebugger::instance()->isSQLDebuggingEnabled()) { StorageDebugger::instance()->queryExecuted(reinterpret_cast(this), query, t.elapsed()); } else { StorageDebugger::instance()->incSequence(); } if (!res) { // Don't do another deadlock detection here, just give up. - qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR:"; + qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR when retrying transaction"; qCCritical(AKONADISERVER_LOG) << " Error code:" << query.lastError().nativeErrorCode(); qCCritical(AKONADISERVER_LOG) << " DB error: " << query.lastError().databaseText(); qCCritical(AKONADISERVER_LOG) << " Error text:" << query.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Query:" << query.executedQuery(); // Return the last query, because that's what caller expects to retrieve // from QueryBuilder. It is in error state anyway. return query; } lastQuery = query; } return lastQuery; } bool DataStore::beginTransaction(const QString &name) { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0) { QElapsedTimer timer; timer.start(); TRANSACTION_MUTEX_LOCK; if (DbType::type(m_database) == DbType::Sqlite) { m_database.exec(QStringLiteral("BEGIN IMMEDIATE TRANSACTION")); StorageDebugger::instance()->addTransaction(reinterpret_cast(this), name, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::beginTransaction (SQLITE)"); TRANSACTION_MUTEX_UNLOCK; return false; } } else { m_database.driver()->beginTransaction(); StorageDebugger::instance()->addTransaction(reinterpret_cast(this), name, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::beginTransaction"); TRANSACTION_MUTEX_UNLOCK; return false; } } if (DbType::type(m_database) == DbType::PostgreSQL) { // Make constraints check deferred in PostgreSQL. Allows for // INSERT INTO mimetypetable (name) VALUES ('foo') RETURNING id; // INSERT INTO collectionmimetyperelation (collection_id, mimetype_id) VALUES (x, y) // where "y" refers to the newly inserted mimetype m_database.exec(QStringLiteral("SET CONSTRAINTS ALL DEFERRED")); } } ++m_transactionLevel; return true; } bool DataStore::rollbackTransaction() { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0) { qCWarning(AKONADISERVER_LOG) << "DataStore::rollbackTransaction(): No transaction in progress!"; return false; } --m_transactionLevel; if (m_transactionLevel == 0) { QSqlDriver *driver = m_database.driver(); Q_EMIT transactionRolledBack(); QElapsedTimer timer; timer.start(); driver->rollbackTransaction(); StorageDebugger::instance()->removeTransaction(reinterpret_cast(this), false, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { TRANSACTION_MUTEX_UNLOCK; debugLastDbError("DataStore::rollbackTransaction"); return false; } TRANSACTION_MUTEX_UNLOCK; m_transactionQueries.clear(); } return true; } bool DataStore::commitTransaction() { if (!m_dbOpened) { return false; } if (m_transactionLevel == 0) { qCWarning(AKONADISERVER_LOG) << "DataStore::commitTransaction(): No transaction in progress!"; return false; } if (m_transactionLevel == 1) { QSqlDriver *driver = m_database.driver(); QElapsedTimer timer; timer.start(); driver->commitTransaction(); StorageDebugger::instance()->removeTransaction(reinterpret_cast(this), true, timer.elapsed(), m_database.lastError().text()); if (m_database.lastError().isValid()) { debugLastDbError("DataStore::commitTransaction"); rollbackTransaction(); return false; } else { TRANSACTION_MUTEX_UNLOCK; m_transactionLevel--; Q_EMIT transactionCommitted(); } m_transactionQueries.clear(); } else { m_transactionLevel--; } return true; } bool DataStore::inTransaction() const { return m_transactionLevel > 0; } void DataStore::sendKeepAliveQuery() { if (m_database.isOpen()) { QSqlQuery query(m_database); query.exec(QStringLiteral("SELECT 1")); } } diff --git a/src/server/storage/dbconfigsqlite.cpp b/src/server/storage/dbconfigsqlite.cpp index 72e02206e..c1e0d00aa 100644 --- a/src/server/storage/dbconfigsqlite.cpp +++ b/src/server/storage/dbconfigsqlite.cpp @@ -1,287 +1,281 @@ /* Copyright (c) 2010 Tobias Koenig This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dbconfigsqlite.h" #include "utils.h" #include "akonadiserver_debug.h" #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; static QString dataDir() { QString akonadiHomeDir = StandardDirs::saveDir("data"); if (!QDir(akonadiHomeDir).exists()) { if (!QDir().mkpath(akonadiHomeDir)) { qCCritical(AKONADISERVER_LOG) << "Unable to create" << akonadiHomeDir << "during database initialization"; return QString(); } } akonadiHomeDir += QDir::separator(); return akonadiHomeDir; } static QString sqliteDataFile() { const QString dir = dataDir(); if (dir.isEmpty()) { return QString(); } const QString akonadiPath = dir + QLatin1String("akonadi.db"); if (!QFile::exists(akonadiPath)) { QFile file(akonadiPath); if (!file.open(QIODevice::WriteOnly)) { qCCritical(AKONADISERVER_LOG) << "Unable to create file" << akonadiPath << "during database initialization."; return QString(); } file.close(); } return akonadiPath; } DbConfigSqlite::DbConfigSqlite(Version driverVersion) : mDriverVersion(driverVersion) { } QString DbConfigSqlite::driverName() const { if (mDriverVersion == Default) { return QStringLiteral("QSQLITE"); } else { return QStringLiteral("QSQLITE3"); } } QString DbConfigSqlite::databaseName() const { return mDatabaseName; } bool DbConfigSqlite::init(QSettings &settings) { // determine default settings depending on the driver const QString defaultDbName = sqliteDataFile(); if (defaultDbName.isEmpty()) { return false; } // read settings for current driver settings.beginGroup(driverName()); mDatabaseName = settings.value(QStringLiteral("Name"), defaultDbName).toString(); mHostName = settings.value(QStringLiteral("Host")).toString(); mUserName = settings.value(QStringLiteral("User")).toString(); mPassword = settings.value(QStringLiteral("Password")).toString(); mConnectionOptions = settings.value(QStringLiteral("Options")).toString(); settings.endGroup(); // store back the default values settings.beginGroup(driverName()); settings.setValue(QStringLiteral("Name"), mDatabaseName); settings.endGroup(); settings.sync(); return true; } void DbConfigSqlite::apply(QSqlDatabase &database) { if (!mDatabaseName.isEmpty()) { database.setDatabaseName(mDatabaseName); } if (!mHostName.isEmpty()) { database.setHostName(mHostName); } if (!mUserName.isEmpty()) { database.setUserName(mUserName); } if (!mPassword.isEmpty()) { database.setPassword(mPassword); } if (driverName() == QLatin1String("QSQLITE3") && !mConnectionOptions.contains(QLatin1String("SQLITE_ENABLE_SHARED_CACHE"))) { mConnectionOptions += QLatin1String(";QSQLITE_ENABLE_SHARED_CACHE"); } database.setConnectOptions(mConnectionOptions); // can we check that during init() already? Q_ASSERT(database.driver()->hasFeature(QSqlDriver::LastInsertId)); } bool DbConfigSqlite::useInternalServer() const { return false; } bool DbConfigSqlite::setPragma(QSqlDatabase &db, QSqlQuery &query, const QString &pragma) { if (!query.exec(QStringLiteral("PRAGMA %1").arg(pragma))) { - qCDebug(AKONADISERVER_LOG) << "Could not set sqlite PRAGMA " << pragma; - qCDebug(AKONADISERVER_LOG) << "Database: " << mDatabaseName; - qCDebug(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); - qCDebug(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Could not set sqlite PRAGMA " << pragma; + qCCritical(AKONADISERVER_LOG) << "Database: " << mDatabaseName; + qCCritical(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); return false; } return true; } void DbConfigSqlite::setup() { const QLatin1String connectionName("initConnection"); { QSqlDatabase db = QSqlDatabase::addDatabase(driverName(), connectionName); if (!db.isValid()) { - qCDebug(AKONADISERVER_LOG) << "Invalid database for " - << mDatabaseName - << " with driver " - << driverName(); + qCCritical(AKONADISERVER_LOG) << "Invalid database for" << mDatabaseName << "with driver" << driverName(); return; } QFileInfo finfo(mDatabaseName); if (!finfo.dir().exists()) { QDir dir; dir.mkpath(finfo.path()); } #ifdef Q_OS_LINUX QFile dbFile(mDatabaseName); // It is recommended to disable CoW feature when running on Btrfs to improve // database performance. It does not have any effect on non-empty files, so // we check, whether the database has not yet been initialized. if (dbFile.size() == 0) { if (Utils::getDirectoryFileSystem(mDatabaseName) == QLatin1String("btrfs")) { Utils::disableCoW(mDatabaseName); } } #endif db.setDatabaseName(mDatabaseName); if (!db.open()) { - qCDebug(AKONADISERVER_LOG) << "Could not open sqlite database " - << mDatabaseName - << " with driver " - << driverName() - << " for initialization"; + qCCritical(AKONADISERVER_LOG) << "Could not open sqlite database" << mDatabaseName << "with driver" + << driverName() << "for initialization"; db.close(); return; } apply(db); QSqlQuery query(db); if (!query.exec(QStringLiteral("SELECT sqlite_version()"))) { - qCDebug(AKONADISERVER_LOG) << "Could not query sqlite version"; - qCDebug(AKONADISERVER_LOG) << "Database: " << mDatabaseName; - qCDebug(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); - qCDebug(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Could not query sqlite version"; + qCCritical(AKONADISERVER_LOG) << "Database: " << mDatabaseName; + qCCritical(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); db.close(); return; } if (!query.next()) { // should never occur - qCDebug(AKONADISERVER_LOG) << "Could not query sqlite version"; - qCDebug(AKONADISERVER_LOG) << "Database: " << mDatabaseName; - qCDebug(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); - qCDebug(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Could not query sqlite version"; + qCCritical(AKONADISERVER_LOG) << "Database: " << mDatabaseName; + qCCritical(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); db.close(); return; } const QString sqliteVersion = query.value(0).toString(); qCDebug(AKONADISERVER_LOG) << "sqlite version is " << sqliteVersion; const QStringList list = sqliteVersion.split(QLatin1Char('.')); const int sqliteVersionMajor = list[0].toInt(); const int sqliteVersionMinor = list[1].toInt(); // set synchronous mode to NORMAL; see http://www.sqlite.org/pragma.html#pragma_synchronous if (!setPragma(db, query, QStringLiteral("synchronous=1"))) { db.close(); return; } if (sqliteVersionMajor < 3 && sqliteVersionMinor < 7) { // wal mode is only supported with >= sqlite 3.7.0 db.close(); return; } // set write-ahead-log mode; see http://www.sqlite.org/wal.html if (!setPragma(db, query, QStringLiteral("journal_mode=wal"))) { db.close(); return; } if (!query.next()) { // should never occur - qCDebug(AKONADISERVER_LOG) << "Could not query sqlite journal mode"; - qCDebug(AKONADISERVER_LOG) << "Database: " << mDatabaseName; - qCDebug(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); - qCDebug(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Could not query sqlite journal mode"; + qCCritical(AKONADISERVER_LOG) << "Database: " << mDatabaseName; + qCCritical(AKONADISERVER_LOG) << "Query error: " << query.lastError().text(); + qCCritical(AKONADISERVER_LOG) << "Database error: " << db.lastError().text(); db.close(); return; } const QString journalMode = query.value(0).toString(); qCDebug(AKONADISERVER_LOG) << "sqlite journal mode is " << journalMode; // as of sqlite 3.12 this is default, previously was 1024. if (!setPragma(db, query, QStringLiteral("page_size=4096"))) { db.close(); return; } // set cache_size to 100000 pages; see https://www.sqlite.org/pragma.html#pragma_cache_size if (!setPragma(db, query, QStringLiteral("cache_size=100000"))) { db.close(); return; } // construct temporary tables in memory; see https://www.sqlite.org/pragma.html#pragma_temp_store if (!setPragma(db, query, QStringLiteral("temp_store=MEMORY"))) { db.close(); return; } // enable foreign key support; see https://www.sqlite.org/pragma.html#pragma_foreign_keys if (!setPragma(db, query, QStringLiteral("foreign_keys=ON"))) { db.close(); return; } db.close(); } QSqlDatabase::removeDatabase(connectionName); } diff --git a/src/server/storage/dbinitializer.cpp b/src/server/storage/dbinitializer.cpp index 2667793cb..9789f6840 100644 --- a/src/server/storage/dbinitializer.cpp +++ b/src/server/storage/dbinitializer.cpp @@ -1,408 +1,408 @@ /*************************************************************************** * Copyright (C) 2006 by Tobias Koenig * * Copyright (C) 2012 by Volker Krause * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "dbinitializer.h" #include "dbinitializer_p.h" #include "querybuilder.h" #include "dbexception.h" #include "schema.h" #include "entities.h" #include "akonadiserver_debug.h" #include #include #include #include #include using namespace Akonadi::Server; DbInitializer::Ptr DbInitializer::createInstance(const QSqlDatabase &database, Schema *schema) { DbInitializer::Ptr i; switch (DbType::type(database)) { case DbType::MySQL: i.reset(new DbInitializerMySql(database)); break; case DbType::Sqlite: i.reset(new DbInitializerSqlite(database)); break; case DbType::PostgreSQL: i.reset(new DbInitializerPostgreSql(database)); break; case DbType::Unknown: qCCritical(AKONADISERVER_LOG) << database.driverName() << "backend not supported"; break; } i->mSchema = schema; return i; } DbInitializer::DbInitializer(const QSqlDatabase &database) : mDatabase(database) , mSchema(nullptr) , mTestInterface(nullptr) { m_introspector = DbIntrospector::createInstance(mDatabase); } DbInitializer::~DbInitializer() { } bool DbInitializer::run() { try { - qCDebug(AKONADISERVER_LOG) << "DbInitializer::run()"; + qCInfo(AKONADISERVER_LOG) << "Running DB initializer"; Q_FOREACH (const TableDescription &table, mSchema->tables()) { if (!checkTable(table)) { return false; } } Q_FOREACH (const RelationDescription &relation, mSchema->relations()) { if (!checkRelation(relation)) { return false; } } #ifndef DBINITIALIZER_UNITTEST // Now finally check and set the generation identifier if necessary SchemaVersion version = SchemaVersion::retrieveAll().first(); if (version.generation() == 0) { version.setGeneration(QDateTime::currentDateTimeUtc().toTime_t()); version.update(); qCDebug(AKONADISERVER_LOG) << "Generation:" << version.generation(); } #endif - qCDebug(AKONADISERVER_LOG) << "DbInitializer::run() done"; + qCInfo(AKONADISERVER_LOG) << "DB initializer done"; return true; } catch (const DbException &e) { mErrorMsg = QString::fromUtf8(e.what()); } return false; } bool DbInitializer::checkTable(const TableDescription &tableDescription) { qCDebug(AKONADISERVER_LOG) << "checking table " << tableDescription.name; if (!m_introspector->hasTable(tableDescription.name)) { // Get the CREATE TABLE statement for the specific SQL dialect const QString createTableStatement = buildCreateTableStatement(tableDescription); qCDebug(AKONADISERVER_LOG) << createTableStatement; execQuery(createTableStatement); } else { // Check for every column whether it exists, and add the missing ones Q_FOREACH (const ColumnDescription &columnDescription, tableDescription.columns) { if (!m_introspector->hasColumn(tableDescription.name, columnDescription.name)) { // Don't add the column on update, DbUpdater will add it if (columnDescription.noUpdate) { continue; } // Get the ADD COLUMN statement for the specific SQL dialect const QString statement = buildAddColumnStatement(tableDescription, columnDescription); qCDebug(AKONADISERVER_LOG) << statement; execQuery(statement); } } // NOTE: we do intentionally not delete any columns here, we defer that to the updater, // very likely previous columns contain data that needs to be moved to a new column first. } // Add initial data if table is empty if (tableDescription.data.isEmpty()) { return true; } if (m_introspector->isTableEmpty(tableDescription.name)) { Q_FOREACH (const DataDescription &dataDescription, tableDescription.data) { // Get the INSERT VALUES statement for the specific SQL dialect const QString statement = buildInsertValuesStatement(tableDescription, dataDescription); qCDebug(AKONADISERVER_LOG) << statement; execQuery(statement); } } return true; } void DbInitializer::checkForeignKeys(const TableDescription &tableDescription) { try { const QVector existingForeignKeys = m_introspector->foreignKeyConstraints(tableDescription.name); Q_FOREACH (const ColumnDescription &column, tableDescription.columns) { DbIntrospector::ForeignKey existingForeignKey; Q_FOREACH (const DbIntrospector::ForeignKey &fk, existingForeignKeys) { if (QString::compare(fk.column, column.name, Qt::CaseInsensitive) == 0) { existingForeignKey = fk; break; } } if (!column.refTable.isEmpty() && !column.refColumn.isEmpty()) { if (!existingForeignKey.column.isEmpty()) { // there's a constraint on this column, check if it's the correct one if (QString::compare(existingForeignKey.refTable, column.refTable + QLatin1Literal("table"), Qt::CaseInsensitive) == 0 && QString::compare(existingForeignKey.refColumn, column.refColumn, Qt::CaseInsensitive) == 0 && QString::compare(existingForeignKey.onUpdate, referentialActionToString(column.onUpdate), Qt::CaseInsensitive) == 0 && QString::compare(existingForeignKey.onDelete, referentialActionToString(column.onDelete), Qt::CaseInsensitive) == 0) { continue; // all good } const auto statements = buildRemoveForeignKeyConstraintStatements(existingForeignKey, tableDescription); if (!statements.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "Found existing foreign constraint that doesn't match the schema:" << existingForeignKey.name << existingForeignKey.column << existingForeignKey.refTable << existingForeignKey.refColumn; m_removedForeignKeys << statements; } } const auto statements = buildAddForeignKeyConstraintStatements(tableDescription, column); if (statements.isEmpty()) { // not supported return; } m_pendingForeignKeys << statements; } else if (!existingForeignKey.column.isEmpty()) { // constraint exists but we don't want one here const auto statements = buildRemoveForeignKeyConstraintStatements(existingForeignKey, tableDescription); if (!statements.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "Found unexpected foreign key constraint:" << existingForeignKey.name << existingForeignKey.column << existingForeignKey.refTable << existingForeignKey.refColumn; m_removedForeignKeys << statements; } } } } catch (const DbException &e) { qCDebug(AKONADISERVER_LOG) << "Fixing foreign key constraints failed:" << e.what(); } } void DbInitializer::checkIndexes(const TableDescription &tableDescription) { // Add indices Q_FOREACH (const IndexDescription &indexDescription, tableDescription.indexes) { // sqlite3 needs unique index identifiers per db const QString indexName = QStringLiteral("%1_%2").arg(tableDescription.name, indexDescription.name); if (!m_introspector->hasIndex(tableDescription.name, indexName)) { // Get the CREATE INDEX statement for the specific SQL dialect m_pendingIndexes << buildCreateIndexStatement(tableDescription, indexDescription); } } } bool DbInitializer::checkRelation(const RelationDescription &relationDescription) { return checkTable(RelationTableDescription(relationDescription)); } QString DbInitializer::errorMsg() const { return mErrorMsg; } bool DbInitializer::updateIndexesAndConstraints() { Q_FOREACH (const TableDescription &table, mSchema->tables()) { // Make sure the foreign key constraints are all there checkForeignKeys(table); checkIndexes(table); } Q_FOREACH (const RelationDescription &relation, mSchema->relations()) { RelationTableDescription relTable(relation); checkForeignKeys(relTable); checkIndexes(relTable); } try { if (!m_pendingIndexes.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "Updating indexes"; execPendingQueries(m_pendingIndexes); m_pendingIndexes.clear(); } if (!m_removedForeignKeys.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "Removing invalid foreign key constraints"; execPendingQueries(m_removedForeignKeys); m_removedForeignKeys.clear(); } if (!m_pendingForeignKeys.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "Adding new foreign key constraints"; execPendingQueries(m_pendingForeignKeys); m_pendingForeignKeys.clear(); } } catch (const DbException &e) { - qCDebug(AKONADISERVER_LOG) << "Updating index failed: " << e.what(); + qCCritical(AKONADISERVER_LOG) << "Updating index failed: " << e.what(); return false; } qCDebug(AKONADISERVER_LOG) << "Indexes successfully created"; return true; } void DbInitializer::execPendingQueries(const QStringList &queries) { for (const QString &statement : queries) { qCDebug(AKONADISERVER_LOG) << statement; execQuery(statement); } } QString DbInitializer::sqlType(const ColumnDescription &col, int size) const { Q_UNUSED(size); if (col.type == QLatin1String("int")) { return QStringLiteral("INTEGER"); } if (col.type == QLatin1String("qint64")) { return QStringLiteral("BIGINT"); } if (col.type == QLatin1String("QString")) { return QStringLiteral("TEXT"); } if (col.type == QLatin1String("QByteArray")) { return QStringLiteral("LONGBLOB"); } if (col.type == QLatin1String("QDateTime")) { return QStringLiteral("TIMESTAMP"); } if (col.type == QLatin1String("bool")) { return QStringLiteral("BOOL"); } if (col.isEnum) { return QStringLiteral("TINYINT"); } - qCDebug(AKONADISERVER_LOG) << "Invalid type" << col.type; + qCCritical(AKONADISERVER_LOG) << "Invalid type" << col.type; Q_ASSERT(false); return QString(); } QString DbInitializer::sqlValue(const ColumnDescription &col, const QString &value) const { if (col.type == QLatin1String("QDateTime") && value == QLatin1String("QDateTime::currentDateTimeUtc()")) { return QStringLiteral("CURRENT_TIMESTAMP"); } else if (col.isEnum) { return QString::number(col.enumValueMap[value]); } return value; } QString DbInitializer::buildAddColumnStatement(const TableDescription &tableDescription, const ColumnDescription &columnDescription) const { return QStringLiteral("ALTER TABLE %1 ADD COLUMN %2").arg(tableDescription.name, buildColumnStatement(columnDescription, tableDescription)); } QString DbInitializer::buildCreateIndexStatement(const TableDescription &tableDescription, const IndexDescription &indexDescription) const { const QString indexName = QStringLiteral("%1_%2").arg(tableDescription.name, indexDescription.name); QStringList columns; if (indexDescription.sort.isEmpty()) { columns = indexDescription.columns; } else { columns.reserve(indexDescription.columns.count()); std::transform(indexDescription.columns.cbegin(), indexDescription.columns.cend(), std::back_insert_iterator(columns), [&indexDescription](const QString & column) { return QStringLiteral("%1 %2").arg(column, indexDescription.sort); }); } return QStringLiteral("CREATE %1 INDEX %2 ON %3 (%4)") .arg(indexDescription.isUnique ? QStringLiteral("UNIQUE") : QString(), indexName, tableDescription.name, columns.join(QLatin1Char(','))); } QStringList DbInitializer::buildAddForeignKeyConstraintStatements(const TableDescription &table, const ColumnDescription &column) const { Q_UNUSED(table); Q_UNUSED(column); return {}; } QStringList DbInitializer::buildRemoveForeignKeyConstraintStatements(const DbIntrospector::ForeignKey &fk, const TableDescription &table) const { Q_UNUSED(fk); Q_UNUSED(table); return {}; } QString DbInitializer::buildReferentialAction(ColumnDescription::ReferentialAction onUpdate, ColumnDescription::ReferentialAction onDelete) { return QLatin1Literal("ON UPDATE ") + referentialActionToString(onUpdate) + QLatin1Literal(" ON DELETE ") + referentialActionToString(onDelete); } QString DbInitializer::referentialActionToString(ColumnDescription::ReferentialAction action) { switch (action) { case ColumnDescription::Cascade: return QStringLiteral("CASCADE"); case ColumnDescription::Restrict: return QStringLiteral("RESTRICT"); case ColumnDescription::SetNull: return QStringLiteral("SET NULL"); } Q_ASSERT(!"invalid referential action enum!"); return QString(); } QString DbInitializer::buildPrimaryKeyStatement(const TableDescription &table) { QStringList cols; for (const ColumnDescription &column : qAsConst(table.columns)) { if (column.isPrimaryKey) { cols.push_back(column.name); } } return QLatin1Literal("PRIMARY KEY (") + cols.join(QStringLiteral(", ")) + QLatin1Char(')'); } void DbInitializer::execQuery(const QString &queryString) { // if ( Q_UNLIKELY( mTestInterface ) ) { Qt 4.7 has no Q_UNLIKELY yet if (mTestInterface) { mTestInterface->execStatement(queryString); return; } QSqlQuery query(mDatabase); if (!query.exec(queryString)) { throw DbException(query); } } void DbInitializer::setTestInterface(TestInterface *interface) { mTestInterface = interface; } void DbInitializer::setIntrospector(const DbIntrospector::Ptr &introspector) { m_introspector = introspector; } diff --git a/src/server/storage/entity.cpp b/src/server/storage/entity.cpp index bfc376195..e178afed1 100644 --- a/src/server/storage/entity.cpp +++ b/src/server/storage/entity.cpp @@ -1,189 +1,183 @@ /*************************************************************************** * Copyright (C) 2006 by Andreas Gungl * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "entity.h" #include "datastore.h" #include "countquerybuilder.h" #include #include using namespace Akonadi::Server; Entity::Entity() : m_id(-1) { } Entity::Entity(qint64 id) : m_id(id) { } Entity::~Entity() { } qint64 Entity::id() const { return m_id; } void Entity::setId(qint64 id) { m_id = id; } bool Entity::isValid() const { return m_id != -1; } QSqlDatabase Entity::database() { return DataStore::self()->database(); } int Entity::countImpl(const QString &tableName, const QString &column, const QVariant &value) { QSqlDatabase db = database(); if (!db.isOpen()) { return -1; } CountQueryBuilder builder(tableName); builder.addValueCondition(column, Query::Equals, value); if (!builder.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during counting records in table" << tableName - << builder.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error counting records in table" << tableName; return -1; } return builder.result(); } bool Entity::removeImpl(const QString &tableName, const QString &column, const QVariant &value) { QSqlDatabase db = database(); if (!db.isOpen()) { return false; } QueryBuilder builder(tableName, QueryBuilder::Delete); builder.addValueCondition(column, Query::Equals, value); if (!builder.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during deleting records from table" - << tableName << builder.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error during deleting records from table" << tableName; return false; } return true; } bool Entity::relatesToImpl(const QString &tableName, const QString &leftColumn, const QString &rightColumn, qint64 leftId, qint64 rightId) { QSqlDatabase db = database(); if (!db.isOpen()) { return false; } CountQueryBuilder builder(tableName); builder.addValueCondition(leftColumn, Query::Equals, leftId); builder.addValueCondition(rightColumn, Query::Equals, rightId); if (!builder.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during counting records in table" << tableName - << builder.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error during counting records in table" << tableName; return false; } if (builder.result() > 0) { return true; } return false; } bool Entity::addToRelationImpl(const QString &tableName, const QString &leftColumn, const QString &rightColumn, qint64 leftId, qint64 rightId) { QSqlDatabase db = database(); if (!db.isOpen()) { return false; } QueryBuilder qb(tableName, QueryBuilder::Insert); qb.setColumnValue(leftColumn, leftId); qb.setColumnValue(rightColumn, rightId); qb.setIdentificationColumn(QString()); if (!qb.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during adding a record to table" << tableName - << qb.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error during adding a record to table" << tableName; return false; } return true; } bool Entity::removeFromRelationImpl(const QString &tableName, const QString &leftColumn, const QString &rightColumn, qint64 leftId, qint64 rightId) { QSqlDatabase db = database(); if (!db.isOpen()) { return false; } QueryBuilder builder(tableName, QueryBuilder::Delete); builder.addValueCondition(leftColumn, Query::Equals, leftId); builder.addValueCondition(rightColumn, Query::Equals, rightId); if (!builder.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during removing a record from relation table" << tableName - << builder.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error during removing a record from relation table" << tableName; return false; } return true; } bool Entity::clearRelationImpl(const QString &tableName, const QString &leftColumn, const QString &rightColumn, qint64 id, RelationSide side) { QSqlDatabase db = database(); if (!db.isOpen()) { return false; } QueryBuilder builder(tableName, QueryBuilder::Delete); switch (side) { case Left: builder.addValueCondition(leftColumn, Query::Equals, id); break; case Right: builder.addValueCondition(rightColumn, Query::Equals, id); break; default: qFatal("Invalid enum value"); } if (!builder.exec()) { - qCDebug(AKONADISERVER_LOG) << "Error during clearing relation table" << tableName - << "for id" << id << builder.query().lastError().text(); + qCWarning(AKONADISERVER_LOG) << "Error during clearing relation table" << tableName << "for ID" << id; return false; } return true; } diff --git a/src/server/storage/itemretrievalmanager.cpp b/src/server/storage/itemretrievalmanager.cpp index f8e9f78bd..ec25e1d16 100644 --- a/src/server/storage/itemretrievalmanager.cpp +++ b/src/server/storage/itemretrievalmanager.cpp @@ -1,255 +1,259 @@ /* Copyright (c) 2009 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "itemretrievalmanager.h" #include "itemretrievalrequest.h" #include "itemretrievaljob.h" #include "dbusconnectionpool.h" #include "akonadiserver_debug.h" #include "resourceinterface.h" #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; ItemRetrievalManager *ItemRetrievalManager::sInstance = nullptr; class ItemRetrievalJobFactory : public AbstractItemRetrievalJobFactory { AbstractItemRetrievalJob *retrievalJob(ItemRetrievalRequest *request, QObject *parent) override { return new ItemRetrievalJob(request, parent); } }; ItemRetrievalManager::ItemRetrievalManager(QObject *parent) : ItemRetrievalManager(new ItemRetrievalJobFactory, parent) { } ItemRetrievalManager::ItemRetrievalManager(AbstractItemRetrievalJobFactory *factory, QObject *parent) : AkThread(QStringLiteral("ItemRetrievalManager"), QThread::HighPriority, parent) , mJobFactory(factory) { qDBusRegisterMetaType(); Q_ASSERT(sInstance == nullptr); sInstance = this; mLock = new QReadWriteLock(); mWaitCondition = new QWaitCondition(); } ItemRetrievalManager::~ItemRetrievalManager() { quitThread(); delete mWaitCondition; delete mLock; sInstance = nullptr; } void ItemRetrievalManager::init() { AkThread::init(); QDBusConnection conn = DBusConnectionPool::threadConnection(); connect(conn.interface(), &QDBusConnectionInterface::serviceOwnerChanged, this, &ItemRetrievalManager::serviceOwnerChanged); connect(this, &ItemRetrievalManager::requestAdded, this, &ItemRetrievalManager::processRequest, Qt::QueuedConnection); } ItemRetrievalManager *ItemRetrievalManager::instance() { Q_ASSERT(sInstance); return sInstance; } // called within the retrieval thread void ItemRetrievalManager::serviceOwnerChanged(const QString &serviceName, const QString &oldOwner, const QString &newOwner) { Q_UNUSED(newOwner); if (oldOwner.isEmpty()) { return; } DBus::AgentType type = DBus::Unknown; const QString resourceId = DBus::parseAgentServiceName(serviceName, type); if (resourceId.isEmpty() || type != DBus::Resource) { return; } - qCDebug(AKONADISERVER_LOG) << "Lost connection to resource" << serviceName << ", discarding cached interface"; + qCDebug(AKONADISERVER_LOG) << "ItemRetrievalManager lost connection to resource" << serviceName << ", discarding cached interface"; mResourceInterfaces.remove(resourceId); } // called within the retrieval thread org::freedesktop::Akonadi::Resource *ItemRetrievalManager::resourceInterface(const QString &id) { if (id.isEmpty()) { return nullptr; } org::freedesktop::Akonadi::Resource *iface = mResourceInterfaces.value(id); if (iface && iface->isValid()) { return iface; } delete iface; iface = new org::freedesktop::Akonadi::Resource(DBus::agentServiceName(id, DBus::Resource), QStringLiteral("/"), DBusConnectionPool::threadConnection(), this); if (!iface || !iface->isValid()) { - qCCritical(AKONADISERVER_LOG) << QStringLiteral("Cannot connect to agent instance with identifier '%1', error message: '%2'") - .arg(id, iface ? iface->lastError().message() : QString()); + qCCritical(AKONADISERVER_LOG, "Cannot connect to agent instance with identifier '%s', error message: '%s'", + qUtf8Printable(id), qUtf8Printable(iface ? iface->lastError().message() : QString())); delete iface; return nullptr; } // DBus calls can take some time to reply -- e.g. if a huge local mbox has to be parsed first. iface->setTimeout(5 * 60 * 1000); // 5 minutes, rather than 25 seconds mResourceInterfaces.insert(id, iface); return iface; } // called from any thread void ItemRetrievalManager::requestItemDelivery(ItemRetrievalRequest *req) { mLock->lockForWrite(); - qCDebug(AKONADISERVER_LOG) << "posting retrieval request for items" << req->ids << " there are " - << mPendingRequests.size() << " queues and " - << mPendingRequests[req->resourceId].size() << " items in mine"; + qCDebug(AKONADISERVER_LOG) << "ItemRetrievalManager posting retrieval request for items" << req->ids + << "to" <resourceId << ". There are" << mPendingRequests.size() << "request queues and" + << mPendingRequests[req->resourceId].size() << "items mine"; mPendingRequests[req->resourceId].append(req); mLock->unlock(); Q_EMIT requestAdded(); #if 0 mLock->lockForRead(); Q_FOREVER { //qCDebug(AKONADISERVER_LOG) << "checking if request for item" << req->id << "has been processed..."; if (req->processed) { QScopedPointer reqDeleter(req); Q_ASSERT(!mPendingRequests[req->resourceId].contains(req)); const QString errorMsg = req->errorMsg; mLock->unlock(); if (errorMsg.isEmpty()) { qCDebug(AKONADISERVER_LOG) << "request for items" << req->ids << "succeeded"; return; } else { qCDebug(AKONADISERVER_LOG) << "request for items" << req->ids << "failed:" << errorMsg; throw ItemRetrieverException(errorMsg); } } else { qCDebug(AKONADISERVER_LOG) << "request for items" << req->ids << "still pending - waiting"; mWaitCondition->wait(mLock); qCDebug(AKONADISERVER_LOG) << "continuing"; } } throw ItemRetrieverException("WTF?"); #endif } // called within the retrieval thread void ItemRetrievalManager::processRequest() { QVector > newJobs; mLock->lockForWrite(); // look for idle resources for (auto it = mPendingRequests.begin(); it != mPendingRequests.end();) { if (it.value().isEmpty()) { it = mPendingRequests.erase(it); continue; } if (!mCurrentJobs.contains(it.key()) || mCurrentJobs.value(it.key()) == nullptr) { // TODO: check if there is another one for the same uid with more parts requested ItemRetrievalRequest *req = it.value().takeFirst(); Q_ASSERT(req->resourceId == it.key()); AbstractItemRetrievalJob *job = mJobFactory->retrievalJob(req, this); connect(job, &AbstractItemRetrievalJob::requestCompleted, this, &ItemRetrievalManager::retrievalJobFinished); mCurrentJobs.insert(req->resourceId, job); // delay job execution until after we unlocked the mutex, since the job can emit the finished signal immediately in some cases newJobs.append(qMakePair(job, req->resourceId)); qCDebug(AKONADISERVER_LOG) << "ItemRetrievalJob" << job << "started for request" << req; } ++it; } bool nothingGoingOn = mPendingRequests.isEmpty() && mCurrentJobs.isEmpty() && newJobs.isEmpty(); mLock->unlock(); if (nothingGoingOn) { // someone asked as to process requests although everything is done already, he might still be waiting return; } for (auto it = newJobs.constBegin(), end = newJobs.constEnd(); it != end; ++it) { if (ItemRetrievalJob *j = qobject_cast((*it).first)) { j->setInterface(resourceInterface((*it).second)); } (*it).first->start(); } } void ItemRetrievalManager::retrievalJobFinished(ItemRetrievalRequest *request, const QString &errorMsg) { - qCDebug(AKONADISERVER_LOG) << "ItemRetrievalJob finished for request" << request << ", error:" << errorMsg; + if (errorMsg.isEmpty()) { + qCInfo(AKONADISERVER_LOG) << "ItemRetrievalJob for request" << request << "finished"; + } else { + qCWarning(AKONADISERVER_LOG) << "ItemRetrievalJob for request" << request << "finished with error:" << errorMsg; + } mLock->lockForWrite(); request->errorMsg = errorMsg; request->processed = true; Q_ASSERT(mCurrentJobs.contains(request->resourceId)); mCurrentJobs.remove(request->resourceId); // TODO check if (*it)->parts is a subset of currentRequest->parts for (QList::Iterator it = mPendingRequests[request->resourceId].begin(); it != mPendingRequests[request->resourceId].end();) { if ((*it)->ids == request->ids) { qCDebug(AKONADISERVER_LOG) << "someone else requested item" << request->ids << "as well, marking as processed"; (*it)->errorMsg = errorMsg; (*it)->processed = true; Q_EMIT requestFinished(*it); it = mPendingRequests[request->resourceId].erase(it); } else { ++it; } } mLock->unlock(); Q_EMIT requestFinished(request); Q_EMIT requestAdded(); // trigger processRequest() again, in case there is more in the queues } void ItemRetrievalManager::triggerCollectionSync(const QString &resource, qint64 colId) { org::freedesktop::Akonadi::Resource *interface = resourceInterface(resource); if (interface) { interface->synchronizeCollection(colId); } } void ItemRetrievalManager::triggerCollectionTreeSync(const QString &resource) { org::freedesktop::Akonadi::Resource *interface = resourceInterface(resource); if (interface) { interface->synchronizeCollectionTree(); } } diff --git a/src/server/storage/notificationcollector.cpp b/src/server/storage/notificationcollector.cpp index 00f100714..341cc8faf 100644 --- a/src/server/storage/notificationcollector.cpp +++ b/src/server/storage/notificationcollector.cpp @@ -1,622 +1,624 @@ /* Copyright (c) 2006 - 2007 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "notificationcollector.h" #include "storage/datastore.h" #include "storage/entity.h" #include "storage/collectionstatistics.h" #include "handlerhelper.h" #include "cachecleaner.h" #include "intervalcheck.h" #include "search/searchmanager.h" #include "akonadi.h" #include "handler/search.h" #include "notificationmanager.h" #include "aggregatedfetchscope.h" #include "selectquerybuilder.h" #include "handler/fetchhelper.h" #include "connection.h" #include "akonadiserver_debug.h" #include using namespace Akonadi; using namespace Akonadi::Server; NotificationCollector::NotificationCollector(DataStore *db) : mDb(db) { QObject::connect(db, &DataStore::transactionCommitted, [this]() { if (!mIgnoreTransactions) { dispatchNotifications(); } }); QObject::connect(db, &DataStore::transactionRolledBack, [this]() { if (!mIgnoreTransactions) { clear(); } }); } void NotificationCollector::itemAdded(const PimItem &item, bool seen, const Collection &collection, const QByteArray &resource) { SearchManager::instance()->scheduleSearchUpdate(); CollectionStatistics::self()->itemAdded(collection, item.size(), seen); itemNotification(Protocol::ItemChangeNotification::Add, item, collection, Collection(), resource); } void NotificationCollector::itemChanged(const PimItem &item, const QSet &changedParts, const Collection &collection, const QByteArray &resource) { SearchManager::instance()->scheduleSearchUpdate(); itemNotification(Protocol::ItemChangeNotification::Modify, item, collection, Collection(), resource, changedParts); } void NotificationCollector::itemsFlagsChanged(const PimItem::List &items, const QSet &addedFlags, const QSet &removedFlags, const Collection &collection, const QByteArray &resource) { int seenCount = (addedFlags.contains(AKONADI_FLAG_SEEN) || addedFlags.contains(AKONADI_FLAG_IGNORED) ? items.count() : 0); seenCount -= (removedFlags.contains(AKONADI_FLAG_SEEN) || removedFlags.contains(AKONADI_FLAG_IGNORED) ? items.count() : 0); CollectionStatistics::self()->itemsSeenChanged(collection, seenCount); itemNotification(Protocol::ItemChangeNotification::ModifyFlags, items, collection, Collection(), resource, QSet(), addedFlags, removedFlags); } void NotificationCollector::itemsTagsChanged(const PimItem::List &items, const QSet &addedTags, const QSet &removedTags, const Collection &collection, const QByteArray &resource) { itemNotification(Protocol::ItemChangeNotification::ModifyTags, items, collection, Collection(), resource, QSet(), QSet(), QSet(), addedTags, removedTags); } void NotificationCollector::itemsRelationsChanged(const PimItem::List &items, const Relation::List &addedRelations, const Relation::List &removedRelations, const Collection &collection, const QByteArray &resource) { itemNotification(Protocol::ItemChangeNotification::ModifyRelations, items, collection, Collection(), resource, QSet(), QSet(), QSet(), QSet(), QSet(), addedRelations, removedRelations); } void NotificationCollector::itemsMoved(const PimItem::List &items, const Collection &collectionSrc, const Collection &collectionDest, const QByteArray &sourceResource) { SearchManager::instance()->scheduleSearchUpdate(); itemNotification(Protocol::ItemChangeNotification::Move, items, collectionSrc, collectionDest, sourceResource); } void NotificationCollector::itemsRemoved(const PimItem::List &items, const Collection &collection, const QByteArray &resource) { itemNotification(Protocol::ItemChangeNotification::Remove, items, collection, Collection(), resource); } void NotificationCollector::itemsLinked(const PimItem::List &items, const Collection &collection) { itemNotification(Protocol::ItemChangeNotification::Link, items, collection, Collection(), QByteArray()); } void NotificationCollector::itemsUnlinked(const PimItem::List &items, const Collection &collection) { itemNotification(Protocol::ItemChangeNotification::Unlink, items, collection, Collection(), QByteArray()); } void NotificationCollector::collectionAdded(const Collection &collection, const QByteArray &resource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionAdded(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionAdded(collection.id()); } collectionNotification(Protocol::CollectionChangeNotification::Add, collection, collection.parentId(), -1, resource); } void NotificationCollector::collectionChanged(const Collection &collection, const QList &changes, const QByteArray &resource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionChanged(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionChanged(collection.id()); } if (changes.contains(AKONADI_PARAM_ENABLED) || changes.contains(AKONADI_PARAM_REFERENCED)) { CollectionStatistics::self()->invalidateCollection(collection); } collectionNotification(Protocol::CollectionChangeNotification::Modify, collection, collection.parentId(), -1, resource, changes.toSet()); } void NotificationCollector::collectionMoved(const Collection &collection, const Collection &source, const QByteArray &resource, const QByteArray &destResource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionChanged(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionChanged(collection.id()); } collectionNotification(Protocol::CollectionChangeNotification::Move, collection, source.id(), collection.parentId(), resource, QSet(), destResource); } void NotificationCollector::collectionRemoved(const Collection &collection, const QByteArray &resource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionRemoved(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionRemoved(collection.id()); } CollectionStatistics::self()->invalidateCollection(collection); collectionNotification(Protocol::CollectionChangeNotification::Remove, collection, collection.parentId(), -1, resource); } void NotificationCollector::collectionSubscribed(const Collection &collection, const QByteArray &resource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionAdded(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionAdded(collection.id()); } collectionNotification(Protocol::CollectionChangeNotification::Subscribe, collection, collection.parentId(), -1, resource, QSet()); } void NotificationCollector::collectionUnsubscribed(const Collection &collection, const QByteArray &resource) { if (auto cleaner = AkonadiServer::instance()->cacheCleaner()) { cleaner->collectionRemoved(collection.id()); } if (auto checker = AkonadiServer::instance()->intervalChecker()) { checker->collectionRemoved(collection.id()); } CollectionStatistics::self()->invalidateCollection(collection); collectionNotification(Protocol::CollectionChangeNotification::Unsubscribe, collection, collection.parentId(), -1, resource, QSet()); } void NotificationCollector::tagAdded(const Tag &tag) { tagNotification(Protocol::TagChangeNotification::Add, tag); } void NotificationCollector::tagChanged(const Tag &tag) { tagNotification(Protocol::TagChangeNotification::Modify, tag); } void NotificationCollector::tagRemoved(const Tag &tag, const QByteArray &resource, const QString &remoteId) { tagNotification(Protocol::TagChangeNotification::Remove, tag, resource, remoteId); } void NotificationCollector::relationAdded(const Relation &relation) { relationNotification(Protocol::RelationChangeNotification::Add, relation); } void NotificationCollector::relationRemoved(const Relation &relation) { relationNotification(Protocol::RelationChangeNotification::Remove, relation); } void NotificationCollector::clear() { mNotifications.clear(); } void NotificationCollector::setConnection(Connection *connection) { mConnection = connection; } void NotificationCollector::itemNotification(Protocol::ItemChangeNotification::Operation op, const PimItem &item, const Collection &collection, const Collection &collectionDest, const QByteArray &resource, const QSet &parts) { PimItem::List items; items << item; itemNotification(op, items, collection, collectionDest, resource, parts); } void NotificationCollector::itemNotification(Protocol::ItemChangeNotification::Operation op, const PimItem::List &items, const Collection &collection, const Collection &collectionDest, const QByteArray &resource, const QSet &parts, const QSet &addedFlags, const QSet &removedFlags, const QSet &addedTags, const QSet &removedTags, const Relation::List &addedRelations, const Relation::List &removedRelations) { QMap > vCollections; if ((op == Protocol::ItemChangeNotification::Modify) || (op == Protocol::ItemChangeNotification::ModifyFlags) || (op == Protocol::ItemChangeNotification::ModifyTags) || (op == Protocol::ItemChangeNotification::ModifyRelations)) { vCollections = DataStore::self()->virtualCollections(items); } auto msg = Protocol::ItemChangeNotificationPtr::create(); if (mConnection) { msg->setSessionId(mConnection->sessionId()); } msg->setOperation(op); msg->setItemParts(parts); msg->setAddedFlags(addedFlags); msg->setRemovedFlags(removedFlags); msg->setAddedTags(addedTags); msg->setRemovedTags(removedTags); if (!addedRelations.isEmpty()) { QSet rels; Q_FOREACH (const Relation &rel, addedRelations) { rels.insert(Protocol::ItemChangeNotification::Relation(rel.leftId(), rel.rightId(), rel.relationType().name())); } msg->setAddedRelations(rels); } if (!removedRelations.isEmpty()) { QSet rels; Q_FOREACH (const Relation &rel, removedRelations) { rels.insert(Protocol::ItemChangeNotification::Relation(rel.leftId(), rel.rightId(), rel.relationType().name())); } msg->setRemovedRelations(rels); } if (collectionDest.isValid()) { QByteArray destResourceName; destResourceName = collectionDest.resource().name().toLatin1(); msg->setDestinationResource(destResourceName); } msg->setParentDestCollection(collectionDest.id()); QVector ntfItems; Q_FOREACH (const PimItem &item, items) { Protocol::FetchItemsResponse i; i.setId(item.id()); i.setRemoteId(item.remoteId()); i.setRemoteRevision(item.remoteRevision()); i.setMimeType(item.mimeType().name()); ntfItems.push_back(std::move(i)); } /* Notify all virtual collections the items are linked to. */ QHash virtItems; for (const auto &ntfItem : ntfItems) { virtItems.insert(ntfItem.id(), std::move(ntfItem)); } auto iter = vCollections.constBegin(), endIter = vCollections.constEnd(); for (; iter != endIter; ++iter) { auto copy = Protocol::ItemChangeNotificationPtr::create(*msg); QVector items; items.reserve(iter->size()); for (const auto &item : qAsConst(*iter)) { items.append(virtItems.value(item.id())); } copy->setItems(items); copy->setParentCollection(iter.key()); copy->setResource(resource); CollectionStatistics::self()->invalidateCollection(Collection::retrieveById(iter.key())); dispatchNotification(copy); } msg->setItems(ntfItems); Collection col; if (!collection.isValid()) { msg->setParentCollection(items.first().collection().id()); col = items.first().collection(); } else { msg->setParentCollection(collection.id()); col = collection; } QByteArray res = resource; if (res.isEmpty()) { if (col.resourceId() <= 0) { col = Collection::retrieveById(col.id()); } res = col.resource().name().toLatin1(); } msg->setResource(res); // Add and ModifyFlags are handled incrementally // (see itemAdded() and itemsFlagsChanged()) if (msg->operation() != Protocol::ItemChangeNotification::Add && msg->operation() != Protocol::ItemChangeNotification::ModifyFlags) { CollectionStatistics::self()->invalidateCollection(col); } dispatchNotification(msg); } void NotificationCollector::collectionNotification(Protocol::CollectionChangeNotification::Operation op, const Collection &collection, Collection::Id source, Collection::Id destination, const QByteArray &resource, const QSet &changes, const QByteArray &destResource) { auto msg = Protocol::CollectionChangeNotificationPtr::create(); msg->setOperation(op); if (mConnection) { msg->setSessionId(mConnection->sessionId()); } msg->setParentCollection(source); msg->setParentDestCollection(destination); msg->setDestinationResource(destResource); msg->setChangedParts(changes); auto msgCollection = HandlerHelper::fetchCollectionsResponse(collection); if (auto mgr = AkonadiServer::instance()->notificationManager()) { auto fetchScope = mgr->collectionFetchScope(); // Make sure we have all the data if (!fetchScope->fetchIdOnly() && msgCollection.name().isEmpty()) { const auto col = Collection::retrieveById(msgCollection.id()); const auto mts = col.mimeTypes(); QStringList mimeTypes; mimeTypes.reserve(mts.size()); for (const auto &mt : mts) { mimeTypes.push_back(mt.name()); } msgCollection = HandlerHelper::fetchCollectionsResponse(col, {}, false, 0, {}, {}, false, mimeTypes); } // Get up-to-date statistics if (fetchScope->fetchStatistics()) { Collection col; col.setId(msgCollection.id()); const auto stats = CollectionStatistics::self()->statistics(col); msgCollection.setStatistics(Protocol::FetchCollectionStatsResponse(stats.count, stats.count - stats.read, stats.size)); } // Get attributes const auto requestedAttrs = fetchScope->attributes(); auto msgColAttrs = msgCollection.attributes(); // TODO: This assumes that we have either none or all attributes in msgCollection if (msgColAttrs.isEmpty() && !requestedAttrs.isEmpty()) { SelectQueryBuilder qb; qb.addColumn(CollectionAttribute::typeFullColumnName()); qb.addColumn(CollectionAttribute::valueFullColumnName()); qb.addValueCondition(CollectionAttribute::collectionIdFullColumnName(), Query::Equals, msgCollection.id()); Query::Condition cond(Query::Or); for (const auto &attr : requestedAttrs) { cond.addValueCondition(CollectionAttribute::typeFullColumnName(), Query::Equals, attr); } qb.addCondition(cond); if (!qb.exec()) { - qCWarning(AKONADISERVER_LOG) << "Failed to obtain collection attributes!"; + qCWarning(AKONADISERVER_LOG) << "NotificationCollector failed to query attributes for Collection" + << collection.name() << "(ID" << collection.id() << ")"; } const auto attrs = qb.result(); for (const auto &attr : attrs) { msgColAttrs.insert(attr.type(), attr.value()); } msgCollection.setAttributes(msgColAttrs); } } msg->setCollection(std::move(msgCollection)); if (!collection.enabled()) { msg->addMetadata("DISABLED"); } QByteArray res = resource; if (res.isEmpty()) { res = collection.resource().name().toLatin1(); } msg->setResource(res); dispatchNotification(msg); } void NotificationCollector::tagNotification(Protocol::TagChangeNotification::Operation op, const Tag &tag, const QByteArray &resource, const QString &remoteId) { auto msg = Protocol::TagChangeNotificationPtr::create(); msg->setOperation(op); if (mConnection) { msg->setSessionId(mConnection->sessionId()); } msg->setResource(resource); Protocol::FetchTagsResponse msgTag; msgTag.setId(tag.id()); msgTag.setRemoteId(remoteId.toUtf8()); if (auto mgr = AkonadiServer::instance()->notificationManager()) { auto fetchScope = mgr->tagFetchScope(); if (!fetchScope->fetchIdOnly() && msgTag.gid().isEmpty()) { msgTag = HandlerHelper::fetchTagsResponse(Tag::retrieveById(msgTag.id()), fetchScope->toFetchScope(), mConnection); } const auto requestedAttrs = fetchScope->attributes(); auto msgTagAttrs = msgTag.attributes(); if (msgTagAttrs.isEmpty() && !requestedAttrs.isEmpty()) { SelectQueryBuilder qb; qb.addColumn(TagAttribute::typeFullColumnName()); qb.addColumn(TagAttribute::valueFullColumnName()); qb.addValueCondition(TagAttribute::tagIdFullColumnName(), Query::Equals, msgTag.id()); Query::Condition cond(Query::Or); for (const auto &attr : requestedAttrs) { cond.addValueCondition(TagAttribute::typeFullColumnName(), Query::Equals, attr); } qb.addCondition(cond); if (!qb.exec()) { - qCWarning(AKONADISERVER_LOG) << "Failed to obtain tag attributes!"; + qCWarning(AKONADISERVER_LOG) << "NotificationCollection failed to query attributes for Tag" << tag.id(); } const auto attrs = qb.result(); for (const auto &attr : attrs) { msgTagAttrs.insert(attr.type(), attr.value()); } msgTag.setAttributes(msgTagAttrs); } } msg->setTag(std::move(msgTag)); dispatchNotification(msg); } void NotificationCollector::relationNotification(Protocol::RelationChangeNotification::Operation op, const Relation &relation) { auto msg = Protocol::RelationChangeNotificationPtr::create(); msg->setOperation(op); if (mConnection) { msg->setSessionId(mConnection->sessionId()); } msg->setRelation(HandlerHelper::fetchRelationsResponse(relation)); dispatchNotification(msg); } void NotificationCollector::completeNotification(const Protocol::ChangeNotificationPtr &changeMsg) { if (changeMsg->type() == Protocol::Command::ItemChangeNotification) { const auto msg = changeMsg.staticCast(); const auto mgr = AkonadiServer::instance()->notificationManager(); if (mgr && msg->operation() != Protocol::ItemChangeNotification::Remove) { if (mDb->inTransaction()) { - qCWarning(AKONADISERVER_LOG) << "FetchHelper requested from within a transaction, aborting, since this would deadlock!"; + qCWarning(AKONADISERVER_LOG) << "NotificationCollector requested FetchHelper from within a transaction." + << "Aborting since this would deadlock!"; return; } auto fetchScope = mgr->itemFetchScope(); // NOTE: Checking and retrieving missing elements for each Item manually // here would require a complex code (and I'm too lazy), so instead we simply // feed the Items to FetchHelper and retrieve them all with the setup from // the aggregated fetch scope. The worst case is that we re-fetch everything // we already have, but that's stil better than the pre-ntf-payload situation QVector ids; const auto items = msg->items(); ids.reserve(items.size()); bool allHaveRID = true; for (const auto &item : items) { ids.push_back(item.id()); allHaveRID &= !item.remoteId().isEmpty(); } // FetchHelper may trigger ItemRetriever, which needs RemoteID. If we // dont have one (maybe because the Resource has not stored it yet, // we emit a notification without it and leave it up to the Monitor // to retrieve the Item on demand - we should have a RID stored in // Akonadi by then. if (mConnection && (allHaveRID || msg->operation() != Protocol::ItemChangeNotification::Add)) { // Prevent transactions inside FetchHelper to recursively call our slot QScopedValueRollback ignoreTransactions(mIgnoreTransactions); mIgnoreTransactions = true; CommandContext context; auto itemFetchScope = fetchScope->toFetchScope(); auto tagFetchScope = mgr->tagFetchScope()->toFetchScope(); itemFetchScope.setFetch(Protocol::ItemFetchScope::CacheOnly); FetchHelper helper(mConnection, &context, Scope(ids), itemFetchScope, tagFetchScope); // The Item was just changed, which means the atime was // updated, no need to do it again a couple milliseconds later. helper.disableATimeUpdates(); QVector fetchedItems; auto callback = [&fetchedItems](Protocol::FetchItemsResponse &&cmd) { fetchedItems.push_back(std::move(cmd)); }; if (helper.fetchItems(std::move(callback))) { msg->setItems(fetchedItems); } else { - qCWarning(AKONADISERVER_LOG) << "Failed to retrieve Items for notification!"; + qCWarning(AKONADISERVER_LOG) << "NotificationCollector railed to retrieve Items for notification!"; } } else { QVector fetchedItems; for (const auto &item : items) { Protocol::FetchItemsResponse resp; resp.setId(item.id()); resp.setRevision(item.revision()); resp.setMimeType(item.mimeType()); resp.setParentId(item.parentId()); resp.setGid(item.gid()); resp.setSize(item.size()); resp.setMTime(item.mTime()); resp.setFlags(item.flags()); fetchedItems.push_back(std::move(resp)); } msg->setItems(fetchedItems); msg->setMustRetrieve(true); } } } } void NotificationCollector::dispatchNotification(const Protocol::ChangeNotificationPtr &msg) { if (!mDb || mDb->inTransaction()) { if (msg->type() == Protocol::Command::CollectionChangeNotification) { Protocol::CollectionChangeNotification::appendAndCompress(mNotifications, msg); } else { mNotifications.append(msg); } } else { completeNotification(msg); notify({msg}); } } void NotificationCollector::dispatchNotifications() { if (!mNotifications.isEmpty()) { for (auto &ntf : mNotifications) { completeNotification(ntf); } notify(std::move(mNotifications)); clear(); } } void NotificationCollector::notify(Protocol::ChangeNotificationList msgs) { if (auto mgr = AkonadiServer::instance()->notificationManager()) { QMetaObject::invokeMethod(mgr, "slotNotify", Qt::QueuedConnection, Q_ARG(Akonadi::Protocol::ChangeNotificationList, msgs)); } } diff --git a/src/server/storage/querybuilder.cpp b/src/server/storage/querybuilder.cpp index e2b2f3025..979fc04b6 100644 --- a/src/server/storage/querybuilder.cpp +++ b/src/server/storage/querybuilder.cpp @@ -1,640 +1,640 @@ /* Copyright (c) 2007 - 2012 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "querybuilder.h" #include "akonadiserver_debug.h" #ifndef QUERYBUILDER_UNITTEST #include "storage/datastore.h" #include "storage/querycache.h" #include "storage/storagedebugger.h" #endif #include #include #include using namespace Akonadi::Server; static QLatin1String compareOperatorToString(Query::CompareOperator op) { switch (op) { case Query::Equals: return QLatin1String(" = "); case Query::NotEquals: return QLatin1String(" <> "); case Query::Is: return QLatin1String(" IS "); case Query::IsNot: return QLatin1String(" IS NOT "); case Query::Less: return QLatin1String(" < "); case Query::LessOrEqual: return QLatin1String(" <= "); case Query::Greater: return QLatin1String(" > "); case Query::GreaterOrEqual: return QLatin1String(" >= "); case Query::In: return QLatin1String(" IN "); case Query::NotIn: return QLatin1String(" NOT IN "); case Query::Like: return QLatin1String(" LIKE "); } Q_ASSERT_X(false, "QueryBuilder::compareOperatorToString()", "Unknown compare operator."); return QLatin1String(""); } static QLatin1String logicOperatorToString(Query::LogicOperator op) { switch (op) { case Query::And: return QLatin1String(" AND "); case Query::Or: return QLatin1String(" OR "); } Q_ASSERT_X(false, "QueryBuilder::logicOperatorToString()", "Unknown logic operator."); return QLatin1String(""); } static QLatin1String sortOrderToString(Query::SortOrder order) { switch (order) { case Query::Ascending: return QLatin1String(" ASC"); case Query::Descending: return QLatin1String(" DESC"); } Q_ASSERT_X(false, "QueryBuilder::sortOrderToString()", "Unknown sort order."); return QLatin1String(""); } static void appendJoined(QString *statement, const QStringList &strings, const QLatin1String &glue = QLatin1String(", ")) { for (int i = 0, c = strings.size(); i < c; ++i) { *statement += strings.at(i); if (i + 1 < c) { *statement += glue; } } } QueryBuilder::QueryBuilder(const QString &table, QueryBuilder::QueryType type) : mTable(table) #ifndef QUERYBUILDER_UNITTEST , mDatabaseType(DbType::type(DataStore::self()->database())) , mQuery(DataStore::self()->database()) #else , mDatabaseType(DbType::Unknown) #endif , mType(type) , mIdentificationColumn() , mLimit(-1) , mDistinct(false) { static const QString defaultIdColumn = QStringLiteral("id"); mIdentificationColumn = defaultIdColumn; } void QueryBuilder::setDatabaseType(DbType::Type type) { mDatabaseType = type; } void QueryBuilder::addJoin(JoinType joinType, const QString &table, const Query::Condition &condition) { Q_ASSERT((joinType == InnerJoin && (mType == Select || mType == Update)) || (joinType == LeftJoin && mType == Select)); if (mJoinedTables.contains(table)) { // InnerJoin is more restrictive than a LeftJoin, hence use that in doubt mJoins[table].first = qMin(joinType, mJoins.value(table).first); mJoins[table].second.addCondition(condition); } else { mJoins[table] = qMakePair(joinType, condition); mJoinedTables << table; } } void QueryBuilder::addJoin(JoinType joinType, const QString &table, const QString &col1, const QString &col2) { Query::Condition condition; condition.addColumnCondition(col1, Query::Equals, col2); addJoin(joinType, table, condition); } void QueryBuilder::addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addValueCondition(column, op, value); } void QueryBuilder::addColumnCondition(const QString &column, Query::CompareOperator op, const QString &column2, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addColumnCondition(column, op, column2); } QSqlQuery &QueryBuilder::query() { return mQuery; } void QueryBuilder::sqliteAdaptUpdateJoin(Query::Condition &condition) { // FIXME: This does not cover all cases by far. It however can handle most // (probably all) of the update-join queries we do in Akonadi and convert them // properly into a SQLite-compatible query. Better than nothing ;-) if (!condition.mSubConditions.isEmpty()) { for (int i = condition.mSubConditions.count() - 1; i >= 0; --i) { sqliteAdaptUpdateJoin(condition.mSubConditions[i]); } return; } QString table; if (condition.mColumn.contains(QLatin1Char('.'))) { table = condition.mColumn.left(condition.mColumn.indexOf(QLatin1Char('.'))); } else { return; } if (!mJoinedTables.contains(table)) { return; } const QPair joinCondition = mJoins.value(table); QueryBuilder qb(table, Select); qb.addColumn(condition.mColumn); qb.addCondition(joinCondition.second); // Convert the subquery to string condition.mColumn.reserve(1024); condition.mColumn.resize(0); condition.mColumn += QLatin1String("( "); qb.buildQuery(&condition.mColumn); condition.mColumn += QLatin1String(" )"); } void QueryBuilder::buildQuery(QString *statement) { // we add the ON conditions of Inner Joins in a Update query here // but don't want to change the mRootCondition on each exec(). Query::Condition whereCondition = mRootCondition[WhereCondition]; switch (mType) { case Select: // Enable forward-only on all SELECT queries, since we never need to // iterate backwards. This is a memory optimization. mQuery.setForwardOnly(true); *statement += QLatin1String("SELECT "); if (mDistinct) { *statement += QLatin1String("DISTINCT "); } Q_ASSERT_X(mColumns.count() > 0, "QueryBuilder::exec()", "No columns specified"); appendJoined(statement, mColumns); *statement += QLatin1String(" FROM "); *statement += mTable; for (const QString &joinedTable : qAsConst(mJoinedTables)) { const QPair &join = mJoins.value(joinedTable); switch (join.first) { case LeftJoin: *statement += QLatin1String(" LEFT JOIN "); break; case InnerJoin: *statement += QLatin1String(" INNER JOIN "); break; } *statement += joinedTable; *statement += QLatin1String(" ON "); buildWhereCondition(statement, join.second); } break; case Insert: { *statement += QLatin1String("INSERT INTO "); *statement += mTable; *statement += QLatin1String(" ("); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { *statement += mColumnValues.at(i).first; if (i + 1 < c) { *statement += QLatin1String(", "); } } *statement += QLatin1String(") VALUES ("); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { bindValue(statement, mColumnValues.at(i).second); if (i + 1 < c) { *statement += QLatin1String(", "); } } *statement += QLatin1Char(')'); if (mDatabaseType == DbType::PostgreSQL && !mIdentificationColumn.isEmpty()) { *statement += QLatin1String(" RETURNING ") + mIdentificationColumn; } break; } case Update: { // put the ON condition into the WHERE part of the UPDATE query if (mDatabaseType != DbType::Sqlite) { for (const QString &table : qAsConst(mJoinedTables)) { const QPair< JoinType, Query::Condition > &join = mJoins.value(table); Q_ASSERT(join.first == InnerJoin); whereCondition.addCondition(join.second); } } else { // Note: this will modify the whereCondition sqliteAdaptUpdateJoin(whereCondition); } *statement += QLatin1String("UPDATE "); *statement += mTable; if (mDatabaseType == DbType::MySQL && !mJoinedTables.isEmpty()) { // for mysql we list all tables directly *statement += QLatin1String(", "); appendJoined(statement, mJoinedTables); } *statement += QLatin1String(" SET "); Q_ASSERT_X(mColumnValues.count() >= 1, "QueryBuilder::exec()", "At least one column needs to be changed"); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { const QPair &p = mColumnValues.at(i); *statement += p.first; *statement += QLatin1String(" = "); bindValue(statement, p.second); if (i + 1 < c) { *statement += QLatin1String(", "); } } if (mDatabaseType == DbType::PostgreSQL && !mJoinedTables.isEmpty()) { // PSQL have this syntax // FROM t1 JOIN t2 JOIN ... *statement += QLatin1String(" FROM "); appendJoined(statement, mJoinedTables, QLatin1String(" JOIN ")); } break; } case Delete: *statement += QLatin1String("DELETE FROM "); *statement += mTable; break; default: Q_ASSERT_X(false, "QueryBuilder::exec()", "Unknown enum value"); } if (!whereCondition.isEmpty()) { *statement += QLatin1String(" WHERE "); buildWhereCondition(statement, whereCondition); } if (!mGroupColumns.isEmpty()) { *statement += QLatin1String(" GROUP BY "); appendJoined(statement, mGroupColumns); } if (!mRootCondition[HavingCondition].isEmpty()) { *statement += QLatin1String(" HAVING "); buildWhereCondition(statement, mRootCondition[HavingCondition]); } if (!mSortColumns.isEmpty()) { Q_ASSERT_X(mType == Select, "QueryBuilder::exec()", "Order statements are only valid for SELECT queries"); *statement += QLatin1String(" ORDER BY "); for (int i = 0, c = mSortColumns.size(); i < c; ++i) { const QPair &order = mSortColumns.at(i); *statement += order.first; *statement += sortOrderToString(order.second); if (i + 1 < c) { *statement += QLatin1String(", "); } } } if (mLimit > 0) { *statement += QLatin1Literal(" LIMIT ") + QString::number(mLimit); } if (mType == Select && mForUpdate) { if (mDatabaseType == DbType::Sqlite) { // SQLite does not support SELECT ... FOR UPDATE syntax, because it does // table-level locking } else { *statement += QLatin1Literal(" FOR UPDATE"); } } } bool QueryBuilder::retryLastTransaction(bool rollback) { #ifndef QUERYBUILDER_UNITTEST mQuery = DataStore::self()->retryLastTransaction(rollback); return !mQuery.lastError().isValid(); #else Q_UNUSED(rollback); return true; #endif } bool QueryBuilder::exec() { QString statement; statement.reserve(1024); buildQuery(&statement); #ifndef QUERYBUILDER_UNITTEST if (QueryCache::contains(statement)) { mQuery = QueryCache::query(statement); } else { mQuery.clear(); if (!mQuery.prepare(statement)) { qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR while PREPARING QUERY:"; qCCritical(AKONADISERVER_LOG) << " Error code:" << mQuery.lastError().nativeErrorCode(); qCCritical(AKONADISERVER_LOG) << " DB error: " << mQuery.lastError().databaseText(); qCCritical(AKONADISERVER_LOG) << " Error text:" << mQuery.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Query:" << statement; return false; } QueryCache::insert(statement, mQuery); } //too heavy debug info but worths to have from time to time //qCDebug(AKONADISERVER_LOG) << "Executing query" << statement; bool isBatch = false; for (int i = 0; i < mBindValues.count(); ++i) { mQuery.bindValue(QLatin1Char(':') + QString::number(i), mBindValues[i]); if (!isBatch && static_cast(mBindValues[i].type()) == QMetaType::QVariantList) { isBatch = true; } //qCDebug(AKONADISERVER_LOG) << QString::fromLatin1( ":%1" ).arg( i ) << mBindValues[i]; } bool ret; if (StorageDebugger::instance()->isSQLDebuggingEnabled()) { QTime t; t.start(); if (isBatch) { ret = mQuery.execBatch(); } else { ret = mQuery.exec(); } StorageDebugger::instance()->queryExecuted(reinterpret_cast(DataStore::self()), mQuery, t.elapsed()); } else { StorageDebugger::instance()->incSequence(); if (isBatch) { ret = mQuery.execBatch(); } else { ret = mQuery.exec(); } } // Add the query to DataStore so that we can replay it in case transaction deadlocks. // The method does nothing when this query is not executed within a transaction. // We don't care whether the query was successful or not. In case of error, the caller // will rollback the transaction anyway, and all cached queries will be removed. DataStore::self()->addQueryToTransaction(statement, mBindValues, isBatch); if (!ret) { // Handle transaction deadlocks and timeouts by attempting to replay the transaction. if (mDatabaseType == DbType::PostgreSQL) { const QString dbError = mQuery.lastError().databaseText(); if (dbError.contains(QLatin1String("40P01" /* deadlock_detected */))) { - qCDebug(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; - qCDebug(AKONADISERVER_LOG) << mQuery.lastError().text(); + qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; + qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } } else if (mDatabaseType == DbType::MySQL) { const QString lastErrorStr = mQuery.lastError().nativeErrorCode(); const int error = lastErrorStr.isEmpty() ? -1 : lastErrorStr.toInt(); if (error == 1213 /* ER_LOCK_DEADLOCK */) { - qCDebug(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; - qCDebug(AKONADISERVER_LOG) << mQuery.lastError().text(); + qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; + qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } else if (error == 1205 /* ER_LOCK_WAIT_TIMEOUT */) { - qCDebug(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; - qCDebug(AKONADISERVER_LOG) << mQuery.lastError().text(); + qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; + qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } } else if (mDatabaseType == DbType::Sqlite && !DbType::isSystemSQLite(DataStore::self()->database())) { const QString lastErrorStr = mQuery.lastError().nativeErrorCode(); const int error = lastErrorStr.isEmpty() ? -1 : lastErrorStr.toInt(); if (error == 6 /* SQLITE_LOCKED */) { - qCDebug(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; - qCDebug(AKONADISERVER_LOG) << mQuery.lastError().text(); + qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; + qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(true); } else if (error == 5 /* SQLITE_BUSY */) { - qCDebug(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; - qCDebug(AKONADISERVER_LOG) << mQuery.lastError().text(); + qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; + qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(true); } } else if (mDatabaseType == DbType::Sqlite) { // We can't have a transaction deadlock in SQLite when using driver shipped // with Qt, because it does not support concurrent transactions and DataStore // serializes them through a global lock. } qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR:"; qCCritical(AKONADISERVER_LOG) << " Error code:" << mQuery.lastError().nativeErrorCode(); qCCritical(AKONADISERVER_LOG) << " DB error: " << mQuery.lastError().databaseText(); qCCritical(AKONADISERVER_LOG) << " Error text:" << mQuery.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Values:" << mQuery.boundValues(); qCCritical(AKONADISERVER_LOG) << " Query:" << statement; return false; } #else mStatement = statement; #endif return true; } void QueryBuilder::addColumns(const QStringList &cols) { mColumns << cols; } void QueryBuilder::addColumn(const QString &col) { mColumns << col; } void QueryBuilder::addColumn(const Query::Case &caseStmt) { QString query; buildCaseStatement(&query, caseStmt); mColumns.append(query); } void QueryBuilder::addAggregation(const QString &col, const QString &aggregate) { mColumns.append(aggregate + QLatin1Char('(') + col + QLatin1Char(')')); } void QueryBuilder::addAggregation(const Query::Case &caseStmt, const QString &aggregate) { QString query(aggregate + QLatin1Char('(')); buildCaseStatement(&query, caseStmt); query += QLatin1Char(')'); mColumns.append(query); } void QueryBuilder::bindValue(QString *query, const QVariant &value) { mBindValues << value; *query += QLatin1Char(':') + QString::number(mBindValues.count() - 1); } void QueryBuilder::buildWhereCondition(QString *query, const Query::Condition &cond) { if (!cond.isEmpty()) { *query += QLatin1String("( "); const QLatin1String glue = logicOperatorToString(cond.mCombineOp); const Query::Condition::List &subConditions = cond.subConditions(); for (int i = 0, c = subConditions.size(); i < c; ++i) { buildWhereCondition(query, subConditions.at(i)); if (i + 1 < c) { *query += glue; } } *query += QLatin1String(" )"); } else { *query += cond.mColumn; *query += compareOperatorToString(cond.mCompareOp); if (cond.mComparedColumn.isEmpty()) { if (cond.mComparedValue.isValid()) { if (cond.mComparedValue.canConvert(QVariant::List)) { *query += QLatin1String("( "); const QVariantList &entries = cond.mComparedValue.toList(); Q_ASSERT_X(!entries.isEmpty(), "QueryBuilder::buildWhereCondition()", "No values given for IN condition."); for (int i = 0, c = entries.size(); i < c; ++i) { bindValue(query, entries.at(i)); if (i + 1 < c) { *query += QLatin1String(", "); } } *query += QLatin1String(" )"); } else { bindValue(query, cond.mComparedValue); } } else { *query += QLatin1String("NULL"); } } else { *query += cond.mComparedColumn; } } } void QueryBuilder::buildCaseStatement(QString *query, const Query::Case &caseStmt) { *query += QLatin1String("CASE "); Q_FOREACH (const auto &whenThen, caseStmt.mWhenThen) { *query += QLatin1String("WHEN "); buildWhereCondition(query, whenThen.first); // When *query += QLatin1String(" THEN ") + whenThen.second; // then } if (!caseStmt.mElse.isEmpty()) { *query += QLatin1String(" ELSE ") + caseStmt.mElse; } *query += QLatin1String(" END"); } void QueryBuilder::setSubQueryMode(Query::LogicOperator op, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].setSubQueryMode(op); } void QueryBuilder::addCondition(const Query::Condition &condition, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addCondition(condition); } void QueryBuilder::addSortColumn(const QString &column, Query::SortOrder order) { mSortColumns << qMakePair(column, order); } void QueryBuilder::addGroupColumn(const QString &column) { Q_ASSERT(mType == Select); mGroupColumns << column; } void QueryBuilder::addGroupColumns(const QStringList &columns) { Q_ASSERT(mType == Select); mGroupColumns += columns; } void QueryBuilder::setColumnValue(const QString &column, const QVariant &value) { mColumnValues << qMakePair(column, value); } void QueryBuilder::setDistinct(bool distinct) { mDistinct = distinct; } void QueryBuilder::setLimit(int limit) { mLimit = limit; } void QueryBuilder::setIdentificationColumn(const QString &column) { mIdentificationColumn = column; } qint64 QueryBuilder::insertId() { if (mDatabaseType == DbType::PostgreSQL) { query().next(); if (mIdentificationColumn.isEmpty()) { return 0; // FIXME: Does this make sense? } return query().record().value(mIdentificationColumn).toLongLong(); } else { const QVariant v = query().lastInsertId(); if (!v.isValid()) { return -1; } bool ok; const qint64 insertId = v.toLongLong(&ok); if (!ok) { return -1; } return insertId; } return -1; } void QueryBuilder::setForUpdate(bool forUpdate) { mForUpdate = forUpdate; } diff --git a/src/server/storagejanitor.cpp b/src/server/storagejanitor.cpp index 58e6df223..8f9c298c1 100644 --- a/src/server/storagejanitor.cpp +++ b/src/server/storagejanitor.cpp @@ -1,840 +1,838 @@ /* Copyright (c) 2011 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "storagejanitor.h" #include "storage/queryhelper.h" #include "storage/transaction.h" #include "storage/datastore.h" #include "storage/selectquerybuilder.h" #include "storage/parthelper.h" #include "storage/dbconfig.h" #include "storage/collectionstatistics.h" #include "search/searchrequest.h" #include "search/searchmanager.h" #include "resourcemanager.h" #include "entities.h" #include "dbusconnectionpool.h" #include "agentmanagerinterface.h" #include "akonadiserver_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; StorageJanitor::StorageJanitor(QObject *parent) : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority, parent) , m_lostFoundCollectionId(-1) { } StorageJanitor::~StorageJanitor() { quitThread(); } void StorageJanitor::init() { AkThread::init(); QDBusConnection conn = DBusConnectionPool::threadConnection(); conn.registerService(DBus::serviceName(DBus::StorageJanitor)); conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), this, QDBusConnection::ExportScriptableSlots | QDBusConnection::ExportScriptableSignals); } void StorageJanitor::quit() { QDBusConnection conn = DBusConnectionPool::threadConnection(); conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree); conn.unregisterService(DBus::serviceName(DBus::StorageJanitor)); conn.disconnectFromBus(conn.name()); // Make sure all children are deleted within context of this thread qDeleteAll(children()); - qCDebug(AKONADISERVER_LOG) << "chainup()"; AkThread::quit(); } void StorageJanitor::check() // implementation of `akonadictl fsck` { m_lostFoundCollectionId = -1; // start with a fresh one each time inform("Looking for resources in the DB not matching a configured resource..."); findOrphanedResources(); inform("Looking for collections not belonging to a valid resource..."); findOrphanedCollections(); inform("Checking collection tree consistency..."); const Collection::List cols = Collection::retrieveAll(); std::for_each(cols.begin(), cols.end(), [this](const Collection & col) { checkPathToRoot(col); }); inform("Looking for items not belonging to a valid collection..."); findOrphanedItems(); inform("Looking for item parts not belonging to a valid item..."); findOrphanedParts(); inform("Looking for item flags not belonging to a valid item..."); findOrphanedPimItemFlags(); inform("Looking for overlapping external parts..."); findOverlappingParts(); inform("Verifying external parts..."); verifyExternalParts(); inform("Checking size treshold changes..."); checkSizeTreshold(); inform("Looking for dirty objects..."); findDirtyObjects(); inform("Looking for rid-duplicates not matching the content mime-type of the parent collection"); findRIDDuplicates(); inform("Migrating parts to new cache hierarchy..."); migrateToLevelledCacheHierarchy(); inform("Checking search index consistency..."); findOrphanSearchIndexEntries(); inform("Flushing collection statistics memory cache..."); CollectionStatistics::self()->expireCache(); /* TODO some ideas for further checks: * the collection tree is non-cyclic * content type constraints of collections are not violated * find unused flags * find unused mimetypes * check for dead entries in relation tables * check if part size matches file size */ inform("Consistency check done."); Q_EMIT done(); } qint64 StorageJanitor::lostAndFoundCollection() { if (m_lostFoundCollectionId > 0) { return m_lostFoundCollectionId; } Transaction transaction(DataStore::self(), QStringLiteral("JANITOR LOST+FOUND")); Resource lfRes = Resource::retrieveByName(QStringLiteral("akonadi_lost+found_resource")); if (!lfRes.isValid()) { lfRes.setName(QStringLiteral("akonadi_lost+found_resource")); if (!lfRes.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!"; } } Collection lfRoot; SelectQueryBuilder qb; qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id()); qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections"; return -1; } const Collection::List cols = qb.result(); if (cols.size() > 1) { qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?"; } else if (cols.size() == 1) { lfRoot = cols.first(); } else { lfRoot.setName(QStringLiteral("lost+found")); lfRoot.setResourceId(lfRes.id()); lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL")); lfRoot.setCachePolicyCacheTimeout(-1); lfRoot.setCachePolicyInherit(false); if (!lfRoot.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root."; } DataStore::self()->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8()); } Collection lfCol; lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss"))); lfCol.setResourceId(lfRes.id()); lfCol.setParentId(lfRoot.id()); if (!lfCol.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!"; } Q_FOREACH (const MimeType &mt, MimeType::retrieveAll()) { lfCol.addMimeType(mt); } DataStore::self()->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8()); transaction.commit(); m_lostFoundCollectionId = lfCol.id(); return m_lostFoundCollectionId; } void StorageJanitor::findOrphanedResources() { SelectQueryBuilder qbres; OrgFreedesktopAkonadiAgentManagerInterface iface( DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus(), this); if (!iface.isValid()) { inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control)); return; } const QStringList knownResources = iface.agentInstances(); if (knownResources.isEmpty()) { inform(QStringLiteral("ERROR: no known resources. This must be a mistake?")); return; } - qCDebug(AKONADISERVER_LOG) << "Known resources:" << knownResources; qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources)); qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource if (!qbres.exec()) { inform("Failed to query known resources, skipping test"); return; } //qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery(); const Resource::List orphanResources = qbres.result(); const int orphanResourcesSize(orphanResources.size()); if (orphanResourcesSize > 0) { QStringList resourceNames; resourceNames.reserve(orphanResourcesSize); for (const Resource &resource : orphanResources) { resourceNames.append(resource.name()); } inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize). arg(resourceNames.join(QLatin1Char(',')))); for (const QString &resourceName : qAsConst(resourceNames)) { inform(QStringLiteral("Removing resource %1").arg(resourceName)); ResourceManager::self()->removeResourceInstance(resourceName); } } } void StorageJanitor::findOrphanedCollections() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName()); qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned collections, skipping test"); return; } const Collection::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan collections.")); // TODO: attach to lost+found resource } } void StorageJanitor::checkPathToRoot(const Collection &col) { if (col.parentId() == 0) { return; } const Collection parent = col.parent(); if (!parent.isValid()) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") has no valid parent.")); // TODO fix that by attaching to a top-level lost+found folder return; } if (col.resourceId() != parent.resourceId()) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") belongs to a different resource than its parent.")); // can/should we actually fix that? } checkPathToRoot(parent); } void StorageJanitor::findOrphanedItems() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName()); qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned items, skipping test"); return; } const PimItem::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan items.")); // Attach to lost+found collection Transaction transaction(DataStore::self(), QStringLiteral("JANITOR ORPHANS")); QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update); qint64 col = lostAndFoundCollection(); if (col == -1) { return; } qb.setColumnValue(PimItem::collectionIdColumn(), col); QVector imapIds; imapIds.reserve(orphans.count()); for (const PimItem &item : qAsConst(orphans)) { imapIds.append(item.id()); } ImapSet set; set.add(imapIds); QueryHelper::setToQuery(set, PimItem::idFullColumnName(), qb); if (qb.exec() && transaction.commit()) { inform(QLatin1Literal("Moved orphan items to collection ") + QString::number(col)); } else { inform(QLatin1Literal("Error moving orphan items to collection ") + QString::number(col) + QLatin1Literal(" : ") + qb.query().lastError().text()); } } } void StorageJanitor::findOrphanedParts() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName()); qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned parts, skipping test"); return; } const Part::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan parts.")); // TODO: create lost+found items for those? delete? } } void StorageJanitor:: findOrphanedPimItemFlags() { QueryBuilder sqb(PimItemFlagRelation::tableName(), QueryBuilder::Select); sqb.addColumn(PimItemFlagRelation::leftFullColumnName()); sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName()); sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant()); if (!sqb.exec()) { inform("Failed to query orphaned item flags, skipping test"); return; } QVector imapIds; int count = 0; while (sqb.query().next()) { ++count; imapIds.append(sqb.query().value(0).toInt()); } if (count > 0) { ImapSet set; set.add(imapIds); QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); QueryHelper::setToQuery(set, PimItemFlagRelation::leftFullColumnName(), qb); if (!qb.exec()) { qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text(); return; } inform(QLatin1Literal("Found and deleted ") + QString::number(count) + QLatin1Literal(" orphan pim item flags.")); } } void StorageJanitor::findOverlappingParts() { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataColumn()); qb.addColumn(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(") as cnt")); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant()); qb.addGroupColumn(Part::dataColumn()); qb.addValueCondition(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(")"), Query::Greater, 1, QueryBuilder::HavingCondition); if (!qb.exec()) { inform("Failed to query overlapping parts, skipping test"); return; } int count = 0; while (qb.query().next()) { ++count; inform(QLatin1Literal("Found overlapping part data: ") + qb.query().value(0).toString()); // TODO: uh oh, this is bad, how do we recover from that? } if (count > 0) { inform(QLatin1Literal("Found ") + QString::number(count) + QLatin1Literal(" overlapping parts - bad.")); } } void StorageJanitor::verifyExternalParts() { QSet existingFiles; QSet usedFiles; // list all files const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data")); QDirIterator it(dataDir, QDir::Files, QDirIterator::Subdirectories); while (it.hasNext()) { existingFiles.insert(it.next()); } existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.')); existingFiles.remove(dataDir + QDir::separator() + QLatin1String("..")); inform(QLatin1Literal("Found ") + QString::number(existingFiles.size()) + QLatin1Literal(" external files.")); // list all parts from the db which claim to have an associated file QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataColumn()); qb.addColumn(Part::pimItemIdColumn()); qb.addColumn(Part::idColumn()); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant()); if (!qb.exec()) { inform("Failed to query existing parts, skipping test"); return; } while (qb.query().next()) { const auto filename = qb.query().value(0).toByteArray(); const Entity::Id pimItemId = qb.query().value(1).value(); const Entity::Id partId = qb.query().value(2).value(); QString partPath; if (!filename.isEmpty()) { partPath = ExternalPartStorage::resolveAbsolutePath(filename); } else { partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId)); } if (existingFiles.contains(partPath)) { usedFiles.insert(partPath); } else { inform(QLatin1Literal("Cleaning up missing external file: ") + partPath + QLatin1Literal(" for item: ") + QString::number(pimItemId) + QLatin1Literal(" on part: ") + QString::number(partId)); Part part; part.setId(partId); part.setPimItemId(pimItemId); part.setData(QByteArray()); part.setDatasize(0); part.setStorage(Part::Internal); part.update(); } } inform(QLatin1Literal("Found ") + QString::number(usedFiles.size()) + QLatin1Literal(" external parts.")); // see what's left and move it to lost+found const QSet unreferencedFiles = existingFiles - usedFiles; if (!unreferencedFiles.isEmpty()) { const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found")); for (const QString &file : unreferencedFiles) { inform(QLatin1Literal("Found unreferenced external file: ") + file); const QFileInfo f(file); QFile::rename(file, lfDir + QDir::separator() + f.fileName()); } inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size())); } else { inform("Found no unreferenced external files."); } } void StorageJanitor::findDirtyObjects() { SelectQueryBuilder cqb; cqb.setSubQueryMode(Query::Or); cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant()); cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString()); if (!cqb.exec()) { inform("Failed to query collections without RID, skipping test"); return; } const Collection::List ridLessCols = cqb.result(); for (const Collection &col : ridLessCols) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") has no RID.")); } inform(QLatin1Literal("Found ") + QString::number(ridLessCols.size()) + QLatin1Literal(" collections without RID.")); SelectQueryBuilder iqb1; iqb1.setSubQueryMode(Query::Or); iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant()); iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString()); if (!iqb1.exec()) { inform("Failed to query items without RID, skipping test"); return; } const PimItem::List ridLessItems = iqb1.result(); for (const PimItem &item : ridLessItems) { inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" in collection \"") + QString::number(item.collectionId()) + QLatin1Literal("\" has no RID.")); } inform(QLatin1Literal("Found ") + QString::number(ridLessItems.size()) + QLatin1Literal(" items without RID.")); SelectQueryBuilder iqb2; iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true); iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant()); iqb2.addSortColumn(PimItem::idFullColumnName()); if (!iqb2.exec()) { inform("Failed to query dirty items, skipping test"); return; } const PimItem::List dirtyItems = iqb2.result(); for (const PimItem &item : dirtyItems) { inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" has RID and is dirty.")); } inform(QLatin1Literal("Found ") + QString::number(dirtyItems.size()) + QLatin1Literal(" dirty items.")); } void StorageJanitor::findRIDDuplicates() { QueryBuilder qb(Collection::tableName(), QueryBuilder::Select); qb.addColumn(Collection::idColumn()); qb.addColumn(Collection::nameColumn()); qb.exec(); while (qb.query().next()) { const Collection::Id colId = qb.query().value(0).value(); const QString name = qb.query().value(1).toString(); inform(QStringLiteral("Checking ") + name); QueryBuilder duplicates(PimItem::tableName(), QueryBuilder::Select); duplicates.addColumn(PimItem::remoteIdColumn()); duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt")); duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant()); duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); duplicates.addGroupColumn(PimItem::remoteIdColumn()); duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition); duplicates.exec(); Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(colId); const QVector contentMimeTypes = col.mimeTypes(); QVariantList contentMimeTypesVariantList; contentMimeTypesVariantList.reserve(contentMimeTypes.count()); for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) { contentMimeTypesVariantList << mimeType.id(); } while (duplicates.query().next()) { const QString rid = duplicates.query().value(0).toString(); Query::Condition condition(Query::And); condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid); condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList); condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); QueryBuilder items(PimItem::tableName(), QueryBuilder::Select); items.addColumn(PimItem::idColumn()); items.addCondition(condition); if (!items.exec()) { inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text()); continue; } QVariantList itemsIds; while (items.query().next()) { itemsIds.push_back(items.query().value(0)); } if (itemsIds.isEmpty()) { // the mimetype filter may have dropped some entries from the // duplicates query continue; } inform(QStringLiteral("Found duplicates ") + rid); SelectQueryBuilder parts; parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds)); parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, (int) Part::External); if (parts.exec()) { const auto partsList = parts.result(); for (const auto &part : partsList) { bool exists = false; const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists); if (exists) { QFile::remove(filename); } } } items = QueryBuilder(PimItem::tableName(), QueryBuilder::Delete); items.addCondition(condition); if (!items.exec()) { inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text()); } } } } void StorageJanitor::vacuum() { const DbType::Type dbType = DbType::type(DataStore::self()->database()); if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) { inform("vacuuming database, that'll take some time and require a lot of temporary disk space..."); Q_FOREACH (const QString &table, allDatabaseTables()) { inform(QStringLiteral("optimizing table %1...").arg(table)); QString queryStr; if (dbType == DbType::MySQL) { queryStr = QLatin1Literal("OPTIMIZE TABLE ") + table; } else if (dbType == DbType::PostgreSQL) { queryStr = QLatin1Literal("VACUUM FULL ANALYZE ") + table; } else { continue; } QSqlQuery q(DataStore::self()->database()); if (!q.exec(queryStr)) { qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text(); } } inform("vacuum done"); } else { inform("Vacuum not supported for this database backend."); } Q_EMIT done(); } void StorageJanitor::checkSizeTreshold() { { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idFullColumnName()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal); qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, DbConfig::configuredDatabase()->sizeThreshold()); if (!qb.exec()) { inform("Failed to query parts larger than treshold, skipping test"); return; } QSqlQuery query = qb.query(); inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size())); while (query.next()) { Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD")); Part part = Part::retrieveById(query.value(0).toLongLong()); const QByteArray name = ExternalPartStorage::nameForPartId(part.id()); const QString partPath = ExternalPartStorage::resolveAbsolutePath(name); QFile f(partPath); if (f.exists()) { qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists"; // That however is not a critical issue, since the part is not external, // so we can safely overwrite it } if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) { qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing"; continue; } if (f.write(part.data()) != part.datasize()) { qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name; f.remove(); continue; } part.setData(name); part.setStorage(Part::External); if (!part.update() || !transaction.commit()) { qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id(); f.remove(); continue; } inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name))); } } { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idFullColumnName()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External); qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold()); if (!qb.exec()) { inform("Failed to query parts smaller than treshold, skipping test"); return; } QSqlQuery query = qb.query(); inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size())); while (query.next()) { Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2")); Part part = Part::retrieveById(query.value(0).toLongLong()); const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data()); QFile f(partPath); if (!f.exists()) { qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist"; continue; } if (!f.open(QIODevice::ReadOnly)) { qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading"; continue; } part.setStorage(Part::Internal); part.setData(f.readAll()); if (part.data().size() != part.datasize()) { qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match"; continue; } if (!part.update() || !transaction.commit()) { qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id(); continue; } f.close(); f.remove(); inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id())); } } } void StorageJanitor::migrateToLevelledCacheHierarchy() { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idColumn()); qb.addColumn(Part::dataColumn()); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); if (!qb.exec()) { inform("Failed to query external payload parts, skipping test"); return; } QSqlQuery query = qb.query(); while (query.next()) { const qint64 id = query.value(0).toLongLong(); const QByteArray data = query.value(1).toByteArray(); const QString fileName = QString::fromUtf8(data); bool oldExists = false, newExists = false; // Resolve the current path const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists); // Resolve the new path with legacy fallback disabled, so that it always // returns the new levelled-cache path, even when the old one exists const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false); if (!oldExists) { qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName; continue; } if (currentPath != newPath) { if (newExists) { qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName; continue; } QFile f(currentPath); if (!f.rename(newPath)) { qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString(); continue; } inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id)); } } } void StorageJanitor::findOrphanSearchIndexEntries() { QueryBuilder qb(Collection::tableName(), QueryBuilder::Select); qb.addSortColumn(Collection::idColumn(), Query::Ascending); qb.addColumn(Collection::idColumn()); qb.addColumn(Collection::isVirtualColumn()); if (!qb.exec()) { inform("Failed to query collections, skipping test"); return; } QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent), QStringLiteral("/"), QStringLiteral("org.freedesktop.Akonadi.Indexer"), DBusConnectionPool::threadConnection()); if (!iface.isValid()) { inform("Akonadi Indexing Agent is not running, skipping test"); return; } QSqlQuery query = qb.query(); while (query.next()) { const qint64 colId = query.value(0).toLongLong(); // Skip virtual collections, they are not indexed if (query.value(1).toBool()) { inform(QStringLiteral("Skipping virtual Collection %1").arg(colId)); continue; } inform(QStringLiteral("Checking Collection %1 search index...").arg(colId)); SearchRequest req("StorageJanitor"); req.setStoreResults(true); req.setCollections({ colId }); req.setRemoteSearch(false); req.setQuery(QStringLiteral("{ }")); // empty query to match all QStringList mts; Collection col; col.setId(colId); const auto colMts = col.mimeTypes(); if (colMts.isEmpty()) { // No mimetypes means we don't know which search store to look into, // skip it. continue; } mts.reserve(colMts.count()); for (const auto &mt : colMts) { mts << mt.name(); } req.setMimeTypes(mts); req.exec(); auto searchResults = req.results(); QueryBuilder iqb(PimItem::tableName(), QueryBuilder::Select); iqb.addColumn(PimItem::idColumn()); iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); if (!iqb.exec()) { inform(QStringLiteral("Failed to query items in collection %1").arg(colId)); continue; } QSqlQuery itemQuery = iqb.query(); while (itemQuery.next()) { searchResults.remove(itemQuery.value(0).toLongLong()); } if (!searchResults.isEmpty()) { inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count())); iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId); } } } void StorageJanitor::inform(const char *msg) { inform(QLatin1String(msg)); } void StorageJanitor::inform(const QString &msg) { qCDebug(AKONADISERVER_LOG) << msg; Q_EMIT information(msg); }