diff --git a/examples/imapresource/imapresource.cpp b/examples/imapresource/imapresource.cpp index 25fe0932..61ff2895 100644 --- a/examples/imapresource/imapresource.cpp +++ b/examples/imapresource/imapresource.cpp @@ -1,1196 +1,1197 @@ /* * Copyright (C) 2015 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "imapresource.h" #include "facade.h" #include "resourceconfig.h" #include "commands.h" #include "index.h" #include "log.h" #include "definitions.h" #include "inspection.h" #include "synchronizer.h" #include "inspector.h" #include "query.h" #include #include #include #include "facadefactory.h" #include "adaptorfactoryregistry.h" #include "imapserverproxy.h" #include "mailpreprocessor.h" #include "specialpurposepreprocessor.h" //This is the resources entity type, and not the domain type #define ENTITY_TYPE_MAIL "mail" #define ENTITY_TYPE_FOLDER "folder" Q_DECLARE_METATYPE(QSharedPointer) using namespace Imap; using namespace Sink; static qint64 sCommitInterval = 100; static qint64 uidFromMailRid(const QByteArray &remoteId) { auto ridParts = remoteId.split(':'); Q_ASSERT(ridParts.size() == 2); return ridParts.last().toLongLong(); } static QByteArray folderIdFromMailRid(const QByteArray &remoteId) { auto ridParts = remoteId.split(':'); Q_ASSERT(ridParts.size() == 2); return ridParts.first(); } static QByteArray assembleMailRid(const QByteArray &folderLocalId, qint64 imapUid) { return folderLocalId + ':' + QByteArray::number(imapUid); } static QByteArray assembleMailRid(const ApplicationDomain::Mail &mail, qint64 imapUid) { return assembleMailRid(mail.getFolder(), imapUid); } static QByteArray folderRid(const Imap::Folder &folder) { return folder.path().toUtf8(); } static QByteArray parentRid(const Imap::Folder &folder) { return folder.parentPath().toUtf8(); } static QByteArray getSpecialPurposeType(const QByteArrayList &flags) { if (Imap::flagsContain(Imap::FolderFlags::Trash, flags)) { return ApplicationDomain::SpecialPurpose::Mail::trash; } if (Imap::flagsContain(Imap::FolderFlags::Drafts, flags)) { return ApplicationDomain::SpecialPurpose::Mail::drafts; } if (Imap::flagsContain(Imap::FolderFlags::Sent, flags)) { return ApplicationDomain::SpecialPurpose::Mail::sent; } return {}; } static bool hasSpecialPurposeFlag(const QByteArrayList &flags) { return !getSpecialPurposeType(flags).isEmpty(); } class ImapSynchronizer : public Sink::Synchronizer { Q_OBJECT public: ImapSynchronizer(const ResourceContext &resourceContext) : Sink::Synchronizer(resourceContext) { } QByteArray createFolder(const Imap::Folder &f) { const auto parentFolderRid = parentRid(f); bool isToplevel = parentFolderRid.isEmpty(); SinkTraceCtx(mLogCtx) << "Creating folder: " << f.name() << parentFolderRid << f.flags; const auto remoteId = folderRid(f); Sink::ApplicationDomain::Folder folder; folder.setName(f.name()); folder.setIcon("folder"); folder.setEnabled(f.subscribed); const auto specialPurpose = [&] { if (hasSpecialPurposeFlag(f.flags)) { return getSpecialPurposeType(f.flags); } else if (SpecialPurpose::isSpecialPurposeFolderName(f.name()) && isToplevel) { return SpecialPurpose::getSpecialPurposeType(f.name()); } return QByteArray{}; }(); if (!specialPurpose.isEmpty()) { folder.setSpecialPurpose({specialPurpose}); } //Always show the inbox if (specialPurpose == ApplicationDomain::SpecialPurpose::Mail::inbox) { folder.setEnabled(true); } if (!isToplevel) { folder.setParent(syncStore().resolveRemoteId(ApplicationDomain::Folder::name, parentFolderRid)); } createOrModify(ApplicationDomain::getTypeName(), remoteId, folder); return remoteId; } static bool contains(const QVector &folderList, const QByteArray &remoteId) { for (const auto &folder : folderList) { if (folderRid(folder) == remoteId) { return true; } } return false; } void synchronizeFolders(const QVector &folderList) { SinkTraceCtx(mLogCtx) << "Found folders " << folderList.size(); scanForRemovals(ENTITY_TYPE_FOLDER, [&folderList](const QByteArray &remoteId) -> bool { return contains(folderList, remoteId); } ); for (const auto &f : folderList) { createFolder(f); } } static void setFlags(Sink::ApplicationDomain::Mail &mail, const KIMAP2::MessageFlags &flags) { mail.setUnread(!flags.contains(Imap::Flags::Seen)); mail.setImportant(flags.contains(Imap::Flags::Flagged)); } static KIMAP2::MessageFlags getFlags(const Sink::ApplicationDomain::Mail &mail) { KIMAP2::MessageFlags flags; if (!mail.getUnread()) { flags << Imap::Flags::Seen; } if (mail.getImportant()) { flags << Imap::Flags::Flagged; } return flags; } void createOrModifyMail(const QByteArray &folderRid, const QByteArray &folderLocalId, const Message &message) { auto time = QSharedPointer::create(); time->start(); SinkTraceCtx(mLogCtx) << "Importing new mail." << folderRid; const auto remoteId = assembleMailRid(folderLocalId, message.uid); Q_ASSERT(message.msg); SinkTraceCtx(mLogCtx) << "Found a mail " << remoteId << message.flags; auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); mail.setFolder(folderLocalId); mail.setMimeMessage(message.msg->encodedContent(true)); mail.setExtractedFullPayloadAvailable(message.fullPayload); setFlags(mail, message.flags); createOrModify(ENTITY_TYPE_MAIL, remoteId, mail); // const auto elapsed = time->elapsed(); // SinkTraceCtx(mLogCtx) << "Synchronized " << count << " mails in " << folderRid << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; } void synchronizeRemovals(const QByteArray &folderRid, const QSet &messages) { auto time = QSharedPointer::create(); time->start(); const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRid); if (folderLocalId.isEmpty()) { SinkWarning() << "Failed to lookup local id of: " << folderRid; return; } SinkTraceCtx(mLogCtx) << "Finding removed mail: " << folderLocalId << " remoteId: " << folderRid; int count = scanForRemovals(ENTITY_TYPE_MAIL, [&](const std::function &callback) { store().indexLookup(folderLocalId, callback); }, [&](const QByteArray &remoteId) { return messages.contains(uidFromMailRid(remoteId)); } ); const auto elapsed = time->elapsed(); SinkLog() << "Removed " << count << " mails in " << folderRid << Sink::Log::TraceTime(elapsed) << " " << elapsed/qMax(count, 1) << " [ms/mail]"; } - KAsync::Job synchronizeFolder(QSharedPointer imap, const Imap::Folder &folder, const QDate &dateFilter, bool fetchHeaderAlso = false) + KAsync::Job fetchFolderContents(QSharedPointer imap, const Imap::Folder &folder, const QDate &dateFilter, const SelectResult &selectResult) { - const auto logCtx = mLogCtx.subContext(folder.path().toUtf8()); - SinkLogCtx(logCtx) << "Synchronizing mails in folder: " << folderRid(folder); - SinkLogCtx(logCtx) << " fetching headers also: " << fetchHeaderAlso; const auto folderRemoteId = folderRid(folder); - if (folder.path().isEmpty() || folderRemoteId.isEmpty()) { - SinkWarningCtx(logCtx) << "Invalid folder " << folderRemoteId << folder.path(); - return KAsync::error("Invalid folder"); - } - - //Start by checking if UIDVALIDITY is still correct - return KAsync::start([=] { - return imap->select(folder) - .then([=](const SelectResult &selectResult) { - bool ok = false; - const auto uidvalidity = syncStore().readValue(folderRemoteId, "uidvalidity").toLongLong(&ok); - SinkLogCtx(logCtx) << "Checking UIDVALIDITY. Local" << uidvalidity << "remote " << selectResult.uidValidity; - if (ok && selectResult.uidValidity != uidvalidity) { - SinkWarningCtx(logCtx) << "UIDVALIDITY changed " << selectResult.uidValidity << uidvalidity; - syncStore().removePrefix(folderRemoteId); - } - syncStore().writeValue(folderRemoteId, "uidvalidity", QByteArray::number(selectResult.uidValidity)); - }); - }) + const auto logCtx = mLogCtx.subContext(folder.path().toUtf8()); //First we fetch flag changes for all messages. Since we don't know which messages are locally available we just get everything and only apply to what we have. - .then([=] { + return KAsync::start([=] { const auto lastSeenUid = qMax(qint64{0}, syncStore().readValue(folderRemoteId, "uidnext").toLongLong() - 1); bool ok = false; const auto changedsince = syncStore().readValue(folderRemoteId, "changedsince").toLongLong(&ok); SinkLogCtx(logCtx) << "About to update flags" << folder.path() << "changedsince: " << changedsince << "last seen uid: " << lastSeenUid; //If we have any mails so far we start off by updating any changed flags using changedsince, unless we don't have any mails at all. if (ok && lastSeenUid >= 1) { - return imap->fetchFlags(folder, KIMAP2::ImapSet(1, lastSeenUid), changedsince, [=](const Message &message) { + + SinkTrace() << "Modeseq " << folder.path() << selectResult.highestModSequence << changedsince; + if (selectResult.highestModSequence == static_cast(changedsince)) { + SinkTrace()<< folder.path() << "Changedsince didn't change, nothing to do."; + return KAsync::value(selectResult.uidNext); + } + + return imap->fetchFlags(KIMAP2::ImapSet(1, lastSeenUid), changedsince, [=](const Message &message) { const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); const auto remoteId = assembleMailRid(folderLocalId, message.uid); SinkLogCtx(logCtx) << "Updating mail flags " << remoteId << message.flags; auto mail = Sink::ApplicationDomain::Mail::create(mResourceInstanceIdentifier); setFlags(mail, message.flags); modify(ENTITY_TYPE_MAIL, remoteId, mail); }) - .then([=](const SelectResult &selectResult) { + .then([=] { SinkLogCtx(logCtx) << "Flags updated. New changedsince value: " << selectResult.highestModSequence; syncStore().writeValue(folderRemoteId, "changedsince", QByteArray::number(selectResult.highestModSequence)); return selectResult.uidNext; }); } else { //We hit this path on initial sync and simply record the current changedsince value - return imap->select(imap->mailboxFromFolder(folder)) - .then([=](const SelectResult &selectResult) { + return KAsync::start([=] { SinkLogCtx(logCtx) << "No flags to update. New changedsince value: " << selectResult.highestModSequence; syncStore().writeValue(folderRemoteId, "changedsince", QByteArray::number(selectResult.highestModSequence)); return selectResult.uidNext; }); } }) //Next we synchronize the full set that is given by the date limit. //We fetch all data for this set. //This will also pull in any new messages in subsequent runs. .then([=] (qint64 serverUidNext){ const auto lastSeenUid = syncStore().contains(folderRemoteId, "uidnext") ? qMax(qint64{0}, syncStore().readValue(folderRemoteId, "uidnext").toLongLong() - 1) : -1; auto job = [=] { if (dateFilter.isValid()) { SinkLogCtx(logCtx) << "Fetching messages since: " << dateFilter << " or uid: " << lastSeenUid; //Avoid creating a gap if we didn't fetch messages older than dateFilter, but aren't in the initial fetch either if (syncStore().contains(folderRemoteId, "uidnext")) { - return imap->fetchUidsSince(imap->mailboxFromFolder(folder), dateFilter, lastSeenUid + 1); + return imap->fetchUidsSince(dateFilter, lastSeenUid + 1); } else { - return imap->fetchUidsSince(imap->mailboxFromFolder(folder), dateFilter); + return imap->fetchUidsSince(dateFilter); } } else { SinkLogCtx(logCtx) << "Fetching messages."; - return imap->fetchUids(imap->mailboxFromFolder(folder)); + return imap->fetchUids(); } }(); return job.then([=](const QVector &uidsToFetch) { SinkTraceCtx(logCtx) << "Received result set " << uidsToFetch; SinkTraceCtx(logCtx) << "About to fetch mail" << folder.path(); //Make sure the uids are sorted in reverse order and drop everything below lastSeenUid (so we don't refetch what we already have) QVector filteredAndSorted = uidsToFetch; std::sort(filteredAndSorted.begin(), filteredAndSorted.end(), std::greater()); //Only filter the set if we have a valid lastSeenUid. Otherwise we would miss uid 1 if (lastSeenUid > 0) { const auto lowerBound = std::lower_bound(filteredAndSorted.begin(), filteredAndSorted.end(), lastSeenUid, std::greater()); if (lowerBound != filteredAndSorted.end()) { filteredAndSorted.erase(lowerBound, filteredAndSorted.end()); } } if (filteredAndSorted.isEmpty()) { SinkTraceCtx(logCtx) << "Nothing new to fetch for full set."; if (serverUidNext) { SinkLogCtx(logCtx) << "Storing the server side uidnext: " << serverUidNext << folder.path(); //If we don't receive a mail we should still record the updated uidnext value. syncStore().writeValue(folderRemoteId, "uidnext", QByteArray::number(serverUidNext)); } if (!syncStore().contains(folderRemoteId, "fullsetLowerbound")) { syncStore().writeValue(folderRemoteId, "fullsetLowerbound", QByteArray::number(serverUidNext)); } return KAsync::null(); } const qint64 lowerBoundUid = filteredAndSorted.last(); auto maxUid = QSharedPointer::create(filteredAndSorted.first()); SinkTraceCtx(logCtx) << "Uids to fetch for full set: " << filteredAndSorted; bool headersOnly = false; const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(folder, filteredAndSorted, headersOnly, [=](const Message &m) { if (*maxUid < m.uid) { *maxUid = m.uid; } createOrModifyMail(folderRemoteId, folderLocalId, m); }, [=](int progress, int total) { reportProgress(progress, total, {folderLocalId}); //commit every 100 messages if ((progress % sCommitInterval) == 0) { commit(); } }) .then([=] { SinkLogCtx(logCtx) << "Highest found uid: " << *maxUid << folder.path() << " Full set lower bound: " << lowerBoundUid; syncStore().writeValue(folderRemoteId, "uidnext", QByteArray::number(*maxUid + 1)); //Remember the lowest full message we fetched. //This is used below to fetch headers for the rest. if (!syncStore().contains(folderRemoteId, "fullsetLowerbound")) { syncStore().writeValue(folderRemoteId, "fullsetLowerbound", QByteArray::number(lowerBoundUid)); } commit(); }); }); }) //For all remaining messages we fetch the headers only //This is supposed to make all existing messages avialable with at least the headers only. //If we succeed this only needs to happen once (everything new is fetched above as full message). .then([=] { bool ok = false; const auto latestHeaderFetched = syncStore().readValue(folderRemoteId, "latestHeaderFetched").toLongLong(); const auto fullsetLowerbound = syncStore().readValue(folderRemoteId, "fullsetLowerbound").toLongLong(&ok); if (ok && latestHeaderFetched < fullsetLowerbound) { SinkLogCtx(logCtx) << "Fetching headers for all messages until " << fullsetLowerbound << ". Already available until " << latestHeaderFetched; - return imap->fetchUids(imap->mailboxFromFolder(folder)) + return imap->fetchUids() .then([=] (const QVector &uids) { //sort in reverse order and remove everything greater than fullsetLowerbound. //This gives us all emails for which we haven't fetched the full content yet. QVector toFetch = uids; std::sort(toFetch.begin(), toFetch.end(), std::greater()); if (fullsetLowerbound) { auto upperBound = std::upper_bound(toFetch.begin(), toFetch.end(), fullsetLowerbound, std::greater()); if (upperBound != toFetch.begin()) { toFetch.erase(toFetch.begin(), upperBound); } } SinkTraceCtx(logCtx) << "Uids to fetch for headers only: " << toFetch; bool headersOnly = true; const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(folder, toFetch, headersOnly, [=](const Message &m) { createOrModifyMail(folderRemoteId, folderLocalId, m); }, [=](int progress, int total) { reportProgress(progress, total, {folderLocalId}); //commit every 100 messages if ((progress % sCommitInterval) == 0) { commit(); } }); }) .then([=] { SinkLogCtx(logCtx) << "Headers fetched for folder: " << folder.path(); syncStore().writeValue(folderRemoteId, "latestHeaderFetched", QByteArray::number(fullsetLowerbound)); commit(); }); } else { SinkLogCtx(logCtx) << "No additional headers to fetch."; } return KAsync::null(); }) //Finally remove messages that are no longer existing on the server. .then([=] { //TODO do an examine with QRESYNC and remove VANISHED messages if supported instead - return imap->fetchUids(folder).then([=](const QVector &uids) { + return imap->fetchUids().then([=](const QVector &uids) { SinkTraceCtx(logCtx) << "Syncing removals: " << folder.path(); synchronizeRemovals(folderRemoteId, uids.toList().toSet()); commit(); }); }); } + KAsync::Job examine(QSharedPointer imap, const Imap::Folder &folder) + { + const auto logCtx = mLogCtx.subContext(folder.path().toUtf8()); + const auto folderRemoteId = folderRid(folder); + Q_ASSERT(!folderRemoteId.isEmpty()); + return imap->examine(folder) + .then([=](const SelectResult &selectResult) { + bool ok = false; + const auto uidvalidity = syncStore().readValue(folderRemoteId, "uidvalidity").toLongLong(&ok); + SinkLogCtx(logCtx) << "Checking UIDVALIDITY. Local" << uidvalidity << "remote " << selectResult.uidValidity; + if (ok && selectResult.uidValidity != uidvalidity) { + SinkWarningCtx(logCtx) << "UIDVALIDITY changed " << selectResult.uidValidity << uidvalidity; + syncStore().removePrefix(folderRemoteId); + } + syncStore().writeValue(folderRemoteId, "uidvalidity", QByteArray::number(selectResult.uidValidity)); + return KAsync::value(selectResult); + }); + } + + KAsync::Job synchronizeFolder(QSharedPointer imap, const Imap::Folder &folder, const QDate &dateFilter, bool countOnly) + { + const auto logCtx = mLogCtx.subContext(folder.path().toUtf8()); + SinkLogCtx(logCtx) << "Synchronizing mails in folder: " << folderRid(folder); + const auto folderRemoteId = folderRid(folder); + if (folder.path().isEmpty() || folderRemoteId.isEmpty()) { + SinkWarningCtx(logCtx) << "Invalid folder " << folderRemoteId << folder.path(); + return KAsync::error("Invalid folder"); + } + + //Start by checking if UIDVALIDITY is still correct + return KAsync::start([=] { + return examine(imap, folder) + .then([=](const SelectResult &selectResult) { + if (countOnly) { + const auto uidNext = syncStore().readValue(folderRemoteId, "uidnext").toLongLong(); + SinkTraceCtx(mLogCtx) << "Checking for new messages." << folderRemoteId << " Local uidnext: " << uidNext << " Server uidnext: " << selectResult.uidNext; + if (selectResult.uidNext > uidNext) { + const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); + emitNotification(Notification::Info, ApplicationDomain::NewContentAvailable, {}, {}, ENTITY_TYPE_FOLDER, {folderLocalId}); + } + return KAsync::null(); + } + return fetchFolderContents(imap, folder, dateFilter, selectResult); + }); + }); + } + Sink::QueryBase applyMailDefaults(const Sink::QueryBase &query) { if (mDaysToSync > 0) { auto defaultDateFilter = QDate::currentDate().addDays(0 - mDaysToSync); auto queryWithDefaults = query; if (!queryWithDefaults.hasFilter()) { queryWithDefaults.filter(ApplicationDomain::Mail::Date::name, QVariant::fromValue(defaultDateFilter)); } return queryWithDefaults; } return query; } QList getSyncRequests(const Sink::QueryBase &query) Q_DECL_OVERRIDE { QList list; if (query.type() == ApplicationDomain::getTypeName()) { auto request = Synchronizer::SyncRequest{applyMailDefaults(query)}; if (query.hasFilter(ApplicationDomain::Mail::Folder::name)) { request.applicableEntities << query.getFilter(ApplicationDomain::Mail::Folder::name).value.toByteArray(); } list << request; } else if (query.type() == ApplicationDomain::getTypeName()) { list << Synchronizer::SyncRequest{query}; + auto mailQuery = Sink::QueryBase(ApplicationDomain::getTypeName()); + //A pseudo property filter to express that we only need to know if there are new messages at all + mailQuery.filter("countOnly", {true}); + list << Synchronizer::SyncRequest{mailQuery, QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; } else { list << Synchronizer::SyncRequest{Sink::QueryBase(ApplicationDomain::getTypeName())}; //This request depends on the previous one so we flush first. list << Synchronizer::SyncRequest{applyMailDefaults(Sink::QueryBase(ApplicationDomain::getTypeName())), QByteArray{}, Synchronizer::SyncRequest::RequestFlush}; } return list; } QByteArray getFolderFromLocalId(const QByteArray &id) { auto mailRemoteId = syncStore().resolveLocalId(ApplicationDomain::getTypeName(), id); if (mailRemoteId.isEmpty()) { return {}; } return folderIdFromMailRid(mailRemoteId); } void mergeIntoQueue(const Synchronizer::SyncRequest &request, QList &queue) Q_DECL_OVERRIDE { auto isIndividualMailSync = [](const Synchronizer::SyncRequest &request) { if (request.requestType == SyncRequest::Synchronization) { const auto query = request.query; if (query.type() == ApplicationDomain::getTypeName()) { return !query.ids().isEmpty(); } } return false; }; if (isIndividualMailSync(request)) { auto newId = request.query.ids().first(); auto requestFolder = getFolderFromLocalId(newId); if (requestFolder.isEmpty()) { SinkWarningCtx(mLogCtx) << "Failed to find folder for local id. Ignoring request: " << request.query; return; } for (auto &r : queue) { if (isIndividualMailSync(r)) { auto queueFolder = getFolderFromLocalId(r.query.ids().first()); if (requestFolder == queueFolder) { //Merge r.query.filter(newId); SinkTrace() << "Merging request " << request.query; SinkTrace() << " to " << r.query; return; } } } } queue << request; } KAsync::Job login(const QSharedPointer &imap) { SinkTrace() << "Connecting to:" << mServer << mPort; SinkTrace() << "as:" << mUser; return imap->login(mUser, secret()) .addToContext(imap); } KAsync::Job> getFolderList(const QSharedPointer &imap, const Sink::QueryBase &query) { - if (query.hasFilter()) { - //If we have a folder filter fetch full payload of date-range & all headers - QVector folders; - auto folderFilter = query.getFilter(); - auto localIds = resolveFilter(folderFilter); - auto folderRemoteIds = syncStore().resolveLocalIds(ApplicationDomain::getTypeName(), localIds); - for (const auto &r : folderRemoteIds) { - Q_ASSERT(!r.isEmpty()); - folders << Folder{r}; + auto localIds = [&] { + if (query.hasFilter()) { + //If we have a folder filter fetch full payload of date-range & all headers + return resolveFilter(query.getFilter()); } - return KAsync::value(folders); - } else { - //Otherwise fetch full payload for daterange - auto folderList = QSharedPointer>::create(); - return imap->fetchFolders([folderList](const Folder &folder) { - if (!folder.noselect && folder.subscribed) { - *folderList << folder; - } - }) - .onError([](const KAsync::Error &error) { - SinkWarning() << "Folder list sync failed."; - }) - .then([folderList] { return *folderList; } ); + Sink::Query folderQuery; + folderQuery.setType(); + folderQuery.filter(true); + //TODO filter noselect folders? + return resolveQuery(folderQuery); + }(); + + QVector folders; + auto folderRemoteIds = syncStore().resolveLocalIds(ApplicationDomain::getTypeName(), localIds); + for (const auto &r : folderRemoteIds) { + Q_ASSERT(!r.isEmpty()); + folders << Folder{r}; } + return KAsync::value(folders); } KAsync::Error getError(const KAsync::Error &error) { if (error) { switch(error.errorCode) { case Imap::CouldNotConnectError: return {ApplicationDomain::ConnectionError, error.errorMessage}; case Imap::SslHandshakeError: case Imap::LoginFailed: return {ApplicationDomain::LoginError, error.errorMessage}; case Imap::HostNotFoundError: return {ApplicationDomain::NoServerError, error.errorMessage}; case Imap::ConnectionLost: return {ApplicationDomain::ConnectionLostError, error.errorMessage}; case Imap::MissingCredentialsError: return {ApplicationDomain::MissingCredentialsError, error.errorMessage}; default: return {ApplicationDomain::UnknownError, error.errorMessage}; } } return {}; } KAsync::Job synchronizeWithSource(const Sink::QueryBase &query) Q_DECL_OVERRIDE { if (!QUrl{mServer}.isValid()) { return KAsync::error(ApplicationDomain::ConfigurationError, "Invalid server url: " + mServer); } auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode, &mSessionCache); if (query.type() == ApplicationDomain::getTypeName()) { return login(imap) .then([=] { auto folderList = QSharedPointer>::create(); - return imap->fetchFolders([folderList](const Folder &folder) { + return imap->fetchFolders([folderList](const Folder &folder) { *folderList << folder; }) .then([=]() { synchronizeFolders(*folderList); - return *folderList; - }) - //The rest is only to check for new messages. - .each([=](const Imap::Folder &folder) { - if (!folder.noselect && folder.subscribed) { - return imap->examine(folder) - .then([=](const SelectResult &result) { - const auto folderRemoteId = folderRid(folder); - const auto uidNext = syncStore().readValue(folderRemoteId, "uidnext").toLongLong(); - SinkTraceCtx(mLogCtx) << "Checking for new messages." << folderRemoteId << " Local uidnext: " << uidNext << " Server uidnext: " << result.uidNext; - if (result.uidNext > uidNext) { - const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); - emitNotification(Notification::Info, ApplicationDomain::NewContentAvailable, {}, {}, ENTITY_TYPE_FOLDER, {folderLocalId}); - } - }).then([=] (const KAsync::Error &error) { - if (error) { - SinkWarningCtx(mLogCtx) << "Examine failed: " << error; - if (error.errorCode == Imap::CommandFailed) { - //Ignore the error because we don't want to fail the synchronization for all folders - return KAsync::null(); - } - return KAsync::error(error); - } - return KAsync::null(); - }); - } return KAsync::null(); }); }) .then([=] (const KAsync::Error &error) { return imap->logout() .then(KAsync::error(getError(error))); }); } else if (query.type() == ApplicationDomain::getTypeName()) { //TODO //if we have a folder filter: //* execute the folder query and resolve the results to the remote identifier //* query only those folders //if we have a date filter: //* apply the date filter to the fetch //if we have no folder filter: //* fetch list of folders from server directly and sync (because we have no guarantee that the folder sync was already processed by the pipeline). return login(imap) .then([=] { if (!query.ids().isEmpty()) { //If we have mail id's simply fetch the full payload of those mails QVector toFetch; auto mailRemoteIds = syncStore().resolveLocalIds(ApplicationDomain::getTypeName(), query.ids()); QByteArray folderRemoteId; for (const auto &r : mailRemoteIds) { const auto folderLocalId = folderIdFromMailRid(r); auto f = syncStore().resolveLocalId(ApplicationDomain::getTypeName(), folderLocalId); if (folderRemoteId.isEmpty()) { folderRemoteId = f; } else { if (folderRemoteId != f) { SinkWarningCtx(mLogCtx) << "Not all messages come from the same folder " << r << folderRemoteId << ". Skipping message."; continue; } } toFetch << uidFromMailRid(r); } SinkLog() << "Fetching messages: " << toFetch << folderRemoteId; bool headersOnly = false; const auto folderLocalId = syncStore().resolveRemoteId(ENTITY_TYPE_FOLDER, folderRemoteId); return imap->fetchMessages(Folder{folderRemoteId}, toFetch, headersOnly, [=](const Message &m) { createOrModifyMail(folderRemoteId, folderLocalId, m); }, [=](int progress, int total) { reportProgress(progress, total, {folderLocalId}); //commit every 100 messages if ((progress % sCommitInterval) == 0) { commit(); } }); } else { - //Otherwise we sync the folder(s) - bool syncHeaders = query.hasFilter(); - const QDate dateFilter = [&] { auto filter = query.getFilter(); if (filter.value.canConvert()) { SinkLog() << " with date-range " << filter.value.value(); return filter.value.value(); } return QDate{}; }(); - //FIXME If we were able to to flush in between we could just query the local store for the folder list. return getFolderList(imap, query) .then([=](const QVector &folders) { auto job = KAsync::null(); for (const auto &folder : folders) { job = job.then([=] { if (aborting()) { return KAsync::null(); } - return synchronizeFolder(imap, folder, dateFilter, syncHeaders) + return synchronizeFolder(imap, folder, dateFilter, query.hasFilter("countOnly")) .then([=](const KAsync::Error &error) { if (error) { if (error.errorCode == Imap::CommandFailed) { SinkWarning() << "Continuing after protocol error: " << folder.path() << "Error: " << error; //Ignore protocol-level errors and continue return KAsync::null(); } SinkWarning() << "Aborting on error: " << folder.path() << "Error: " << error; //Abort otherwise, e.g. if we disconnected return KAsync::error(error); } return KAsync::null(); }); }); } return job; }); } }) .then([=] (const KAsync::Error &error) { return imap->logout() .then(KAsync::error(getError(error))); }); } return KAsync::error("Nothing to do"); } static QByteArray ensureCRLF(const QByteArray &data) { auto index = data.indexOf('\n'); if (index > 0 && data.at(index - 1) == '\r') { //First line is LF-only terminated //Convert back and forth in case there's a mix. We don't want to expand CRLF into CRCRLF. return KMime::LFtoCRLF(KMime::CRLFtoLF(data)); } else { return data; } } static bool validateContent(const QByteArray &data) { if (data.isEmpty()) { SinkError() << "No data available."; return false; } if (data.contains('\0')) { SinkError() << "Data contains NUL, this will fail with IMAP."; return false; } return true; } KAsync::Job replay(const ApplicationDomain::Mail &mail, Sink::Operation operation, const QByteArray &oldRemoteId, const QList &changedProperties) Q_DECL_OVERRIDE { if (operation != Sink::Operation_Creation) { if(oldRemoteId.isEmpty()) { SinkWarning() << "Tried to replay modification without old remoteId."; // Since we can't recover from the situation we just skip over the revision. // This can for instance happen if creation failed, and we then process a removal or modification. return KAsync::null(); } } auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode, &mSessionCache); auto login = imap->login(mUser, secret()); KAsync::Job job = KAsync::null(); if (operation == Sink::Operation_Creation) { const QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); const auto content = ensureCRLF(mail.getMimeMessage()); if (!validateContent(content)) { SinkError() << "Validation failed during creation replay " << mail.identifier() << "\n Content:" << content; //We can't recover from this other than deleting the mail, so we skip it. return KAsync::null(); } const auto flags = getFlags(mail); const QDateTime internalDate = mail.getDate(); job = login.then(imap->append(mailbox, content, flags, internalDate)) .addToContext(imap) .then([mail](qint64 uid) { const auto remoteId = assembleMailRid(mail, uid); SinkTrace() << "Finished creating a new mail: " << remoteId; return remoteId; }); } else if (operation == Sink::Operation_Removal) { const auto folderId = folderIdFromMailRid(oldRemoteId); const QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, folderId); const auto uid = uidFromMailRid(oldRemoteId); SinkTrace() << "Removing a mail: " << oldRemoteId << "in the mailbox: " << mailbox; KIMAP2::ImapSet set; set.add(uid); job = login.then(imap->remove(mailbox, set)) .then([imap, oldRemoteId] { SinkTrace() << "Finished removing a mail: " << oldRemoteId; return QByteArray(); }); } else if (operation == Sink::Operation_Modification) { const QString mailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); const auto uid = uidFromMailRid(oldRemoteId); SinkTrace() << "Modifying a mail: " << oldRemoteId << " in the mailbox: " << mailbox << changedProperties; auto flags = getFlags(mail); const bool messageMoved = changedProperties.contains(ApplicationDomain::Mail::Folder::name); const bool messageChanged = changedProperties.contains(ApplicationDomain::Mail::MimeMessage::name); if (messageChanged || messageMoved) { const auto folderId = folderIdFromMailRid(oldRemoteId); const QString oldMailbox = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, folderId); const auto content = ensureCRLF(mail.getMimeMessage()); if (!validateContent(content)) { SinkError() << "Validation failed during modification replay " << mail.identifier() << "\n Content:" << content; //We can't recover from this other than deleting the mail, so we skip it. return KAsync::null(); } const QDateTime internalDate = mail.getDate(); SinkTrace() << "Replacing message. Old mailbox: " << oldMailbox << "New mailbox: " << mailbox << "Flags: " << flags << "Content: " << content; KIMAP2::ImapSet set; set.add(uid); job = login.then(imap->append(mailbox, content, flags, internalDate)) .addToContext(imap) .then([=](qint64 uid) { const auto remoteId = assembleMailRid(mail, uid); SinkTrace() << "Finished creating a modified mail: " << remoteId; return imap->remove(oldMailbox, set).then(KAsync::value(remoteId)); }); } else { SinkTrace() << "Updating flags only."; KIMAP2::ImapSet set; set.add(uid); job = login.then(imap->select(mailbox)) .addToContext(imap) .then(imap->storeFlags(set, flags)) .then([=] { SinkTrace() << "Finished modifying mail"; return oldRemoteId; }); } } return job .then([=] (const KAsync::Error &error, const QByteArray &remoteId) { if (error) { SinkWarning() << "Error during changereplay: " << error.errorMessage; return imap->logout() .then(KAsync::error(getError(error))); } return imap->logout() .then(KAsync::value(remoteId)); }); } KAsync::Job replay(const ApplicationDomain::Folder &folder, Sink::Operation operation, const QByteArray &oldRemoteId, const QList &changedProperties) Q_DECL_OVERRIDE { if (operation != Sink::Operation_Creation) { if(oldRemoteId.isEmpty()) { Q_ASSERT(false); return KAsync::error("Tried to replay modification without old remoteId."); } } auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode, &mSessionCache); auto login = imap->login(mUser, secret()); if (operation == Sink::Operation_Creation) { QString parentFolder; if (!folder.getParent().isEmpty()) { parentFolder = syncStore().resolveLocalId(ENTITY_TYPE_FOLDER, folder.getParent()); } SinkTraceCtx(mLogCtx) << "Creating a new folder: " << parentFolder << folder.getName(); auto rid = QSharedPointer::create(); auto createFolder = login.then(imap->createSubfolder(parentFolder, folder.getName())) .then([this, imap, rid](const QString &createdFolder) { SinkTraceCtx(mLogCtx) << "Finished creating a new folder: " << createdFolder; *rid = createdFolder.toUtf8(); }); if (folder.getSpecialPurpose().isEmpty()) { return createFolder .then([rid](){ return *rid; }); } else { //We try to merge special purpose folders first auto specialPurposeFolders = QSharedPointer>::create(); auto mergeJob = imap->login(mUser, secret()) .then(imap->fetchFolders([=](const Imap::Folder &folder) { if (SpecialPurpose::isSpecialPurposeFolderName(folder.name())) { specialPurposeFolders->insert(SpecialPurpose::getSpecialPurposeType(folder.name()), folder.path()); }; })) .then([this, specialPurposeFolders, folder, imap, parentFolder, rid]() -> KAsync::Job { for (const auto &purpose : folder.getSpecialPurpose()) { if (specialPurposeFolders->contains(purpose)) { auto f = specialPurposeFolders->value(purpose); SinkTraceCtx(mLogCtx) << "Merging specialpurpose folder with: " << f << " with purpose: " << purpose; *rid = f.toUtf8(); return KAsync::null(); } } SinkTraceCtx(mLogCtx) << "No match found for merging, creating a new folder"; return imap->createSubfolder(parentFolder, folder.getName()) .then([this, imap, rid](const QString &createdFolder) { SinkTraceCtx(mLogCtx) << "Finished creating a new folder: " << createdFolder; *rid = createdFolder.toUtf8(); }); }) .then([rid](){ return *rid; }); return mergeJob; } } else if (operation == Sink::Operation_Removal) { SinkTraceCtx(mLogCtx) << "Removing a folder: " << oldRemoteId; return login.then(imap->remove(oldRemoteId)) .then([this, oldRemoteId, imap] { SinkTraceCtx(mLogCtx) << "Finished removing a folder: " << oldRemoteId; return QByteArray(); }); } else if (operation == Sink::Operation_Modification) { SinkTraceCtx(mLogCtx) << "Renaming a folder: " << oldRemoteId << folder.getName(); auto rid = QSharedPointer::create(); return login.then(imap->renameSubfolder(oldRemoteId, folder.getName())) .then([this, imap, rid](const QString &createdFolder) { SinkTraceCtx(mLogCtx) << "Finished renaming a folder: " << createdFolder; *rid = createdFolder.toUtf8(); }) .then([rid] { return *rid; }); } return KAsync::null(); } public: QString mServer; int mPort; Imap::EncryptionMode mEncryptionMode = Imap::NoEncryption; QString mUser; int mDaysToSync = 0; QByteArray mResourceInstanceIdentifier; Imap::SessionCache mSessionCache; }; class ImapInspector : public Sink::Inspector { public: ImapInspector(const Sink::ResourceContext &resourceContext) : Sink::Inspector(resourceContext) { } protected: KAsync::Job inspect(int inspectionType, const QByteArray &inspectionId, const QByteArray &domainType, const QByteArray &entityId, const QByteArray &property, const QVariant &expectedValue) Q_DECL_OVERRIDE { auto synchronizationStore = QSharedPointer::create(Sink::storageLocation(), mResourceContext.instanceId() + ".synchronization", Sink::Storage::DataStore::ReadOnly); auto synchronizationTransaction = synchronizationStore->createTransaction(Sink::Storage::DataStore::ReadOnly); auto mainStore = QSharedPointer::create(Sink::storageLocation(), mResourceContext.instanceId(), Sink::Storage::DataStore::ReadOnly); auto transaction = mainStore->createTransaction(Sink::Storage::DataStore::ReadOnly); Sink::Storage::EntityStore entityStore(mResourceContext, {"imapresource"}); auto syncStore = QSharedPointer::create(synchronizationTransaction); SinkTrace() << "Inspecting " << inspectionType << domainType << entityId << property << expectedValue; if (domainType == ENTITY_TYPE_MAIL) { const auto mail = entityStore.readLatest(entityId); const auto folder = entityStore.readLatest(mail.getFolder()); const auto folderRemoteId = syncStore->resolveLocalId(ENTITY_TYPE_FOLDER, mail.getFolder()); const auto mailRemoteId = syncStore->resolveLocalId(ENTITY_TYPE_MAIL, mail.identifier()); if (mailRemoteId.isEmpty() || folderRemoteId.isEmpty()) { //There is no remote id to find if we expect the message to not exist if (inspectionType == Sink::ResourceControl::Inspection::ExistenceInspectionType && !expectedValue.toBool()) { return KAsync::null(); } SinkWarning() << "Missing remote id for folder or mail. " << mailRemoteId << folderRemoteId; return KAsync::error(); } const auto uid = uidFromMailRid(mailRemoteId); SinkTrace() << "Mail remote id: " << folderRemoteId << mailRemoteId << mail.identifier() << folder.identifier(); KIMAP2::ImapSet set; set.add(uid); if (set.isEmpty()) { return KAsync::error(1, "Couldn't determine uid of mail."); } KIMAP2::FetchJob::FetchScope scope; scope.mode = KIMAP2::FetchJob::FetchScope::Full; auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode); auto messageByUid = QSharedPointer>::create(); SinkTrace() << "Connecting to:" << mServer << mPort; SinkTrace() << "as:" << mUser; auto inspectionJob = imap->login(mUser, secret()) .then(imap->select(folderRemoteId)) .then([](Imap::SelectResult){}) .then(imap->fetch(set, scope, [imap, messageByUid](const Imap::Message &message) { //We avoid parsing normally, so we have to do it explicitly here if (message.msg) { message.msg->parse(); } messageByUid->insert(message.uid, message); })); if (inspectionType == Sink::ResourceControl::Inspection::PropertyInspectionType) { if (property == "unread") { return inspectionJob.then([=] { auto msg = messageByUid->value(uid); if (expectedValue.toBool() && msg.flags.contains(Imap::Flags::Seen)) { return KAsync::error(1, "Expected unread but couldn't find it."); } if (!expectedValue.toBool() && !msg.flags.contains(Imap::Flags::Seen)) { return KAsync::error(1, "Expected read but couldn't find it."); } return KAsync::null(); }); } if (property == "subject") { return inspectionJob.then([=] { auto msg = messageByUid->value(uid); if (msg.msg->subject(true)->asUnicodeString() != expectedValue.toString()) { return KAsync::error(1, "Subject not as expected: " + msg.msg->subject(true)->asUnicodeString()); } return KAsync::null(); }); } } if (inspectionType == Sink::ResourceControl::Inspection::ExistenceInspectionType) { return inspectionJob.then([=] { if (!messageByUid->contains(uid)) { SinkWarning() << "Existing messages are: " << messageByUid->keys(); SinkWarning() << "We're looking for: " << uid; return KAsync::error(1, "Couldn't find message: " + mailRemoteId); } return KAsync::null(); }); } } if (domainType == ENTITY_TYPE_FOLDER) { const auto remoteId = syncStore->resolveLocalId(ENTITY_TYPE_FOLDER, entityId); const auto folder = entityStore.readLatest(entityId); if (inspectionType == Sink::ResourceControl::Inspection::CacheIntegrityInspectionType) { SinkLog() << "Inspecting cache integrity" << remoteId; int expectedCount = 0; Index index("mail.index.folder", transaction); index.lookup(entityId, [&](const QByteArray &sinkId) { expectedCount++; }, [&](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); auto set = KIMAP2::ImapSet::fromImapSequenceSet("1:*"); KIMAP2::FetchJob::FetchScope scope; scope.mode = KIMAP2::FetchJob::FetchScope::Headers; auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode); auto messageByUid = QSharedPointer>::create(); return imap->login(mUser, secret()) .then(imap->select(remoteId)) .then(imap->fetch(set, scope, [=](const Imap::Message message) { messageByUid->insert(message.uid, message); })) .then([imap, messageByUid, expectedCount] { if (messageByUid->size() != expectedCount) { return KAsync::error(1, QString("Wrong number of messages on the server; found %1 instead of %2.").arg(messageByUid->size()).arg(expectedCount)); } return KAsync::null(); }); } if (inspectionType == Sink::ResourceControl::Inspection::ExistenceInspectionType) { auto folderByPath = QSharedPointer>::create(); auto folderByName = QSharedPointer>::create(); auto imap = QSharedPointer::create(mServer, mPort, mEncryptionMode); auto inspectionJob = imap->login(mUser, secret()) .then(imap->fetchFolders([=](const Imap::Folder &f) { *folderByPath << f.path(); *folderByName << f.name(); })) .then([folderByName, folderByPath, folder, remoteId, imap] { if (!folderByName->contains(folder.getName())) { SinkWarning() << "Existing folders are: " << *folderByPath; SinkWarning() << "We're looking for: " << folder.getName(); return KAsync::error(1, "Wrong folder name: " + remoteId); } return KAsync::null(); }); return inspectionJob; } } return KAsync::null(); } public: QString mServer; int mPort; Imap::EncryptionMode mEncryptionMode = Imap::NoEncryption; QString mUser; }; class FolderCleanupPreprocessor : public Sink::Preprocessor { public: void deletedEntity(const ApplicationDomain::ApplicationDomainType &oldEntity) override { //Remove all mails of a folder when removing the folder. const auto revision = entityStore().maxRevision(); entityStore().indexLookup(oldEntity.identifier(), [&] (const QByteArray &identifier) { deleteEntity(ApplicationDomain::ApplicationDomainType{{}, identifier, revision, {}}, ApplicationDomain::getTypeName(), false); }); } }; ImapResource::ImapResource(const ResourceContext &resourceContext) : Sink::GenericResource(resourceContext) { auto config = ResourceConfig::getConfiguration(resourceContext.instanceId()); auto server = config.value("server").toString(); auto port = config.value("port").toInt(); auto user = config.value("username").toString(); auto daysToSync = config.value("daysToSync", 14).toInt(); auto starttls = config.value("starttls", false).toBool(); auto encryption = Imap::NoEncryption; if (server.startsWith("imaps")) { encryption = Imap::Tls; } if (starttls) { encryption = Imap::Starttls; } if (server.startsWith("imap")) { server.remove("imap://"); server.remove("imaps://"); } if (server.contains(':')) { auto list = server.split(':'); server = list.at(0); port = list.at(1).toInt(); } //Backwards compatibilty //For kolabnow we assumed that port 143 means starttls if (encryption == Imap::Tls && port == 143) { encryption = Imap::Starttls; } if (!QSslSocket::supportsSsl()) { SinkWarning() << "Qt doesn't support ssl. This is likely a distribution/packaging problem."; //On windows this means that the required ssl dll's are missing SinkWarning() << "Ssl Library Build Version Number: " << QSslSocket::sslLibraryBuildVersionString(); SinkWarning() << "Ssl Library Runtime Version Number: " << QSslSocket::sslLibraryVersionString(); } else { SinkTrace() << "Ssl support available"; SinkTrace() << "Ssl Library Build Version Number: " << QSslSocket::sslLibraryBuildVersionString(); SinkTrace() << "Ssl Library Runtime Version Number: " << QSslSocket::sslLibraryVersionString(); } auto synchronizer = QSharedPointer::create(resourceContext); synchronizer->mServer = server; synchronizer->mPort = port; synchronizer->mEncryptionMode = encryption; synchronizer->mUser = user; synchronizer->mDaysToSync = daysToSync; setupSynchronizer(synchronizer); auto inspector = QSharedPointer::create(resourceContext); inspector->mServer = server; inspector->mPort = port; inspector->mEncryptionMode = encryption; inspector->mUser = user; setupInspector(inspector); setupPreprocessors(ENTITY_TYPE_MAIL, {new SpecialPurposeProcessor, new MailPropertyExtractor}); setupPreprocessors(ENTITY_TYPE_FOLDER, {new FolderCleanupPreprocessor}); } ImapResourceFactory::ImapResourceFactory(QObject *parent) : Sink::ResourceFactory(parent, {Sink::ApplicationDomain::ResourceCapabilities::Mail::mail, Sink::ApplicationDomain::ResourceCapabilities::Mail::folder, Sink::ApplicationDomain::ResourceCapabilities::Mail::storage, Sink::ApplicationDomain::ResourceCapabilities::Mail::drafts, Sink::ApplicationDomain::ResourceCapabilities::Mail::folderhierarchy, Sink::ApplicationDomain::ResourceCapabilities::Mail::trash, Sink::ApplicationDomain::ResourceCapabilities::Mail::sent} ) { } Sink::Resource *ImapResourceFactory::createResource(const ResourceContext &context) { return new ImapResource(context); } void ImapResourceFactory::registerFacades(const QByteArray &name, Sink::FacadeFactory &factory) { factory.registerFacade>(name); factory.registerFacade>(name); } void ImapResourceFactory::registerAdaptorFactories(const QByteArray &name, Sink::AdaptorFactoryRegistry ®istry) { registry.registerFactory>(name); registry.registerFactory>(name); } void ImapResourceFactory::removeDataFromDisk(const QByteArray &instanceIdentifier) { ImapResource::removeFromDisk(instanceIdentifier); } #include "imapresource.moc" diff --git a/examples/imapresource/imapserverproxy.cpp b/examples/imapresource/imapserverproxy.cpp index 9a30eeb1..c75f9a4a 100644 --- a/examples/imapresource/imapserverproxy.cpp +++ b/examples/imapresource/imapserverproxy.cpp @@ -1,739 +1,720 @@ /* * Copyright (C) 2015 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #include "imapserverproxy.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "log.h" #include "test.h" using namespace Imap; const char* Imap::Flags::Seen = "\\Seen"; const char* Imap::Flags::Deleted = "\\Deleted"; const char* Imap::Flags::Answered = "\\Answered"; const char* Imap::Flags::Flagged = "\\Flagged"; const char* Imap::FolderFlags::Noselect = "\\Noselect"; const char* Imap::FolderFlags::Noinferiors = "\\Noinferiors"; const char* Imap::FolderFlags::Marked = "\\Marked"; const char* Imap::FolderFlags::Unmarked = "\\Unmarked"; const char* Imap::FolderFlags::Subscribed = "\\Subscribed"; //Special use const char* Imap::FolderFlags::Sent = "\\Sent"; const char* Imap::FolderFlags::Trash = "\\Trash"; const char* Imap::FolderFlags::Archive = "\\Archive"; const char* Imap::FolderFlags::Junk = "\\Junk"; const char* Imap::FolderFlags::Flagged = "\\Flagged"; const char* Imap::FolderFlags::Drafts = "\\Drafts"; const char* Imap::Capabilities::Namespace = "NAMESPACE"; const char* Imap::Capabilities::Uidplus = "UIDPLUS"; const char* Imap::Capabilities::Condstore = "CONDSTORE"; static int translateImapError(KJob *job) { switch (job->error()) { case KIMAP2::HostNotFound: return Imap::HostNotFoundError; case KIMAP2::CouldNotConnect: return Imap::CouldNotConnectError; case KIMAP2::SslHandshakeFailed: return Imap::SslHandshakeError; case KIMAP2::ConnectionLost: return Imap::ConnectionLost; case KIMAP2::LoginFailed: return Imap::LoginFailed; case KIMAP2::CommandFailed: return Imap::CommandFailed; } return Imap::UnknownError; } template static KAsync::Job runJob(KJob *job, const std::function &f) { return KAsync::start([job, f](KAsync::Future &future) { QObject::connect(job, &KJob::result, [&future, f](KJob *job) { SinkTrace() << "Job done: " << job->metaObject()->className(); if (job->error()) { SinkWarning() << "Job failed: " << job->errorString() << job->metaObject()->className() << job->error(); auto proxyError = translateImapError(job); future.setError(proxyError, job->errorString()); } else { future.setValue(f(job)); future.setFinished(); } }); SinkTrace() << "Starting job: " << job->metaObject()->className(); job->start(); }); } static KAsync::Job runJob(KJob *job) { return KAsync::start([job](KAsync::Future &future) { QObject::connect(job, &KJob::result, [&future](KJob *job) { SinkTrace() << "Job done: " << job->metaObject()->className(); if (job->error()) { SinkWarning() << "Job failed: " << job->errorString() << job->metaObject()->className() << job->error(); auto proxyError = translateImapError(job); future.setError(proxyError, job->errorString()); } else { future.setFinished(); } }); SinkTrace() << "Starting job: " << job->metaObject()->className(); job->start(); }); } static int socketTimeout() { if (Sink::Test::testModeEnabled()) { return 5; } return 40; } KIMAP2::Session *createNewSession(const QString &serverUrl, int port) { auto newSession = new KIMAP2::Session(serverUrl, qint16(port)); newSession->setTimeout(socketTimeout()); QObject::connect(newSession, &KIMAP2::Session::sslErrors, [=](const QList &errors) { SinkWarning() << "Received SSL errors:"; for (const auto &e : errors) { SinkWarning() << " " << e.error() << ":" << e.errorString() << "Certificate: " << e.certificate().toText(); } newSession->ignoreErrors(errors); }); return newSession; } ImapServerProxy::ImapServerProxy(const QString &serverUrl, int port, EncryptionMode encryptionMode, SessionCache *sessionCache) : mSessionCache(sessionCache), mSession(nullptr), mEncryptionMode(encryptionMode) { if (!mSessionCache || mSessionCache->isEmpty()) { mSession = createNewSession(serverUrl, port); } } QDebug operator<<(QDebug debug, const KIMAP2::MailBoxDescriptor &c) { QDebugStateSaver saver(debug); debug.nospace() << c.name; return debug; } KAsync::Job ImapServerProxy::login(const QString &username, const QString &password) { if (password.isEmpty()) { return KAsync::error(Imap::MissingCredentialsError); } if (mSessionCache) { auto session = mSessionCache->getSession(); if (session.isValid()) { mSession = session.mSession; mCapabilities = session.mCapabilities; mNamespaces = session.mNamespaces; } } Q_ASSERT(mSession); if (mSession->state() == KIMAP2::Session::Authenticated || mSession->state() == KIMAP2::Session::Selected) { //If we blindly reuse the socket it may very well be stale and then we have to wait for it to time out. //A hostlookup should be fast (a couple of milliseconds once cached), and can typcially tell us quickly //if the host is no longer available. auto info = QHostInfo::fromName(mSession->hostName()); if (info.error()) { SinkLog() << "Failed host lookup, closing the socket" << info.errorString(); mSession->close(); return KAsync::error(Imap::HostNotFoundError); } else { //Prevent the socket from timing out right away, right here (otherwise it just might time out right before we were able to start the job) mSession->setTimeout(socketTimeout()); SinkLog() << "Reusing existing session."; return KAsync::null(); } } auto loginJob = new KIMAP2::LoginJob(mSession); loginJob->setUserName(username); loginJob->setPassword(password); if (mEncryptionMode == Starttls) { loginJob->setEncryptionMode(QSsl::TlsV1_0OrLater, true); } else if (mEncryptionMode == Tls) { loginJob->setEncryptionMode(QSsl::AnyProtocol, false); } loginJob->setAuthenticationMode(KIMAP2::LoginJob::Plain); auto capabilitiesJob = new KIMAP2::CapabilitiesJob(mSession); QObject::connect(capabilitiesJob, &KIMAP2::CapabilitiesJob::capabilitiesReceived, &mGuard, [this](const QStringList &capabilities) { mCapabilities = capabilities; }); auto namespaceJob = new KIMAP2::NamespaceJob(mSession); return runJob(loginJob).then(runJob(capabilitiesJob)).then([this](){ SinkTrace() << "Supported capabilities: " << mCapabilities; QStringList requiredExtensions = QStringList() << Capabilities::Uidplus << Capabilities::Namespace; for (const auto &requiredExtension : requiredExtensions) { if (!mCapabilities.contains(requiredExtension)) { SinkWarning() << "Server doesn't support required capability: " << requiredExtension; //TODO fail the job } } }).then(runJob(namespaceJob)).then([this, namespaceJob] { mNamespaces.personal = namespaceJob->personalNamespaces(); mNamespaces.shared = namespaceJob->sharedNamespaces(); mNamespaces.user = namespaceJob->userNamespaces(); // SinkTrace() << "Found personal namespaces: " << mNamespaces.personal; // SinkTrace() << "Found shared namespaces: " << mNamespaces.shared; // SinkTrace() << "Found user namespaces: " << mNamespaces.user; }); } KAsync::Job ImapServerProxy::logout() { if (mSessionCache) { auto session = CachedSession{mSession, mCapabilities, mNamespaces}; if (session.isConnected()) { mSessionCache->recycleSession(session); return KAsync::null(); } } if (mSession->state() == KIMAP2::Session::State::Authenticated || mSession->state() == KIMAP2::Session::State::Selected) { return runJob(new KIMAP2::LogoutJob(mSession)); } else { return KAsync::null(); } } bool ImapServerProxy::isGmail() const { //Magic capability that only gmail has return mCapabilities.contains("X-GM-EXT-1"); } KAsync::Job ImapServerProxy::select(const QString &mailbox) { auto select = new KIMAP2::SelectJob(mSession); select->setMailBox(mailbox); select->setCondstoreEnabled(mCapabilities.contains(Capabilities::Condstore)); return runJob(select, [select](KJob* job) -> SelectResult { return {select->uidValidity(), select->nextUid(), select->highestModSequence()}; }).then([=] (const KAsync::Error &error, const SelectResult &result) { if (error) { SinkWarning() << "Select failed: " << mailbox; return KAsync::error(error); } return KAsync::value(result); }); } KAsync::Job ImapServerProxy::select(const Folder &folder) { return select(mailboxFromFolder(folder)); } KAsync::Job ImapServerProxy::examine(const QString &mailbox) { auto select = new KIMAP2::SelectJob(mSession); select->setOpenReadOnly(true); select->setMailBox(mailbox); select->setCondstoreEnabled(mCapabilities.contains(Capabilities::Condstore)); return runJob(select, [select](KJob* job) -> SelectResult { return {select->uidValidity(), select->nextUid(), select->highestModSequence()}; }).then([=] (const KAsync::Error &error, const SelectResult &result) { if (error) { SinkWarning() << "Examine failed: " << mailbox; return KAsync::error(error); } return KAsync::value(result); }); } KAsync::Job ImapServerProxy::examine(const Folder &folder) { return examine(mailboxFromFolder(folder)); } KAsync::Job ImapServerProxy::append(const QString &mailbox, const QByteArray &content, const QList &flags, const QDateTime &internalDate) { auto append = new KIMAP2::AppendJob(mSession); append->setMailBox(mailbox); append->setContent(content); append->setFlags(flags); append->setInternalDate(internalDate); return runJob(append, [](KJob *job) -> qint64{ return static_cast(job)->uid(); }); } KAsync::Job ImapServerProxy::store(const KIMAP2::ImapSet &set, const QList &flags) { return storeFlags(set, flags); } KAsync::Job ImapServerProxy::storeFlags(const KIMAP2::ImapSet &set, const QList &flags) { auto store = new KIMAP2::StoreJob(mSession); store->setUidBased(true); store->setMode(KIMAP2::StoreJob::SetFlags); store->setSequenceSet(set); store->setFlags(flags); return runJob(store); } KAsync::Job ImapServerProxy::addFlags(const KIMAP2::ImapSet &set, const QList &flags) { auto store = new KIMAP2::StoreJob(mSession); store->setUidBased(true); store->setMode(KIMAP2::StoreJob::AppendFlags); store->setSequenceSet(set); store->setFlags(flags); return runJob(store); } KAsync::Job ImapServerProxy::removeFlags(const KIMAP2::ImapSet &set, const QList &flags) { auto store = new KIMAP2::StoreJob(mSession); store->setUidBased(true); store->setMode(KIMAP2::StoreJob::RemoveFlags); store->setSequenceSet(set); store->setFlags(flags); return runJob(store); } KAsync::Job ImapServerProxy::create(const QString &mailbox) { auto create = new KIMAP2::CreateJob(mSession); create->setMailBox(mailbox); return runJob(create); } KAsync::Job ImapServerProxy::subscribe(const QString &mailbox) { auto job = new KIMAP2::SubscribeJob(mSession); job->setMailBox(mailbox); return runJob(job); } KAsync::Job ImapServerProxy::rename(const QString &mailbox, const QString &newMailbox) { auto rename = new KIMAP2::RenameJob(mSession); rename->setSourceMailBox(mailbox); rename->setDestinationMailBox(newMailbox); return runJob(rename); } KAsync::Job ImapServerProxy::remove(const QString &mailbox) { auto job = new KIMAP2::DeleteJob(mSession); job->setMailBox(mailbox); return runJob(job); } KAsync::Job ImapServerProxy::expunge() { auto job = new KIMAP2::ExpungeJob(mSession); return runJob(job); } KAsync::Job ImapServerProxy::expunge(const KIMAP2::ImapSet &set) { //FIXME implement UID EXPUNGE auto job = new KIMAP2::ExpungeJob(mSession); return runJob(job); } KAsync::Job ImapServerProxy::copy(const KIMAP2::ImapSet &set, const QString &newMailbox) { auto copy = new KIMAP2::CopyJob(mSession); copy->setSequenceSet(set); copy->setUidBased(true); copy->setMailBox(newMailbox); return runJob(copy); } KAsync::Job ImapServerProxy::fetch(const KIMAP2::ImapSet &set, KIMAP2::FetchJob::FetchScope scope, FetchCallback callback) { auto fetch = new KIMAP2::FetchJob(mSession); fetch->setSequenceSet(set); fetch->setUidBased(true); fetch->setScope(scope); fetch->setAvoidParsing(true); QObject::connect(fetch, &KIMAP2::FetchJob::resultReceived, callback); return runJob(fetch); } KAsync::Job> ImapServerProxy::search(const KIMAP2::ImapSet &set) { return search(KIMAP2::Term(KIMAP2::Term::Uid, set)); } KAsync::Job> ImapServerProxy::search(const KIMAP2::Term &term) { auto search = new KIMAP2::SearchJob(mSession); search->setTerm(term); search->setUidBased(true); return runJob>(search, [](KJob *job) -> QVector { return static_cast(job)->results(); }); } KAsync::Job ImapServerProxy::fetch(const KIMAP2::ImapSet &set, KIMAP2::FetchJob::FetchScope scope, const std::function &callback) { const bool fullPayload = (scope.mode == KIMAP2::FetchJob::FetchScope::Full); return fetch(set, scope, [callback, fullPayload](const KIMAP2::FetchJob::Result &result) { callback(Message{result.uid, result.size, result.attributes, result.flags, result.message, fullPayload}); }); } QStringList ImapServerProxy::getCapabilities() const { return mCapabilities; } KAsync::Job> ImapServerProxy::fetchHeaders(const QString &mailbox, const qint64 minUid) { auto list = QSharedPointer>::create(); KIMAP2::FetchJob::FetchScope scope; scope.mode = KIMAP2::FetchJob::FetchScope::Flags; //Fetch headers of all messages return fetch(KIMAP2::ImapSet(minUid, 0), scope, [list](const KIMAP2::FetchJob::Result &result) { // SinkTrace() << "Received " << uids.size() << " headers from " << mailbox; // SinkTrace() << uids.size() << sizes.size() << attrs.size() << flags.size() << messages.size(); //TODO based on the data available here, figure out which messages to actually fetch //(we only fetched headers and structure so far) //We could i.e. build chunks to fetch based on the size list->append(result.uid); }) .then([list](){ return *list; }); } -KAsync::Job> ImapServerProxy::fetchUids(const QString &mailbox) +KAsync::Job> ImapServerProxy::fetchUids() { auto notDeleted = KIMAP2::Term(KIMAP2::Term::Deleted); notDeleted.setNegated(true); - return select(mailbox).then>(search(notDeleted)); + return search(notDeleted); } -KAsync::Job> ImapServerProxy::fetchUidsSince(const QString &mailbox, const QDate &since, qint64 lowerBound) +KAsync::Job> ImapServerProxy::fetchUidsSince(const QDate &since, qint64 lowerBound) { auto notDeleted = KIMAP2::Term{KIMAP2::Term::Deleted}; notDeleted.setNegated(true); - return select(mailbox) - .then>( - search( - KIMAP2::Term{KIMAP2::Term::Or, { - KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Since, since}, notDeleted}}, - KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Uid, KIMAP2::ImapSet{lowerBound, 0}}, notDeleted}} - }} - )); + return search( + KIMAP2::Term{KIMAP2::Term::Or, { + KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Since, since}, notDeleted}}, + KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Uid, KIMAP2::ImapSet{lowerBound, 0}}, notDeleted}} + }} + ); } -KAsync::Job> ImapServerProxy::fetchUidsSince(const QString &mailbox, const QDate &since) +KAsync::Job> ImapServerProxy::fetchUidsSince(const QDate &since) { auto notDeleted = KIMAP2::Term{KIMAP2::Term::Deleted}; notDeleted.setNegated(true); - return select(mailbox) - .then>( - search(KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Since, since}, notDeleted}}) - ); + return search(KIMAP2::Term{KIMAP2::Term::And, {{KIMAP2::Term::Since, since}, notDeleted}}); } KAsync::Job ImapServerProxy::list(KIMAP2::ListJob::Option option, const std::function &flags)> &callback) { auto listJob = new KIMAP2::ListJob(mSession); listJob->setOption(option); // listJob->setQueriedNamespaces(serverNamespaces()); QObject::connect(listJob, &KIMAP2::ListJob::resultReceived, listJob, callback); return runJob(listJob); } KAsync::Job ImapServerProxy::remove(const QString &mailbox, const KIMAP2::ImapSet &set) { return select(mailbox).then(store(set, QByteArrayList() << Flags::Deleted)).then(expunge(set)); } KAsync::Job ImapServerProxy::remove(const QString &mailbox, const QByteArray &imapSet) { const auto set = KIMAP2::ImapSet::fromImapSequenceSet(imapSet); return remove(mailbox, set); } KAsync::Job ImapServerProxy::move(const QString &mailbox, const KIMAP2::ImapSet &set, const QString &newMailbox) { return select(mailbox).then(copy(set, newMailbox)).then(store(set, QByteArrayList() << Flags::Deleted)).then(expunge(set)); } KAsync::Job ImapServerProxy::createSubfolder(const QString &parentMailbox, const QString &folderName) { return KAsync::start([this, parentMailbox, folderName]() { QString folder; if (parentMailbox.isEmpty()) { auto ns = mNamespaces.getDefaultNamespace(); folder = ns.name + folderName; } else { auto ns = mNamespaces.getNamespace(parentMailbox); folder = parentMailbox + ns.separator + folderName; } SinkTrace() << "Creating subfolder: " << folder; return create(folder) .then([=]() { return folder; }); }); } KAsync::Job ImapServerProxy::renameSubfolder(const QString &oldMailbox, const QString &newName) { return KAsync::start([this, oldMailbox, newName] { auto ns = mNamespaces.getNamespace(oldMailbox); auto parts = oldMailbox.split(ns.separator); parts.removeLast(); QString folder = parts.join(ns.separator) + ns.separator + newName; SinkTrace() << "Renaming subfolder: " << oldMailbox << folder; return rename(oldMailbox, folder) .then([=]() { return folder; }); }); } QString ImapServerProxy::getNamespace(const QString &name) { auto ns = mNamespaces.getNamespace(name); return ns.name; } static bool caseInsensitiveContains(const QByteArray &f, const QByteArrayList &list) { return list.contains(f) || list.contains(f.toLower()); } bool Imap::flagsContain(const QByteArray &f, const QByteArrayList &flags) { return caseInsensitiveContains(f, flags); } static void reportFolder(const Folder &f, QSharedPointer> reportedList, std::function callback) { if (!reportedList->contains(f.path())) { reportedList->insert(f.path()); auto c = f; c.noselect = true; callback(c); if (!f.parentPath().isEmpty()){ reportFolder(f.parentFolder(), reportedList, callback); } } } KAsync::Job ImapServerProxy::getMetaData(std::function > &metadata)> callback) { if (!mCapabilities.contains("METADATA")) { return KAsync::null(); } KIMAP2::GetMetaDataJob *meta = new KIMAP2::GetMetaDataJob(mSession); meta->setMailBox(QLatin1String("*")); meta->setServerCapability( KIMAP2::MetaDataJobBase::Metadata ); meta->setDepth(KIMAP2::GetMetaDataJob::AllLevels); meta->addRequestedEntry("/shared/vendor/kolab/folder-type"); meta->addRequestedEntry("/private/vendor/kolab/folder-type"); return runJob(meta).then([callback, meta] () { callback(meta->allMetaDataForMailboxes()); }); } KAsync::Job ImapServerProxy::fetchFolders(std::function callback) { SinkTrace() << "Fetching folders"; auto subscribedList = QSharedPointer>::create() ; auto reportedList = QSharedPointer>::create() ; auto metaData = QSharedPointer>>::create() ; return getMetaData([=] (const QHash> &m) { *metaData = m; }).then(list(KIMAP2::ListJob::NoOption, [=](const KIMAP2::MailBoxDescriptor &mailbox, const QList &){ *subscribedList << mailbox.name; })).then(list(KIMAP2::ListJob::IncludeUnsubscribed, [=](const KIMAP2::MailBoxDescriptor &mailbox, const QList &flags) { bool noselect = caseInsensitiveContains(FolderFlags::Noselect, flags); bool subscribed = subscribedList->contains(mailbox.name); if (isGmail()) { bool inbox = mailbox.name.toLower() == "inbox"; bool sent = caseInsensitiveContains(FolderFlags::Sent, flags); bool drafts = caseInsensitiveContains(FolderFlags::Drafts, flags); bool trash = caseInsensitiveContains(FolderFlags::Trash, flags); /** * Because gmail duplicates messages all over the place we only support a few selected folders for now that should be mostly exclusive. */ if (!(inbox || sent || drafts || trash)) { return; } } SinkTrace() << "Found mailbox: " << mailbox.name << flags << FolderFlags::Noselect << noselect << " sub: " << subscribed; //Ignore all non-mail folders if (metaData->contains(mailbox.name)) { auto m = metaData->value(mailbox.name); auto sharedType = m.value("/shared/vendor/kolab/folder-type"); auto privateType = m.value("/private/vendor/kolab/folder-type"); auto type = !privateType.isEmpty() ? privateType : sharedType; if (!type.isEmpty() && !type.contains("mail")) { SinkTrace() << "Skipping due to folder type: " << type; return; } } auto ns = getNamespace(mailbox.name); auto folder = Folder{mailbox.name, ns, mailbox.separator, noselect, subscribed, flags}; //call callback for parents if that didn't already happen. //This is necessary because we can have missing bits in the hierarchy in IMAP, but this will not work in sink because we'd end up with an incomplete tree. if (!folder.parentPath().isEmpty() && !reportedList->contains(folder.parentPath())) { reportFolder(folder.parentFolder(), reportedList, callback); } reportedList->insert(folder.path()); callback(folder); })); } QString ImapServerProxy::mailboxFromFolder(const Folder &folder) const { Q_ASSERT(!folder.path().isEmpty()); return folder.path(); } -KAsync::Job ImapServerProxy::fetchFlags(const Folder &folder, const KIMAP2::ImapSet &set, qint64 changedsince, std::function callback) +KAsync::Job ImapServerProxy::fetchFlags(const KIMAP2::ImapSet &set, qint64 changedsince, std::function callback) { - SinkTrace() << "Fetching flags " << folder.path(); - return select(folder).then([=](const SelectResult &selectResult) -> KAsync::Job { - SinkTrace() << "Modeseq " << folder.path() << selectResult.highestModSequence << changedsince; - - if (selectResult.highestModSequence == static_cast(changedsince)) { - SinkTrace()<< folder.path() << "Changedsince didn't change, nothing to do."; - return KAsync::value(selectResult); - } - - SinkTrace() << "Fetching flags " << folder.path() << set << selectResult.highestModSequence << changedsince; - - KIMAP2::FetchJob::FetchScope scope; - scope.mode = KIMAP2::FetchJob::FetchScope::Flags; - scope.changedSince = changedsince; + KIMAP2::FetchJob::FetchScope scope; + scope.mode = KIMAP2::FetchJob::FetchScope::Flags; + scope.changedSince = changedsince; - return fetch(set, scope, callback).then([selectResult] { - return selectResult; - }); - }); + return fetch(set, scope, callback); } KAsync::Job ImapServerProxy::fetchMessages(const Folder &folder, qint64 uidNext, std::function callback, std::function progress) { auto time = QSharedPointer::create(); time->start(); return select(folder).then([this, callback, folder, time, progress, uidNext](const SelectResult &selectResult) -> KAsync::Job { SinkTrace() << "UIDNEXT " << folder.path() << selectResult.uidNext << uidNext; if (selectResult.uidNext == (uidNext + 1)) { SinkTrace()<< folder.path() << "Uidnext didn't change, nothing to do."; return KAsync::null(); } SinkTrace() << "Fetching messages from " << folder.path() << selectResult.uidNext << uidNext; return fetchHeaders(mailboxFromFolder(folder), (uidNext + 1)).then>([this, callback, time, progress, folder](const QVector &uidsToFetch){ SinkTrace() << "Fetched headers" << folder.path(); SinkTrace() << " Total: " << uidsToFetch.size(); SinkTrace() << " Uids to fetch: " << uidsToFetch; SinkTrace() << " Took: " << Sink::Log::TraceTime(time->elapsed()); return fetchMessages(folder, uidsToFetch, false, callback, progress); }); }); } KAsync::Job ImapServerProxy::fetchMessages(const Folder &folder, const QVector &uidsToFetch, bool headersOnly, std::function callback, std::function progress) { auto time = QSharedPointer::create(); time->start(); return select(folder).then([this, callback, folder, time, progress, uidsToFetch, headersOnly](const SelectResult &selectResult) -> KAsync::Job { SinkTrace() << "Fetching messages" << folder.path(); SinkTrace() << " Total: " << uidsToFetch.size(); SinkTrace() << " Uids to fetch: " << uidsToFetch; auto totalCount = uidsToFetch.size(); if (progress) { progress(0, totalCount); } if (uidsToFetch.isEmpty()) { SinkTrace() << "Nothing to fetch"; return KAsync::null(); } KIMAP2::FetchJob::FetchScope scope; scope.parts.clear(); if (headersOnly) { scope.mode = KIMAP2::FetchJob::FetchScope::Headers; } else { scope.mode = KIMAP2::FetchJob::FetchScope::Full; } KIMAP2::ImapSet set; set.add(uidsToFetch); auto count = QSharedPointer::create(); return fetch(set, scope, [=](const Message &message) { *count += 1; if (progress) { progress(*count, totalCount); } callback(message); }); }) .then([time]() { SinkTrace() << "The fetch took: " << Sink::Log::TraceTime(time->elapsed()); }); } KAsync::Job ImapServerProxy::fetchMessages(const Folder &folder, std::function callback, std::function progress) { return fetchMessages(folder, 0, callback, progress); } KAsync::Job> ImapServerProxy::fetchUids(const Folder &folder) { - return fetchUids(mailboxFromFolder(folder)); + return select(mailboxFromFolder(folder)).then(fetchUids()); } diff --git a/examples/imapresource/imapserverproxy.h b/examples/imapresource/imapserverproxy.h index 4270e79d..9021b231 100644 --- a/examples/imapresource/imapserverproxy.h +++ b/examples/imapresource/imapserverproxy.h @@ -1,323 +1,323 @@ /* * Copyright (C) 2015 Christian Mollekopf * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the * Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #pragma once #include #include #include #include #include #include namespace Imap { enum ErrorCode { NoError, LoginFailed, HostNotFoundError, CouldNotConnectError, SslHandshakeError, ConnectionLost, MissingCredentialsError, CommandFailed, UnknownError }; namespace Flags { /// The flag for a message being seen (i.e. opened by user). extern const char* Seen; /// The flag for a message being deleted by the user. extern const char* Deleted; /// The flag for a message being replied to by the user. extern const char* Answered; /// The flag for a message being marked as flagged. extern const char* Flagged; } namespace FolderFlags { extern const char* Noinferiors; extern const char* Noselect; extern const char* Marked; extern const char* Unmarked; extern const char* Subscribed; extern const char* Sent; extern const char* Trash; extern const char* Archive; extern const char* Junk; extern const char* Flagged; extern const char* All; extern const char* Drafts; } namespace Capabilities { extern const char* Condstore; extern const char* Uidplus; extern const char* Namespace; } struct Message { qint64 uid; qint64 size; KIMAP2::MessageAttributes attributes; KIMAP2::MessageFlags flags; KMime::Message::Ptr msg; bool fullPayload; }; bool flagsContain(const QByteArray &f, const QByteArrayList &flags); struct Folder { Folder() = default; Folder(const QString &path, const QString &ns, const QChar &separator, bool noselect_, bool subscribed_, const QByteArrayList &flags_) : noselect(noselect_), subscribed(subscribed_), flags(flags_), mPath(path), mNamespace(ns), mSeparator(separator) { } Folder(const QString &path_) : mPath(path_) { } QString path() const { Q_ASSERT(!mPath.isEmpty()); return mPath; } QString parentPath() const { Q_ASSERT(!mSeparator.isNull()); auto parts = mPath.split(mSeparator); parts.removeLast(); auto parentPath = parts.join(mSeparator); //Don't return the namespace for root folders as parent folder if (mNamespace.startsWith(parentPath)) { return QString{}; } return parentPath; } Folder parentFolder() const { Folder parent; parent.mPath = parentPath(); parent.mNamespace = mNamespace; parent.mSeparator = mSeparator; return parent; } QString name() const { auto pathParts = mPath.split(mSeparator); Q_ASSERT(!pathParts.isEmpty()); return pathParts.last(); } bool noselect = false; bool subscribed = false; QByteArrayList flags; private: QString mPath; QString mNamespace; QChar mSeparator; }; struct SelectResult { qint64 uidValidity; qint64 uidNext; quint64 highestModSequence; }; class Namespaces { public: QList personal; QList shared; QList user; KIMAP2::MailBoxDescriptor getDefaultNamespace() { return personal.isEmpty() ? KIMAP2::MailBoxDescriptor{} : personal.first(); } KIMAP2::MailBoxDescriptor getNamespace(const QString &mailbox) { for (const auto &ns : personal) { if (mailbox.startsWith(ns.name)) { return ns; } } for (const auto &ns : shared) { if (mailbox.startsWith(ns.name)) { return ns; } } for (const auto &ns : user) { if (mailbox.startsWith(ns.name)) { return ns; } } return KIMAP2::MailBoxDescriptor{}; } }; class CachedSession { public: CachedSession() = default; CachedSession(KIMAP2::Session *session, const QStringList &cap, const Namespaces &ns) : mSession(session), mCapabilities(cap), mNamespaces(ns) { } bool operator==(const CachedSession &other) const { return mSession && (mSession == other.mSession); } bool isConnected() { return (mSession->state() == KIMAP2::Session::State::Authenticated || mSession->state() == KIMAP2::Session::State::Selected) ; } bool isValid() { return mSession; } KIMAP2::Session *mSession = nullptr; QStringList mCapabilities; Namespaces mNamespaces; }; class SessionCache : public QObject { Q_OBJECT public: void recycleSession(const CachedSession &session) { QObject::connect(session.mSession, &KIMAP2::Session::stateChanged, this, [this, session](KIMAP2::Session::State newState, KIMAP2::Session::State oldState) { if (newState == KIMAP2::Session::Disconnected) { mSessions.removeOne(session); } }); mSessions << session; } CachedSession getSession() { while (!mSessions.isEmpty()) { auto session = mSessions.takeLast(); if (session.isConnected()) { return session; } } return {}; } bool isEmpty() const { return mSessions.isEmpty(); } private: QList mSessions; }; enum EncryptionMode { NoEncryption, Tls, Starttls }; class ImapServerProxy { public: ImapServerProxy(const QString &serverUrl, int port, EncryptionMode encryption, SessionCache *sessionCache = nullptr); //Standard IMAP calls KAsync::Job login(const QString &username, const QString &password); KAsync::Job logout(); KAsync::Job select(const QString &mailbox); KAsync::Job select(const Folder &mailbox); KAsync::Job examine(const QString &mailbox); KAsync::Job examine(const Folder &mailbox); KAsync::Job append(const QString &mailbox, const QByteArray &content, const QList &flags = QList(), const QDateTime &internalDate = QDateTime()); KAsync::Job store(const KIMAP2::ImapSet &set, const QList &flags); KAsync::Job storeFlags(const KIMAP2::ImapSet &set, const QList &flags); KAsync::Job addFlags(const KIMAP2::ImapSet &set, const QList &flags); KAsync::Job removeFlags(const KIMAP2::ImapSet &set, const QList &flags); KAsync::Job create(const QString &mailbox); KAsync::Job rename(const QString &mailbox, const QString &newMailbox); KAsync::Job remove(const QString &mailbox); KAsync::Job subscribe(const QString &mailbox); KAsync::Job expunge(); KAsync::Job expunge(const KIMAP2::ImapSet &set); KAsync::Job copy(const KIMAP2::ImapSet &set, const QString &newMailbox); KAsync::Job> search(const KIMAP2::ImapSet &set); KAsync::Job> search(const KIMAP2::Term &term); typedef std::function FetchCallback; KAsync::Job fetch(const KIMAP2::ImapSet &set, KIMAP2::FetchJob::FetchScope scope, FetchCallback callback); KAsync::Job fetch(const KIMAP2::ImapSet &set, KIMAP2::FetchJob::FetchScope scope, const std::function &callback); KAsync::Job list(KIMAP2::ListJob::Option option, const std::function &flags)> &callback); QStringList getCapabilities() const; //Composed calls that do login etc. KAsync::Job> fetchHeaders(const QString &mailbox, qint64 minUid = 1); KAsync::Job remove(const QString &mailbox, const KIMAP2::ImapSet &set); KAsync::Job remove(const QString &mailbox, const QByteArray &imapSet); KAsync::Job move(const QString &mailbox, const KIMAP2::ImapSet &set, const QString &newMailbox); KAsync::Job createSubfolder(const QString &parentMailbox, const QString &folderName); KAsync::Job renameSubfolder(const QString &mailbox, const QString &newName); - KAsync::Job> fetchUids(const QString &mailbox); - KAsync::Job> fetchUidsSince(const QString &mailbox, const QDate &since); - KAsync::Job> fetchUidsSince(const QString &mailbox, const QDate &since, qint64 lowerBound); + KAsync::Job> fetchUids(); + KAsync::Job> fetchUidsSince(const QDate &since); + KAsync::Job> fetchUidsSince(const QDate &since, qint64 lowerBound); QString mailboxFromFolder(const Folder &) const; KAsync::Job fetchFolders(std::function callback); KAsync::Job fetchMessages(const Folder &folder, std::function callback, std::function progress = std::function()); KAsync::Job fetchMessages(const Folder &folder, qint64 uidNext, std::function callback, std::function progress = std::function()); KAsync::Job fetchMessages(const Folder &folder, const QVector &uidsToFetch, bool headersOnly, std::function callback, std::function progress); - KAsync::Job fetchFlags(const Folder &folder, const KIMAP2::ImapSet &set, qint64 changedsince, std::function callback); + KAsync::Job fetchFlags(const KIMAP2::ImapSet &set, qint64 changedsince, std::function callback); KAsync::Job> fetchUids(const Folder &folder); private: KAsync::Job getMetaData(std::function > &metadata)> callback); bool isGmail() const; QString getNamespace(const QString &name); QObject mGuard; SessionCache *mSessionCache; KIMAP2::Session *mSession; QStringList mCapabilities; Namespaces mNamespaces; EncryptionMode mEncryptionMode; }; }