diff --git a/autotests/libs/testrunner/setup.cpp b/autotests/libs/testrunner/setup.cpp index 02d3d4554..0f7b61279 100644 --- a/autotests/libs/testrunner/setup.cpp +++ b/autotests/libs/testrunner/setup.cpp @@ -1,457 +1,457 @@ /* * Copyright (c) 2008 Igor Trindade Oliveira * Copyright (c) 2013 Volker Krause * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . */ #include "setup.h" #include "config.h" //krazy:exclude=includes #include "akonaditest_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include bool SetupTest::startAkonadiDaemon() { Q_ASSERT(Akonadi::ServerManager::hasInstanceIdentifier()); if (!mAkonadiDaemonProcess) { mAkonadiDaemonProcess = new KProcess(this); - connect(mAkonadiDaemonProcess, QOverload::of(&KProcess::finished), + connect(mAkonadiDaemonProcess, QOverload::of(&KProcess::finished), this, &SetupTest::slotAkonadiDaemonProcessFinished); } mAkonadiDaemonProcess->setProgram(Akonadi::StandardDirs::findExecutable(QStringLiteral("akonadi_control")), { QStringLiteral("--instance"), instanceId() }); mAkonadiDaemonProcess->start(); const bool started = mAkonadiDaemonProcess->waitForStarted(5000); qCInfo(AKONADITEST_LOG) << "Started akonadi daemon with pid:" << mAkonadiDaemonProcess->pid(); return started; } void SetupTest::stopAkonadiDaemon() { if (!mAkonadiDaemonProcess) { return; } disconnect(mAkonadiDaemonProcess, SIGNAL(finished(int)), this, nullptr); mAkonadiDaemonProcess->terminate(); const bool finished = mAkonadiDaemonProcess->waitForFinished(5000); if (!finished) { qCDebug(AKONADITEST_LOG) << "Problem finishing process."; } mAkonadiDaemonProcess->close(); mAkonadiDaemonProcess->deleteLater(); mAkonadiDaemonProcess = nullptr; } void SetupTest::setupAgents() { if (mAgentsCreated) { return; } mAgentsCreated = true; Config *config = Config::instance(); const auto agents = config->agents(); for (const auto agent : agents) { qCDebug(AKONADITEST_LOG) << "Creating agent" << agent.first << "..."; ++mSetupJobCount; Akonadi::AgentInstanceCreateJob *job = new Akonadi::AgentInstanceCreateJob(agent.first, this); job->setProperty("sync", agent.second); connect(job, &Akonadi::AgentInstanceCreateJob::result, this, &SetupTest::agentCreationResult); job->start(); } checkSetupDone(); } void SetupTest::agentCreationResult(KJob *job) { qCDebug(AKONADITEST_LOG) << "Agent created"; --mSetupJobCount; if (job->error()) { qCritical() << "Failed to create agent:" << job->errorString(); setupFailed(); } else { const bool needsSync = job->property("sync").toBool(); const auto instance = qobject_cast(job)->instance(); qCDebug(AKONADITEST_LOG) << "Agent" << instance.identifier() << "created"; if (needsSync) { ++mSetupJobCount; qCDebug(AKONADITEST_LOG) << "Scheduling Agent sync of" << instance.identifier(); Akonadi::ResourceSynchronizationJob *sync = new Akonadi::ResourceSynchronizationJob(instance, this); connect(sync, &Akonadi::ResourceSynchronizationJob::result, this, &SetupTest::synchronizationResult); sync->start(); } } checkSetupDone(); } void SetupTest::synchronizationResult(KJob *job) { auto instance = qobject_cast(job)->resource(); qCDebug(AKONADITEST_LOG) << "Sync of" << instance.identifier() << "done"; --mSetupJobCount; if (job->error()) { qCritical() << job->errorString(); setupFailed(); } checkSetupDone(); } void SetupTest::serverStateChanged(Akonadi::ServerManager::State state) { if (state == Akonadi::ServerManager::Running) { setupAgents(); } else if (mShuttingDown && state == Akonadi::ServerManager::NotRunning) { shutdownHarder(); } } void SetupTest::copyXdgDirectory(const QString &src, const QString &dst) { qCDebug(AKONADITEST_LOG) << "Copying" << src << "to" << dst; const QDir srcDir(src); const auto entries = srcDir.entryInfoList(QDir::Dirs | QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot); for (const auto &fi : entries) { if (fi.isDir()) { if (fi.fileName() == QLatin1String("akonadi")) { // namespace according to instance identifier #ifdef Q_OS_WIN const bool isXdgConfig = src.contains(QLatin1String("/xdgconfig/")); copyDirectory(fi.absoluteFilePath(), dst + QStringLiteral("/akonadi/") + (isXdgConfig ? QStringLiteral("config/") : QStringLiteral("data/")) + QStringLiteral("instance/") + instanceId()); #else copyDirectory(fi.absoluteFilePath(), dst + QStringLiteral("/akonadi/instance/") + instanceId()); #endif } else { copyDirectory(fi.absoluteFilePath(), dst + QLatin1Char('/') + fi.fileName()); } } else { if (fi.fileName().startsWith(QLatin1String("akonadi_")) && fi.fileName().endsWith(QLatin1String("rc"))) { // namespace according to instance identifier const QString baseName = fi.fileName().left(fi.fileName().size() - 2); const QString dstPath = dst + QLatin1Char('/') + Akonadi::ServerManager::addNamespace(baseName) + QStringLiteral("rc"); if (!QFile::copy(fi.absoluteFilePath(), dstPath)) { qCWarning(AKONADITEST_LOG) << "Failed to copy" << fi.absoluteFilePath() << "to" << dstPath; } } else { const QString dstPath = dst + QLatin1Char('/') + fi.fileName(); if (!QFile::copy(fi.absoluteFilePath(), dstPath)) { qCWarning(AKONADITEST_LOG) << "Failed to copy" << fi.absoluteFilePath() << "to" << dstPath; } } } } } void SetupTest::copyDirectory(const QString &src, const QString &dst) { const QDir srcDir(src); QDir::root().mkpath(dst); const auto entries = srcDir.entryInfoList(QDir::Dirs | QDir::Files | QDir::NoSymLinks | QDir::NoDotAndDotDot); for (const auto &fi : entries) { const QString dstPath = dst + QLatin1Char('/') + fi.fileName(); if (fi.isDir()) { copyDirectory(fi.absoluteFilePath(), dstPath); } else { if (!QFile::copy(fi.absoluteFilePath(), dstPath)) { qCWarning(AKONADITEST_LOG) << "Failed to copy" << fi.absoluteFilePath() << "to" << dstPath; } } } } void SetupTest::createTempEnvironment() { qCDebug(AKONADITEST_LOG) << "Creating test environment in" << basePath(); const Config *config = Config::instance(); #ifdef Q_OS_WIN // Always copy the generic xdgconfig dir copyXdgDirectory(config->basePath() + QStringLiteral("/xdgconfig"), basePath()); if (!config->xdgConfigHome().isEmpty()) { copyXdgDirectory(config->xdgConfigHome(), basePath()); } copyXdgDirectory(config->xdgDataHome(), basePath()); setEnvironmentVariable("XDG_DATA_HOME", basePath()); setEnvironmentVariable("XDG_CONFIG_HOME", basePath()); writeAkonadiserverrc(basePath() + QStringLiteral("/akonadi/config/instance/%1/akonadiserverrc").arg(instanceId())); #else const QDir tmpDir(basePath()); const QString testRunnerDataDir = QStringLiteral("data"); const QString testRunnerConfigDir = QStringLiteral("config"); const QString testRunnerTmpDir = QStringLiteral("tmp"); tmpDir.mkpath(testRunnerConfigDir); tmpDir.mkpath(testRunnerDataDir); tmpDir.mkpath(testRunnerTmpDir); // Always copy the generic xdgconfig dir copyXdgDirectory(config->basePath() + QStringLiteral("/xdgconfig"), basePath() + testRunnerConfigDir); if (!config->xdgConfigHome().isEmpty()) { copyXdgDirectory(config->xdgConfigHome(), basePath() + testRunnerConfigDir); } copyXdgDirectory(config->xdgDataHome(), basePath() + testRunnerDataDir); setEnvironmentVariable("XDG_DATA_HOME", basePath() + testRunnerDataDir); setEnvironmentVariable("XDG_CONFIG_HOME", basePath() + testRunnerConfigDir); setEnvironmentVariable("TMPDIR", basePath() + testRunnerTmpDir); writeAkonadiserverrc(basePath() + testRunnerConfigDir + QStringLiteral("/akonadi/instance/%1/akonadiserverrc").arg(instanceId())); #endif QString backend; if (Config::instance()->dbBackend() == QLatin1String("pgsql")) { backend = QStringLiteral("postgresql"); } else { backend = Config::instance()->dbBackend(); } setEnvironmentVariable("TESTRUNNER_DB_ENVIRONMENT", backend); } void SetupTest::writeAkonadiserverrc(const QString &path) { QString backend; if (Config::instance()->dbBackend() == QLatin1String("sqlite")) { backend = QStringLiteral("QSQLITE3"); } else if (Config::instance()->dbBackend() == QLatin1String("mysql")) { backend = QStringLiteral("QMYSQL"); } else if (Config::instance()->dbBackend() == QLatin1String("pgsql")) { backend = QStringLiteral("QPSQL"); } else { qCCritical(AKONADITEST_LOG, "Invalid backend name %s", qPrintable(backend)); return; } QSettings settings(path, QSettings::IniFormat); settings.beginGroup(QStringLiteral("General")); settings.setValue(QStringLiteral("Driver"), backend); settings.endGroup(); settings.beginGroup(QStringLiteral("Search")); settings.setValue(QStringLiteral("Manager"), QStringLiteral("Dummy")); settings.endGroup(); settings.beginGroup(QStringLiteral("Debug")); settings.setValue(QStringLiteral("Tracer"), QStringLiteral("null")); settings.endGroup(); qCDebug(AKONADITEST_LOG) << "Written akonadiserverrc to" << settings.fileName(); } void SetupTest::cleanTempEnvironment() { #ifdef Q_OS_WIN QDir(basePath() + QStringLiteral("akonadi/config/instance/") + instanceId()).removeRecursively(); QDir(basePath() + QStringLiteral("akonadi/data/instance/") + instanceId()).removeRecursively(); #else QDir(basePath()).removeRecursively(); #endif } SetupTest::SetupTest() : mAkonadiDaemonProcess(nullptr) , mShuttingDown(false) , mAgentsCreated(false) , mTrackAkonadiProcess(true) , mSetupJobCount(0) , mExitCode(0) { setupInstanceId(); cleanTempEnvironment(); createTempEnvironment(); // switch off agent auto-starting by default, can be re-enabled if really needed inside the config.xml setEnvironmentVariable("AKONADI_DISABLE_AGENT_AUTOSTART", QStringLiteral("true")); setEnvironmentVariable("AKONADI_TESTRUNNER_PID", QString::number(QCoreApplication::instance()->applicationPid())); // enable all debugging, so we get some useful information when test fails setEnvironmentVariable("QT_LOGGING_RULES", QStringLiteral("* = true\n" "qt.* = false\n" "kf5.coreaddons.desktopparser.debug = false")); // avoid KIO starting klauncher which can get the CI stuck setEnvironmentVariable("KDE_FORK_SLAVES", QStringLiteral("yes")); setEnvironmentVariable("KIO_DISABLE_CACHE_CLEANER", QStringLiteral("yes")); QHashIterator iter(Config::instance()->envVars()); while (iter.hasNext()) { iter.next(); qCDebug(AKONADITEST_LOG) << "Setting environment variable" << iter.key() << "=" << iter.value(); setEnvironmentVariable(iter.key().toLocal8Bit(), iter.value()); } // No kres-migrator please KConfig migratorConfig(basePath() + QStringLiteral("config/kres-migratorrc")); KConfigGroup migrationCfg(&migratorConfig, "Migration"); migrationCfg.writeEntry("Enabled", false); connect(Akonadi::ServerManager::self(), &Akonadi::ServerManager::stateChanged, this, &SetupTest::serverStateChanged); QDBusConnection::sessionBus().registerService(QStringLiteral("org.kde.Akonadi.Testrunner-") + QString::number(QCoreApplication::instance()->applicationPid())); QDBusConnection::sessionBus().registerObject(QStringLiteral("/"), this, QDBusConnection::ExportScriptableSlots); } SetupTest::~SetupTest() { cleanTempEnvironment(); } void SetupTest::shutdown() { if (mShuttingDown) { return; } mShuttingDown = true; switch (Akonadi::ServerManager::self()->state()) { case Akonadi::ServerManager::Running: case Akonadi::ServerManager::Starting: case Akonadi::ServerManager::Upgrading: qCInfo(AKONADITEST_LOG) << "Shutting down Akonadi control..."; Akonadi::ServerManager::self()->stop(); // safety timeout QTimer::singleShot(30 * 1000, this, &SetupTest::shutdownHarder); break; case Akonadi::ServerManager::NotRunning: case Akonadi::ServerManager::Broken: shutdownHarder(); break; case Akonadi::ServerManager::Stopping: // safety timeout QTimer::singleShot(30 * 1000, this, &SetupTest::shutdownHarder); break; } } void SetupTest::shutdownHarder() { qCDebug(AKONADITEST_LOG) << "Forcing akonaditest shutdown"; mShuttingDown = false; stopAkonadiDaemon(); QCoreApplication::instance()->exit(mExitCode); } void SetupTest::restartAkonadiServer() { qCDebug(AKONADITEST_LOG) << "Restarting Akonadi"; disconnect(mAkonadiDaemonProcess, SIGNAL(finished(int)), this, nullptr); Akonadi::ServerManager::self()->stop(); const bool shutdownResult = mAkonadiDaemonProcess->waitForFinished(); if (!shutdownResult) { qCWarning(AKONADITEST_LOG) << "Akonadi control did not shut down in time, killing it."; mAkonadiDaemonProcess->kill(); } // we don't use Control::start() since we want to be able to kill // it forcefully, if necessary, and know the pid startAkonadiDaemon(); // from here on, the server exiting is an error again connect(mAkonadiDaemonProcess, SIGNAL(finished(int)), this, SLOT(slotAkonadiDaemonProcessFinished(int))); } QString SetupTest::basePath() const { #ifdef Q_OS_WIN // On Windows we are forced to share the same data directory as production instances // because there's no way to override QStandardPaths like we can on Unix. // This means that on Windows we rely on Instances providing us the necessary isolation return QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation); #else QString sysTempDirPath = QDir::tempPath(); #ifdef Q_OS_UNIX // QDir::tempPath() makes sure to use the fully sym-link exploded // absolute path to the temp dir. That is nice, but on OSX it makes // that path really long. MySQL chokes on this, for it's socket path, // so work around that sysTempDirPath = QStringLiteral("/tmp"); #endif const QDir sysTempDir(sysTempDirPath); const QString tempDir = QStringLiteral("/aktestrunner-%1/") .arg(QCoreApplication::instance()->applicationPid()); if (!sysTempDir.exists(tempDir)) { sysTempDir.mkdir(tempDir); } return sysTempDirPath + tempDir; #endif } void SetupTest::slotAkonadiDaemonProcessFinished(int exitCode) { if (mTrackAkonadiProcess || exitCode != EXIT_SUCCESS) { qCWarning(AKONADITEST_LOG) << "Akonadi server process was terminated externally!"; Q_EMIT serverExited(exitCode); } mAkonadiDaemonProcess = nullptr; } void SetupTest::trackAkonadiProcess(bool track) { mTrackAkonadiProcess = track; } QString SetupTest::instanceId() const { return QStringLiteral("testrunner-") + QString::number(QCoreApplication::instance()->applicationPid()); } void SetupTest::setupInstanceId() { setEnvironmentVariable("AKONADI_INSTANCE", instanceId()); } void SetupTest::checkSetupDone() { qCDebug(AKONADITEST_LOG) << "checkSetupDone: pendingJobs =" << mSetupJobCount << ", exitCode =" << mExitCode; if (mSetupJobCount == 0) { if (mExitCode != 0) { qCInfo(AKONADITEST_LOG) << "Setup has failed, aborting test."; shutdown(); } else { qCInfo(AKONADITEST_LOG) << "Setup successful"; Q_EMIT setupDone(); } } } void SetupTest::setupFailed() { mExitCode = 1; } void SetupTest::setEnvironmentVariable(const QByteArray &name, const QString &value) { mEnvVars.push_back(qMakePair(name, value.toLocal8Bit())); qputenv(name.constData(), value.toLatin1()); } QVector SetupTest::environmentVariables() const { return mEnvVars; } diff --git a/autotests/libs/testrunner/testrunner.cpp b/autotests/libs/testrunner/testrunner.cpp index b69cfae00..5cb4fe1e6 100644 --- a/autotests/libs/testrunner/testrunner.cpp +++ b/autotests/libs/testrunner/testrunner.cpp @@ -1,83 +1,83 @@ /* * Copyright (c) 2009 Volker Krause * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . */ #include "testrunner.h" #include "akonaditest_debug.h" #include TestRunner::TestRunner(const QStringList &args, QObject *parent) : QObject(parent) , mArguments(args) , mExitCode(0) , mProcess(nullptr) { } int TestRunner::exitCode() const { return mExitCode; } void TestRunner::run() { qCDebug(AKONADITEST_LOG) << "Starting test" << mArguments; mProcess = new KProcess(this); mProcess->setProgram(mArguments); - connect(mProcess, QOverload::of(&KProcess::finished), this, &TestRunner::processFinished); + connect(mProcess, QOverload::of(&KProcess::finished), this, &TestRunner::processFinished); connect(mProcess, &KProcess::errorOccurred, this, &TestRunner::processError); // environment setup seems to have been done by setuptest globally already mProcess->start(); if (!mProcess->waitForStarted()) { qCWarning(AKONADITEST_LOG) << mArguments << "failed to start!"; mExitCode = 255; Q_EMIT finished(); } } void TestRunner::triggerTermination(int exitCode) { processFinished(exitCode); } void TestRunner::processFinished(int exitCode) { // Only update the exit code when it is 0. This prevents overwriting a non-zero // value with 0. This can happen when multiple processes finish or triggerTermination // is called after a process has finished. if (mExitCode == 0) { mExitCode = exitCode; qCInfo(AKONADITEST_LOG) << "Test finished with exist code" << exitCode; } Q_EMIT finished(); } void TestRunner::processError(QProcess::ProcessError error) { qCWarning(AKONADITEST_LOG) << mArguments << "exited with an error:" << error; mExitCode = 255; Q_EMIT finished(); } void TestRunner::terminate() { if (mProcess) { mProcess->terminate(); } } diff --git a/src/core/jobs/job.cpp b/src/core/jobs/job.cpp index 83a995b0c..26c028002 100644 --- a/src/core/jobs/job.cpp +++ b/src/core/jobs/job.cpp @@ -1,415 +1,416 @@ /* Copyright (c) 2006 Tobias Koenig 2006 Marc Mutz 2006 - 2007 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "job.h" #include "job_p.h" #include "akonadicore_debug.h" #include "KDBusConnectionPool" #include #include "private/protocol_p.h" #include "private/instance_p.h" #include "session.h" #include "session_p.h" #include #include +#include #include #include using namespace Akonadi; static QDBusAbstractInterface *s_jobtracker = nullptr; //@cond PRIVATE void JobPrivate::handleResponse(qint64 tag, const Protocol::CommandPtr &response) { Q_Q(Job); if (mCurrentSubJob) { mCurrentSubJob->d_ptr->handleResponse(tag, response); return; } if (tag == mTag) { if (response->isResponse()) { const auto &resp = Protocol::cmdCast(response); if (resp.isError()) { q->setError(Job::Unknown); q->setErrorText(resp.errorMessage()); q->emitResult(); return; } } } if (mTag != tag) { qCWarning(AKONADICORE_LOG) << "Received response with a different tag!"; qCDebug(AKONADICORE_LOG) << "Response tag:" << tag << ", response type:" << response->type(); qCDebug(AKONADICORE_LOG) << "Job tag:" << mTag << ", job:" << q; return; } if (mStarted) { if (mReadingFinished) { qCWarning(AKONADICORE_LOG) << "Received response for a job that does not expect any more data, ignoring"; qCDebug(AKONADICORE_LOG) << "Response tag:" << tag << ", response type:" << response->type(); qCDebug(AKONADICORE_LOG) << "Job tag:" << mTag << ", job:" << q; Q_ASSERT(!mReadingFinished); return; } if (q->doHandleResponse(tag, response)) { mReadingFinished = true; QTimer::singleShot(0, q, [this]() {delayedEmitResult(); }); } } } void JobPrivate::init(QObject *parent) { Q_Q(Job); mParentJob = qobject_cast(parent); mSession = qobject_cast(parent); if (!mSession) { if (!mParentJob) { mSession = Session::defaultSession(); } else { mSession = mParentJob->d_ptr->mSession; } } if (!mParentJob) { mSession->d->addJob(q); } else { mParentJob->addSubjob(q); } publishJob(); } void JobPrivate::publishJob() { Q_Q(Job); // if there's a job tracker running, tell it about the new job if (!s_jobtracker) { // Let's only check for the debugging console every 3 seconds, otherwise every single job // makes a dbus call to the dbus daemon, doesn't help performance. - static QTime s_lastTime; - if (s_lastTime.isNull() || s_lastTime.elapsed() > 3000) { - if (s_lastTime.isNull()) { + static QElapsedTimer s_lastTime; + if (!s_lastTime.isValid() || s_lastTime.elapsed() > 3000) { + if (!s_lastTime.isValid()) { s_lastTime.start(); } const QString suffix = Akonadi::Instance::identifier().isEmpty() ? QString() : QLatin1Char('-') + Akonadi::Instance::identifier(); if (KDBusConnectionPool::threadConnection().interface()->isServiceRegistered(QStringLiteral("org.kde.akonadiconsole") + suffix)) { s_jobtracker = new QDBusInterface(QLatin1String("org.kde.akonadiconsole") + suffix, QStringLiteral("/jobtracker"), QStringLiteral("org.freedesktop.Akonadi.JobTracker"), KDBusConnectionPool::threadConnection(), nullptr); mSession->d->publishOtherJobs(q); } else { s_lastTime.restart(); } } // Note: we never reset s_jobtracker to 0 when a call fails; but if we did // then we should restart s_lastTime. } QMetaObject::invokeMethod(q, "signalCreationToJobTracker", Qt::QueuedConnection); } void JobPrivate::signalCreationToJobTracker() { Q_Q(Job); if (s_jobtracker) { // We do these dbus calls manually, so as to avoid having to install (or copy) the console's // xml interface document. Since this is purely a debugging aid, that seems preferable to // publishing something not intended for public consumption. // WARNING: for any signature change here, apply it to resourcescheduler.cpp too const QList argumentList = QList() << QLatin1String(mSession->sessionId()) << QString::number(reinterpret_cast(q), 16) << (mParentJob ? QString::number(reinterpret_cast(mParentJob), 16) : QString()) << QString::fromLatin1(q->metaObject()->className()) << jobDebuggingString(); QDBusPendingCall call = s_jobtracker->asyncCallWithArgumentList(QStringLiteral("jobCreated"), argumentList); QDBusPendingCallWatcher *watcher = new QDBusPendingCallWatcher(call, s_jobtracker); QObject::connect(watcher, &QDBusPendingCallWatcher::finished, s_jobtracker, [](QDBusPendingCallWatcher *w) { QDBusPendingReply reply = *w; if (reply.isError() && s_jobtracker) { qDebug() << reply.error().name() << reply.error().message(); s_jobtracker->deleteLater(); s_jobtracker = nullptr; } w->deleteLater(); }); } } void JobPrivate::signalStartedToJobTracker() { Q_Q(Job); if (s_jobtracker) { // if there's a job tracker running, tell it a job started const QList argumentList = { QString::number(reinterpret_cast(q), 16) }; s_jobtracker->callWithArgumentList(QDBus::NoBlock, QStringLiteral("jobStarted"), argumentList); } } void JobPrivate::aboutToFinish() { // Dummy } void JobPrivate::delayedEmitResult() { Q_Q(Job); if (q->hasSubjobs()) { // We still have subjobs, wait for them to finish mFinishPending = true; } else { aboutToFinish(); q->emitResult(); } } void JobPrivate::startQueued() { Q_Q(Job); mStarted = true; Q_EMIT q->aboutToStart(q); q->doStart(); QTimer::singleShot(0, q, [this]() { startNext(); }); QMetaObject::invokeMethod(q, "signalStartedToJobTracker", Qt::QueuedConnection); } void JobPrivate::lostConnection() { Q_Q(Job); if (mCurrentSubJob) { mCurrentSubJob->d_ptr->lostConnection(); } else { q->setError(Job::ConnectionFailed); q->emitResult(); } } void JobPrivate::slotSubJobAboutToStart(Job *job) { Q_ASSERT(mCurrentSubJob == nullptr); mCurrentSubJob = job; } void JobPrivate::startNext() { Q_Q(Job); if (mStarted && !mCurrentSubJob && q->hasSubjobs()) { Job *job = qobject_cast(q->subjobs().at(0)); Q_ASSERT(job); job->d_ptr->startQueued(); } else if (mFinishPending && !q->hasSubjobs()) { // The last subjob we've been waiting for has finished, emitResult() finally QTimer::singleShot(0, q, [this]() {delayedEmitResult(); }); } } qint64 JobPrivate::newTag() { if (mParentJob) { mTag = mParentJob->d_ptr->newTag(); } else { mTag = mSession->d->nextTag(); } return mTag; } qint64 JobPrivate::tag() const { return mTag; } void JobPrivate::sendCommand(qint64 tag, const Protocol::CommandPtr &cmd) { if (mParentJob) { mParentJob->d_ptr->sendCommand(tag, cmd); } else { mSession->d->sendCommand(tag, cmd); } } void JobPrivate::sendCommand(const Protocol::CommandPtr &cmd) { sendCommand(newTag(), cmd); } void JobPrivate::itemRevisionChanged(Akonadi::Item::Id itemId, int oldRevision, int newRevision) { mSession->d->itemRevisionChanged(itemId, oldRevision, newRevision); } void JobPrivate::updateItemRevision(Akonadi::Item::Id itemId, int oldRevision, int newRevision) { Q_Q(Job); foreach (KJob *j, q->subjobs()) { Akonadi::Job *job = qobject_cast(j); if (job) { job->d_ptr->updateItemRevision(itemId, oldRevision, newRevision); } } doUpdateItemRevision(itemId, oldRevision, newRevision); } void JobPrivate::doUpdateItemRevision(Akonadi::Item::Id itemId, int oldRevision, int newRevision) { Q_UNUSED(itemId); Q_UNUSED(oldRevision); Q_UNUSED(newRevision); } int JobPrivate::protocolVersion() const { return mSession->d->protocolVersion; } //@endcond Job::Job(QObject *parent) : KCompositeJob(parent) , d_ptr(new JobPrivate(this)) { d_ptr->init(parent); } Job::Job(JobPrivate *dd, QObject *parent) : KCompositeJob(parent) , d_ptr(dd) { d_ptr->init(parent); } Job::~Job() { delete d_ptr; // if there is a job tracer listening, tell it the job is done now if (s_jobtracker) { const QList argumentList = {QString::number(reinterpret_cast(this), 16), errorString()}; s_jobtracker->callWithArgumentList(QDBus::NoBlock, QStringLiteral("jobEnded"), argumentList); } } void Job::start() { } bool Job::doKill() { Q_D(Job); if (d->mStarted) { // the only way to cancel an already started job is reconnecting to the server d->mSession->d->forceReconnect(); } d->mStarted = false; return true; } QString Job::errorString() const { QString str; switch (error()) { case NoError: break; case ConnectionFailed: str = i18n("Cannot connect to the Akonadi service."); break; case ProtocolVersionMismatch: str = i18n("The protocol version of the Akonadi server is incompatible. Make sure you have a compatible version installed."); break; case UserCanceled: str = i18n("User canceled operation."); break; case Unknown: return errorText(); case UserError: str = i18n("Unknown error."); break; } if (!errorText().isEmpty()) { str += QStringLiteral(" (%1)").arg(errorText()); } return str; } bool Job::addSubjob(KJob *job) { bool rv = KCompositeJob::addSubjob(job); if (rv) { connect(job, SIGNAL(aboutToStart(Akonadi::Job*)), SLOT(slotSubJobAboutToStart(Akonadi::Job*))); QTimer::singleShot(0, this, [this]() { d_ptr->startNext(); }); } return rv; } bool Job::removeSubjob(KJob *job) { bool rv = KCompositeJob::removeSubjob(job); if (job == d_ptr->mCurrentSubJob) { d_ptr->mCurrentSubJob = nullptr; QTimer::singleShot(0, this, [this]() { d_ptr->startNext(); }); } return rv; } bool Akonadi::Job::doHandleResponse(qint64 tag, const Akonadi::Protocol::CommandPtr &response) { qCDebug(AKONADICORE_LOG) << this << "Unhandled response: " << tag << Protocol::debugString(response); setError(Unknown); setErrorText(i18n("Unexpected response")); emitResult(); return true; } void Job::slotResult(KJob *job) { if (d_ptr->mCurrentSubJob == job) { // current job finished, start the next one d_ptr->mCurrentSubJob = nullptr; KCompositeJob::slotResult(job); if (!job->error()) { QTimer::singleShot(0, this, [this]() { d_ptr->startNext(); }); } } else { // job that was still waiting for execution finished, probably canceled, // so just remove it from the queue and move on without caring about // its error code KCompositeJob::removeSubjob(job); } } void Job::emitWriteFinished() { d_ptr->mWriteFinished = true; Q_EMIT writeFinished(this); } #include "moc_job.cpp" diff --git a/src/core/models/entitytreemodel_p.cpp b/src/core/models/entitytreemodel_p.cpp index 340c1c8ee..9bb2fd7d1 100644 --- a/src/core/models/entitytreemodel_p.cpp +++ b/src/core/models/entitytreemodel_p.cpp @@ -1,1980 +1,1980 @@ /* Copyright (c) 2008 Stephen Kelly This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "entitytreemodel_p.h" #include "entitytreemodel.h" #include "agentmanagerinterface.h" #include "monitor_p.h" // For friend ref/deref #include "servermanager.h" #include "akranges.h" #include #include "agentmanager.h" #include "agenttype.h" #include "monitor.h" #include "changerecorder.h" #include "collectioncopyjob.h" #include "collectionfetchjob.h" #include "collectionfetchscope.h" #include "collectionmovejob.h" #include "collectionstatistics.h" #include "collectionstatisticsjob.h" #include "entityhiddenattribute.h" #include "itemcopyjob.h" #include "itemfetchjob.h" #include "itemmodifyjob.h" #include "itemmovejob.h" #include "linkjob.h" #include "session.h" #include "private/protocol_p.h" #include "akonadicore_debug.h" #include #include #include QHash jobTimeTracker; Q_LOGGING_CATEGORY(DebugETM, "org.kde.pim.akonadi.ETM", QtInfoMsg) using namespace Akonadi; using namespace AkRanges; static CollectionFetchJob::Type getFetchType(EntityTreeModel::CollectionFetchStrategy strategy) { switch (strategy) { case EntityTreeModel::FetchFirstLevelChildCollections: return CollectionFetchJob::FirstLevel; case EntityTreeModel::InvisibleCollectionFetch: case EntityTreeModel::FetchCollectionsRecursive: default: break; } return CollectionFetchJob::Recursive; } EntityTreeModelPrivate::EntityTreeModelPrivate(EntityTreeModel *parent) : q_ptr(parent) , m_monitor(nullptr) , m_rootNode(nullptr) , m_collectionFetchStrategy(EntityTreeModel::FetchCollectionsRecursive) , m_itemPopulation(EntityTreeModel::ImmediatePopulation) , m_listFilter(CollectionFetchScope::NoFilter) , m_includeStatistics(false) , m_showRootCollection(false) , m_collectionTreeFetched(false) , m_session(nullptr) , m_showSystemEntities(false) { // using collection as a parameter of a queued call in runItemFetchJob() qRegisterMetaType(); Akonadi::AgentManager *agentManager = Akonadi::AgentManager::self(); QObject::connect(agentManager, SIGNAL(instanceRemoved(Akonadi::AgentInstance)), q_ptr, SLOT(agentInstanceRemoved(Akonadi::AgentInstance))); } EntityTreeModelPrivate::~EntityTreeModelPrivate() { if (m_needDeleteRootNode) { delete m_rootNode; } m_rootNode = nullptr; } void EntityTreeModelPrivate::init(Monitor *monitor) { Q_Q(EntityTreeModel); Q_ASSERT(!m_monitor); m_monitor = monitor; // The default is to FetchCollectionsRecursive, so we tell the monitor to fetch collections // That way update signals from the monitor will contain the full collection. // This may be updated if the CollectionFetchStrategy is changed. m_monitor->fetchCollection(true); m_session = m_monitor->session(); m_rootCollectionDisplayName = QStringLiteral("[*]"); if (Akonadi::ChangeRecorder *cr = qobject_cast(m_monitor)) { cr->setChangeRecordingEnabled(false); } m_includeStatistics = true; m_monitor->fetchCollectionStatistics(true); m_monitor->collectionFetchScope().setAncestorRetrieval(Akonadi::CollectionFetchScope::All); q->connect(monitor, SIGNAL(mimeTypeMonitored(QString,bool)), SLOT(monitoredMimeTypeChanged(QString,bool))); q->connect(monitor, SIGNAL(collectionMonitored(Akonadi::Collection,bool)), SLOT(monitoredCollectionsChanged(Akonadi::Collection,bool))); q->connect(monitor, SIGNAL(itemMonitored(Akonadi::Item,bool)), SLOT(monitoredItemsChanged(Akonadi::Item,bool))); q->connect(monitor, SIGNAL(resourceMonitored(QByteArray,bool)), SLOT(monitoredResourcesChanged(QByteArray,bool))); // monitor collection changes q->connect(monitor, SIGNAL(collectionChanged(Akonadi::Collection)), SLOT(monitoredCollectionChanged(Akonadi::Collection))); q->connect(monitor, SIGNAL(collectionAdded(Akonadi::Collection,Akonadi::Collection)), SLOT(monitoredCollectionAdded(Akonadi::Collection,Akonadi::Collection))); q->connect(monitor, SIGNAL(collectionRemoved(Akonadi::Collection)), SLOT(monitoredCollectionRemoved(Akonadi::Collection))); q->connect(monitor, SIGNAL(collectionMoved(Akonadi::Collection,Akonadi::Collection,Akonadi::Collection)), SLOT(monitoredCollectionMoved(Akonadi::Collection,Akonadi::Collection,Akonadi::Collection))); // Monitor item changes. q->connect(monitor, SIGNAL(itemAdded(Akonadi::Item,Akonadi::Collection)), SLOT(monitoredItemAdded(Akonadi::Item,Akonadi::Collection))); q->connect(monitor, SIGNAL(itemChanged(Akonadi::Item,QSet)), SLOT(monitoredItemChanged(Akonadi::Item,QSet))); q->connect(monitor, SIGNAL(itemRemoved(Akonadi::Item)), SLOT(monitoredItemRemoved(Akonadi::Item))); q->connect(monitor, SIGNAL(itemMoved(Akonadi::Item,Akonadi::Collection,Akonadi::Collection)), SLOT(monitoredItemMoved(Akonadi::Item,Akonadi::Collection,Akonadi::Collection))); q->connect(monitor, SIGNAL(itemLinked(Akonadi::Item,Akonadi::Collection)), SLOT(monitoredItemLinked(Akonadi::Item,Akonadi::Collection))); q->connect(monitor, SIGNAL(itemUnlinked(Akonadi::Item,Akonadi::Collection)), SLOT(monitoredItemUnlinked(Akonadi::Item,Akonadi::Collection))); q->connect(monitor, SIGNAL(collectionStatisticsChanged(Akonadi::Collection::Id,Akonadi::CollectionStatistics)), SLOT(monitoredCollectionStatisticsChanged(Akonadi::Collection::Id,Akonadi::CollectionStatistics))); Akonadi::ServerManager *serverManager = Akonadi::ServerManager::self(); q->connect(serverManager, SIGNAL(started()), SLOT(serverStarted())); fillModel(); } void EntityTreeModelPrivate::serverStarted() { // Don't emit about to be reset. Too late for that endResetModel(); } void EntityTreeModelPrivate::changeFetchState(const Collection &parent) { Q_Q(EntityTreeModel); const QModelIndex collectionIndex = indexForCollection(parent); if (!collectionIndex.isValid()) { // Because we are called delayed, it is possible that @p parent has been deleted. return; } Q_EMIT q->dataChanged(collectionIndex, collectionIndex); } void EntityTreeModelPrivate::agentInstanceRemoved(const Akonadi::AgentInstance &instance) { Q_Q(EntityTreeModel); if (!instance.type().capabilities().contains(QLatin1String("Resource"))) { return; } if (m_rootCollection.isValid()) { if (m_rootCollection != Collection::root()) { if (m_rootCollection.resource() == instance.identifier()) { q->clearAndReset(); } return; } foreach (Node *node, m_childEntities[Collection::root().id()]) { Q_ASSERT(node->type == Node::Collection); const Collection collection = m_collections[node->id]; if (collection.resource() == instance.identifier()) { monitoredCollectionRemoved(collection); } } } } void EntityTreeModelPrivate::fetchItems(const Collection &parent) { Q_Q(const EntityTreeModel); Q_ASSERT(parent.isValid()); Q_ASSERT(m_collections.contains(parent.id())); // TODO: Use a more specific fetch scope to get only the envelope for mails etc. ItemFetchJob *itemFetchJob = new Akonadi::ItemFetchJob(parent, m_session); itemFetchJob->setFetchScope(m_monitor->itemFetchScope()); itemFetchJob->fetchScope().setAncestorRetrieval(ItemFetchScope::All); itemFetchJob->fetchScope().setIgnoreRetrievalErrors(true); itemFetchJob->setDeliveryOption(ItemFetchJob::EmitItemsInBatches); itemFetchJob->setProperty(FetchCollectionId().constData(), QVariant(parent.id())); if (m_showRootCollection || parent != m_rootCollection) { m_pendingCollectionRetrieveJobs.insert(parent.id()); // If collections are not in the model, there will be no valid index for them. if ((m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch) && (m_collectionFetchStrategy != EntityTreeModel::FetchNoCollections)) { // We need to invoke this delayed because we would otherwise be emitting a sequence like // - beginInsertRows // - dataChanged // - endInsertRows // which would confuse proxies. QMetaObject::invokeMethod(const_cast(q), "changeFetchState", Qt::QueuedConnection, Q_ARG(Akonadi::Collection, parent)); } } q->connect(itemFetchJob, SIGNAL(itemsReceived(Akonadi::Item::List)), q, SLOT(itemsFetched(Akonadi::Item::List))); q->connect(itemFetchJob, SIGNAL(result(KJob*)), q, SLOT(itemFetchJobDone(KJob*))); qCDebug(DebugETM) << "collection:" << parent.name(); jobTimeTracker[itemFetchJob].start(); } void EntityTreeModelPrivate::fetchCollections(Akonadi::CollectionFetchJob *job) { Q_Q(EntityTreeModel); job->fetchScope().setListFilter(m_listFilter); job->fetchScope().setContentMimeTypes(m_monitor->mimeTypesMonitored()); m_pendingCollectionFetchJobs.insert(static_cast(job)); if (m_collectionFetchStrategy == EntityTreeModel::InvisibleCollectionFetch) { q->connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)), q, SLOT(collectionListFetched(Akonadi::Collection::List))); } else { job->fetchScope().setIncludeStatistics(m_includeStatistics); job->fetchScope().setAncestorRetrieval(Akonadi::CollectionFetchScope::All); q->connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)), q, SLOT(collectionsFetched(Akonadi::Collection::List))); } q->connect(job, SIGNAL(result(KJob*)), q, SLOT(collectionFetchJobDone(KJob*))); qCDebug(DebugETM) << "collection:" << job->collections(); jobTimeTracker[job].start(); } void EntityTreeModelPrivate::fetchCollections(const Collection::List &collections, CollectionFetchJob::Type type) { fetchCollections(new CollectionFetchJob(collections, type, m_session)); } void EntityTreeModelPrivate::fetchCollections(const Collection &collection, CollectionFetchJob::Type type) { Q_ASSERT(collection.isValid()); CollectionFetchJob *job = new CollectionFetchJob(collection, type, m_session); fetchCollections(job); } namespace Akonadi { template inline bool EntityTreeModelPrivate::isHiddenImpl(const T &entity, Node::Type type) const { if (m_showSystemEntities) { return false; } if (type == Node::Collection && entity.id() == m_rootCollection.id()) { return false; } // entity.hasAttribute() does not compile w/ GCC for // some reason if (entity.hasAttribute(EntityHiddenAttribute().type())) { return true; } const Collection parent = entity.parentCollection(); if (parent.isValid()) { return isHiddenImpl(parent, Node::Collection); } return false; } } bool EntityTreeModelPrivate::isHidden(const Akonadi::Collection &collection) const { return isHiddenImpl(collection, Node::Collection); } bool EntityTreeModelPrivate::isHidden(const Akonadi::Item &item) const { return isHiddenImpl(item, Node::Item); } void EntityTreeModelPrivate::collectionListFetched(const Akonadi::Collection::List &collections) { QVectorIterator it(collections); while (it.hasNext()) { const Collection collection = it.next(); if (isHidden(collection)) { continue; } m_collections.insert(collection.id(), collection); Node *node = new Node; node->id = collection.id(); node->parent = -1; node->type = Node::Collection; m_childEntities[-1].prepend(node); fetchItems(collection); } } static QSet getChildren(Collection::Id parent, const QHash &childParentMap) { QSet children; for (auto it = childParentMap.cbegin(), e = childParentMap.cend(); it != e; ++it) { if (it.value() == parent) { children << it.key(); children += getChildren(it.key(), childParentMap); } } return children; } void EntityTreeModelPrivate::collectionsFetched(const Akonadi::Collection::List &collections) { Q_Q(EntityTreeModel); - QTime t; + QElapsedTimer t; t.start(); QVectorIterator it(collections); QHash collectionsToInsert; while (it.hasNext()) { const Collection collection = it.next(); const Collection::Id collectionId = collection.id(); if (isHidden(collection)) { continue; } auto collectionIt = m_collections.find(collectionId); if (collectionIt != m_collections.end()) { // This is probably the result of a parent of a previous collection already being in the model. // Replace the dummy collection with the real one and move on. // This could also be the result of a monitor signal having already inserted the collection // into this model. There's no way to tell, so we just emit dataChanged. *collectionIt = collection; const QModelIndex collectionIndex = indexForCollection(collection); dataChanged(collectionIndex, collectionIndex); Q_EMIT q->collectionFetched(collectionId); continue; } //If we're monitoring collections somewhere in the tree we need to retrieve their ancestors now if (collection.parentCollection() != m_rootCollection && m_monitor->collectionsMonitored().contains(collection)) { retrieveAncestors(collection, false); } collectionsToInsert.insert(collectionId, collection); } //Build a list of subtrees to insert, with the root of the subtree on the left, and the complete subtree including root on the right QHash > subTreesToInsert; { //Build a child-parent map that allows us to build the subtrees afterwards QHash childParentMap; Q_FOREACH (const Collection &col, collectionsToInsert) { childParentMap.insert(col.id(), col.parentCollection().id()); //Complete the subtree up to the last known parent Collection parent = col.parentCollection(); while (parent.isValid() && parent != m_rootCollection && !m_collections.contains(parent.id())) { childParentMap.insert(parent.id(), parent.parentCollection().id()); if (!collectionsToInsert.contains(parent.id())) { collectionsToInsert.insert(parent.id(), parent); } parent = parent.parentCollection(); } } QSet parents; //Find toplevel parents of the subtrees for (auto it = childParentMap.cbegin(), e = childParentMap.cend(); it != e; ++it) { //The child has a parent without parent (it's a toplevel node that is not yet in m_collections) if (!childParentMap.contains(it.value())) { Q_ASSERT(!m_collections.contains(it.key())); parents << it.key(); } } //Find children of each subtree Q_FOREACH (Collection::Id p, parents) { QSet children; //We add the parent itself as well so it can be inserted below as part of the same loop children << p; children += getChildren(p, childParentMap); subTreesToInsert[p] = children; } } const int row = 0; QHashIterator > collectionIt(subTreesToInsert); while (collectionIt.hasNext()) { collectionIt.next(); const Collection::Id topCollectionId = collectionIt.key(); qCDebug(DebugETM) << "Subtree: " << topCollectionId << collectionIt.value(); Q_ASSERT(!m_collections.contains(topCollectionId)); Collection topCollection = collectionsToInsert.value(topCollectionId); Q_ASSERT(topCollection.isValid()); //The toplevels parent must already be part of the model Q_ASSERT(m_collections.contains(topCollection.parentCollection().id())); const QModelIndex parentIndex = indexForCollection(topCollection.parentCollection()); q->beginInsertRows(parentIndex, row, row); Q_ASSERT(!collectionIt.value().isEmpty()); foreach (Collection::Id collectionId, collectionIt.value()) { const Collection collection = collectionsToInsert.take(collectionId); Q_ASSERT(collection.isValid()); m_collections.insert(collectionId, collection); Node *node = new Node; node->id = collectionId; Q_ASSERT(collection.parentCollection().isValid()); node->parent = collection.parentCollection().id(); node->type = Node::Collection; m_childEntities[node->parent].prepend(node); } q->endInsertRows(); if (m_itemPopulation == EntityTreeModel::ImmediatePopulation) { foreach (const Collection::Id &collectionId, collectionIt.value()) { const auto col = m_collections.value(collectionId); if (!m_mimeChecker.hasWantedMimeTypes() || m_mimeChecker.isWantedCollection(col)) { fetchItems(col); } else { // Consider collections that don't contain relevant mimetypes to be populated m_populatedCols.insert(collectionId); Q_EMIT q_ptr->collectionPopulated(collectionId); const auto idx = indexForCollection(Collection(collectionId)); Q_ASSERT(idx.isValid()); dataChanged(idx, idx); } } } } } void EntityTreeModelPrivate::itemsFetched(const Akonadi::Item::List &items) { Q_Q(EntityTreeModel); const Collection::Id collectionId = q->sender()->property(FetchCollectionId().constData()).value(); itemsFetched(collectionId, items); } void EntityTreeModelPrivate::itemsFetched(const Collection::Id collectionId, const Akonadi::Item::List &items) { Q_Q(EntityTreeModel); if (!m_collections.contains(collectionId)) { qCWarning(AKONADICORE_LOG) << "Collection has been removed while fetching items"; return; } Item::List itemsToInsert; const Collection collection = m_collections.value(collectionId); Q_ASSERT(collection.isValid()); // if there are any items at all, remove from set of collections known to be empty if (!items.isEmpty()) { m_collectionsWithoutItems.remove(collectionId); } foreach (const Item &item, items) { if (isHidden(item)) { continue; } if ((!m_mimeChecker.hasWantedMimeTypes() || m_mimeChecker.isWantedItem(item))) { // When listing virtual collections we might get results for items which are already in // the model if their concrete collection has already been listed. // In that case the collectionId should be different though. // As an additional complication, new items might be both part of fetch job results and // part of monitor notifications. We only insert items which are not already in the model // considering their (possibly virtual) parent. bool isNewItem = true; auto itemIt = m_items.find(item.id()); if (itemIt != m_items.end()) { const Akonadi::Collection::List parents = getParentCollections(item); for (const Akonadi::Collection &parent : parents) { if (parent.id() == collectionId) { qCWarning(AKONADICORE_LOG) << "Fetched an item which is already in the model"; // Update it in case the revision changed; itemIt->value.apply(item); isNewItem = false; break; } } } if (isNewItem) { itemsToInsert << item; } } } if (!itemsToInsert.isEmpty()) { const Collection::Id colId = m_collectionFetchStrategy == EntityTreeModel::InvisibleCollectionFetch ? m_rootCollection.id() : m_collectionFetchStrategy == EntityTreeModel::FetchNoCollections ? m_rootCollection.id() : collectionId; const int startRow = m_childEntities.value(colId).size(); Q_ASSERT(m_collections.contains(colId)); const QModelIndex parentIndex = indexForCollection(m_collections.value(colId)); q->beginInsertRows(parentIndex, startRow, startRow + itemsToInsert.size() - 1); foreach (const Item &item, itemsToInsert) { const Item::Id itemId = item.id(); m_items.ref(itemId, item); Node *node = new Node; node->id = itemId; node->parent = collectionId; node->type = Node::Item; m_childEntities[colId].append(node); } q->endInsertRows(); } } void EntityTreeModelPrivate::monitoredMimeTypeChanged(const QString &mimeType, bool monitored) { beginResetModel(); if (monitored) { m_mimeChecker.addWantedMimeType(mimeType); } else { m_mimeChecker.removeWantedMimeType(mimeType); } endResetModel(); } void EntityTreeModelPrivate::monitoredCollectionsChanged(const Akonadi::Collection &collection, bool monitored) { if (monitored) { const CollectionFetchJob::Type fetchType = getFetchType(m_collectionFetchStrategy); fetchCollections(collection, CollectionFetchJob::Base); fetchCollections(collection, fetchType); } else { //If a collection is dereferenced and no longer explicitly monitored it might still match other filters if (!shouldBePartOfModel(collection)) { monitoredCollectionRemoved(collection); } } } void EntityTreeModelPrivate::monitoredItemsChanged(const Akonadi::Item &item, bool monitored) { Q_UNUSED(item) Q_UNUSED(monitored) beginResetModel(); endResetModel(); } void EntityTreeModelPrivate::monitoredResourcesChanged(const QByteArray &resource, bool monitored) { Q_UNUSED(resource) Q_UNUSED(monitored) beginResetModel(); endResetModel(); } void EntityTreeModelPrivate::retrieveAncestors(const Akonadi::Collection &collection, bool insertBaseCollection) { Q_Q(EntityTreeModel); Collection parentCollection = collection.parentCollection(); Q_ASSERT(parentCollection.isValid()); Q_ASSERT(parentCollection != Collection::root()); Collection::List ancestors; while (parentCollection != Collection::root() && !m_collections.contains(parentCollection.id())) { // Put a temporary node in the tree later. ancestors.prepend(parentCollection); parentCollection = parentCollection.parentCollection(); } Q_ASSERT(parentCollection.isValid()); // if m_rootCollection is Collection::root(), we always have common ancestor and do the retrieval // if we traversed up to Collection::root() but are looking at a subtree only (m_rootCollection != Collection::root()) // we have no common ancestor, and we don't have to retrieve anything if (parentCollection == Collection::root() && m_rootCollection != Collection::root()) { return; } if (ancestors.isEmpty() && !insertBaseCollection) { //Nothing to do, avoid emitting insert signals return; } if (!ancestors.isEmpty()) { // Fetch the real ancestors CollectionFetchJob *job = new CollectionFetchJob(ancestors, CollectionFetchJob::Base, m_session); job->fetchScope().setListFilter(m_listFilter); job->fetchScope().setIncludeStatistics(m_includeStatistics); q->connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)), q, SLOT(ancestorsFetched(Akonadi::Collection::List))); q->connect(job, SIGNAL(result(KJob*)), q, SLOT(collectionFetchJobDone(KJob*))); } // Q_ASSERT( parentCollection != m_rootCollection ); const QModelIndex parent = indexForCollection(parentCollection); // Still prepending all collections for now. int row = 0; // Although we insert several Collections here, we only need to notify though the model // about the top-level one. The rest will be found automatically by the view. q->beginInsertRows(parent, row, row); Collection::List::const_iterator it = ancestors.constBegin(); const Collection::List::const_iterator end = ancestors.constEnd(); for (; it != end; ++it) { const Collection ancestor = *it; Q_ASSERT(ancestor.parentCollection().isValid()); m_collections.insert(ancestor.id(), ancestor); Node *node = new Node; node->id = ancestor.id(); node->parent = ancestor.parentCollection().id(); node->type = Node::Collection; m_childEntities[node->parent].prepend(node); } if (insertBaseCollection) { m_collections.insert(collection.id(), collection); Node *node = new Node; node->id = collection.id(); // Can't just use parentCollection because that doesn't necessarily refer to collection. node->parent = collection.parentCollection().id(); node->type = Node::Collection; m_childEntities[node->parent].prepend(node); } q->endInsertRows(); } void EntityTreeModelPrivate::ancestorsFetched(const Akonadi::Collection::List &collectionList) { for (const Collection &collection : collectionList) { m_collections[collection.id()] = collection; const QModelIndex index = indexForCollection(collection); Q_ASSERT(index.isValid()); dataChanged(index, index); } } void EntityTreeModelPrivate::insertCollection(const Akonadi::Collection &collection, const Akonadi::Collection &parent) { Q_ASSERT(collection.isValid()); Q_ASSERT(parent.isValid()); Q_Q(EntityTreeModel); const int row = 0; const QModelIndex parentIndex = indexForCollection(parent); q->beginInsertRows(parentIndex, row, row); m_collections.insert(collection.id(), collection); Node *node = new Node; node->id = collection.id(); node->parent = parent.id(); node->type = Node::Collection; m_childEntities[parent.id()].prepend(node); q->endInsertRows(); } bool EntityTreeModelPrivate::hasChildCollection(const Collection &collection) const { foreach (Node *node, m_childEntities[collection.id()]) { if (node->type == Node::Collection) { const Collection subcol = m_collections[node->id]; if (shouldBePartOfModel(subcol)) { return true; } } } return false; } bool EntityTreeModelPrivate::isAncestorMonitored(const Collection &collection) const { Akonadi::Collection parent = collection.parentCollection(); while (parent.isValid()) { if (m_monitor->collectionsMonitored().contains(parent)) { return true; } parent = parent.parentCollection(); } return false; } bool EntityTreeModelPrivate::shouldBePartOfModel(const Collection &collection) const { if (isHidden(collection)) { return false; } // We want a parent collection if it has at least one child that matches the // wanted mimetype if (hasChildCollection(collection)) { return true; } //Explicitly monitored collection if (m_monitor->collectionsMonitored().contains(collection)) { return true; } //We're explicitly monitoring collections, but didn't match the filter if (!m_mimeChecker.hasWantedMimeTypes() && !m_monitor->collectionsMonitored().isEmpty()) { //The collection should be included if one of the parents is monitored if (isAncestorMonitored(collection)) { return true; } return false; } // Some collection trees contain multiple mimetypes. Even though server side filtering ensures we // only get the ones we're interested in from the job, we have to filter on collections received through signals too. if (m_mimeChecker.hasWantedMimeTypes() && !m_mimeChecker.isWantedCollection(collection)) { return false; } if (m_listFilter == CollectionFetchScope::Enabled) { if (!collection.enabled()) { return false; } } else if (m_listFilter == CollectionFetchScope::Display) { if (!collection.shouldList(Collection::ListDisplay)) { return false; } } else if (m_listFilter == CollectionFetchScope::Sync) { if (!collection.shouldList(Collection::ListSync)) { return false; } } else if (m_listFilter == CollectionFetchScope::Index) { if (!collection.shouldList(Collection::ListIndex)) { return false; } } return true; } void EntityTreeModelPrivate::monitoredCollectionAdded(const Akonadi::Collection &collection, const Akonadi::Collection &parent) { // If the resource is removed while populating the model with it, we might still // get some monitor signals. These stale/out-of-order signals can't be completely eliminated // in the akonadi server due to implementation details, so we also handle such signals in the model silently // in all the monitored slots. // Stephen Kelly, 28, July 2009 // If a fetch job is started and a collection is added to akonadi after the fetch job is started, the // new collection will be added to the fetch job results. It will also be notified through the monitor. // We return early here in that case. if (m_collections.contains(collection.id())) { return; } //If the resource is explicitly monitored all other checks are skipped. topLevelCollectionsFetched still checks the hidden attribute. if (m_monitor->resourcesMonitored().contains(collection.resource().toUtf8()) && collection.parentCollection() == Collection::root()) { return topLevelCollectionsFetched(Collection::List() << collection); } if (!shouldBePartOfModel(collection)) { return; } if (!m_collections.contains(parent.id())) { // The collection we're interested in is contained in a collection we're not interested in. // We download the ancestors of the collection we're interested in to complete the tree. if (collection != Collection::root()) { retrieveAncestors(collection); } if (m_itemPopulation == EntityTreeModel::ImmediatePopulation) { fetchItems(collection); } return; } insertCollection(collection, parent); if (m_itemPopulation == EntityTreeModel::ImmediatePopulation) { fetchItems(collection); } } void EntityTreeModelPrivate::monitoredCollectionRemoved(const Akonadi::Collection &collection) { //if an explicitly monitored collection is removed, we would also have to remove collections which were included to show it (as in the move case) if ((collection == m_rootCollection) || m_monitor->collectionsMonitored().contains(collection)) { beginResetModel(); endResetModel(); return; } Collection::Id parentId = collection.parentCollection().id(); if (parentId < 0) { parentId = -1; } if (!m_collections.contains(parentId)) { return; } // This may be a signal for a collection we've already removed by removing its ancestor. // Or the collection may have been hidden. if (!m_collections.contains(collection.id())) { return; } Q_Q(EntityTreeModel); Q_ASSERT(m_childEntities.contains(parentId)); const int row = indexOf(m_childEntities.value(parentId), collection.id()); Q_ASSERT(row >= 0); Q_ASSERT(m_collections.contains(parentId)); const Collection parentCollection = m_collections.value(parentId); m_populatedCols.remove(collection.id()); const QModelIndex parentIndex = indexForCollection(parentCollection); q->beginRemoveRows(parentIndex, row, row); // Delete all descendant collections and items. removeChildEntities(collection.id()); // Remove deleted collection from its parent. delete m_childEntities[parentId].takeAt(row); // Remove deleted collection itself. m_collections.remove(collection.id()); q->endRemoveRows(); // After removing a collection, check whether it's parent should be removed too if (!shouldBePartOfModel(parentCollection)) { monitoredCollectionRemoved(parentCollection); } } void EntityTreeModelPrivate::removeChildEntities(Collection::Id collectionId) { QList childList = m_childEntities.value(collectionId); QList::const_iterator it = childList.constBegin(); const QList::const_iterator end = childList.constEnd(); for (; it != end; ++it) { if (Node::Item == (*it)->type) { m_items.unref((*it)->id); } else { removeChildEntities((*it)->id); m_collections.remove((*it)->id); m_populatedCols.remove((*it)->id); } } qDeleteAll(m_childEntities.take(collectionId)); } QStringList EntityTreeModelPrivate::childCollectionNames(const Collection &collection) const { QStringList names; foreach (Node *node, m_childEntities[collection.id()]) { if (node->type == Node::Collection) { names << m_collections.value(node->id).name(); } } return names; } void EntityTreeModelPrivate::monitoredCollectionMoved(const Akonadi::Collection &collection, const Akonadi::Collection &sourceCollection, const Akonadi::Collection &destCollection) { if (isHidden(collection)) { return; } if (isHidden(sourceCollection)) { if (isHidden(destCollection)) { return; } monitoredCollectionAdded(collection, destCollection); return; } else if (isHidden(destCollection)) { monitoredCollectionRemoved(collection); return; } if (!m_collections.contains(collection.id())) { return; } if (m_monitor->collectionsMonitored().contains(collection)) { //if we don't reset here, we would have to make sure that destination collection is actually available, //and remove the sources parents if they were only included as parents of the moved collection beginResetModel(); endResetModel(); return; } Q_Q(EntityTreeModel); const QModelIndex srcParentIndex = indexForCollection(sourceCollection); const QModelIndex destParentIndex = indexForCollection(destCollection); Q_ASSERT(collection.parentCollection().isValid()); Q_ASSERT(destCollection.isValid()); Q_ASSERT(collection.parentCollection() == destCollection); const int srcRow = indexOf(m_childEntities.value(sourceCollection.id()), collection.id()); const int destRow = 0; // Prepend collections if (!q->beginMoveRows(srcParentIndex, srcRow, srcRow, destParentIndex, destRow)) { qCWarning(AKONADICORE_LOG) << "Invalid move"; return; } Node *node = m_childEntities[sourceCollection.id()].takeAt(srcRow); // collection has the correct parentCollection etc. We need to set it on the // internal data structure to not corrupt things. m_collections.insert(collection.id(), collection); node->parent = destCollection.id(); m_childEntities[destCollection.id()].prepend(node); q->endMoveRows(); } void EntityTreeModelPrivate::monitoredCollectionChanged(const Akonadi::Collection &collection) { if (!m_collections.contains(collection.id())) { // This can happen if // * we get a change notification after removing the collection. // * a collection of a non-monitored mimetype is changed elsewhere. Monitor does not // filter by content mimetype of Collections so we get notifications for all of them. //We might match the filter now, retry adding the collection monitoredCollectionAdded(collection, collection.parentCollection()); return; } if (!shouldBePartOfModel(collection)) { monitoredCollectionRemoved(collection); return; } m_collections[collection.id()] = collection; if (!m_showRootCollection && collection == m_rootCollection) { // If the root of the model is not Collection::root it might be modified. // But it doesn't exist in the accessible model structure, so we need to early return return; } const QModelIndex index = indexForCollection(collection); Q_ASSERT(index.isValid()); dataChanged(index, index); } void EntityTreeModelPrivate::monitoredCollectionStatisticsChanged(Akonadi::Collection::Id id, const Akonadi::CollectionStatistics &statistics) { if (!m_collections.contains(id)) { return; } m_collections[id].setStatistics(statistics); // if the item count becomes 0, add to set of collections we know to be empty // otherwise remove if in there if (statistics.count() == 0) { m_collectionsWithoutItems.insert(id); } else { m_collectionsWithoutItems.remove(id); } if (!m_showRootCollection && id == m_rootCollection.id()) { // If the root of the model is not Collection::root it might be modified. // But it doesn't exist in the accessible model structure, so we need to early return return; } const QModelIndex index = indexForCollection(m_collections[id]); dataChanged(index, index); } void EntityTreeModelPrivate::monitoredItemAdded(const Akonadi::Item &item, const Akonadi::Collection &collection) { Q_Q(EntityTreeModel); if (isHidden(item)) { return; } if (m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch && !m_collections.contains(collection.id())) { qCWarning(AKONADICORE_LOG) << "Got a stale 'added' notification for an item whose collection was already removed." << item.id() << item.remoteId(); return; } if (m_items.contains(item.id())) { return; } Q_ASSERT(m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch ? m_collections.contains(collection.id()) : true); if (m_mimeChecker.hasWantedMimeTypes() && !m_mimeChecker.isWantedItem(item)) { return; } //Adding items to not yet populated collections would block fetchMore, resulting in only new items showing up in the collection //This is only a problem with lazy population, otherwise fetchMore is not used at all if ((m_itemPopulation == EntityTreeModel::LazyPopulation) && !m_populatedCols.contains(collection.id())) { return; } int row; QModelIndex parentIndex; if (m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch) { row = m_childEntities.value(collection.id()).size(); parentIndex = indexForCollection(m_collections.value(collection.id())); } else { row = q->rowCount(); } q->beginInsertRows(parentIndex, row, row); m_items.ref(item.id(), item); Node *node = new Node; node->id = item.id(); node->parent = collection.id(); node->type = Node::Item; if (m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch) { m_childEntities[collection.id()].append(node); } else { m_childEntities[m_rootCollection.id()].append(node); } q->endInsertRows(); } void EntityTreeModelPrivate::monitoredItemRemoved(const Akonadi::Item &item, const Akonadi::Collection &parentCollection) { Q_Q(EntityTreeModel); if (isHidden(item)) { return; } if ((m_itemPopulation == EntityTreeModel::LazyPopulation) && !m_populatedCols.contains(parentCollection.isValid() ? parentCollection.id() : item.parentCollection().id())) { return; } const Collection::List parents = getParentCollections(item); if (parents.isEmpty()) { return; } if (!m_items.contains(item.id())) { qCWarning(AKONADICORE_LOG) << "Got a stale 'removed' notification for an item which was already removed." << item.id() << item.remoteId(); return; } for (const auto &collection : parents) { Q_ASSERT(m_collections.contains(collection.id())); Q_ASSERT(m_childEntities.contains(collection.id())); const int row = indexOf(m_childEntities.value(collection.id()), item.id()); Q_ASSERT(row >= 0); const QModelIndex parentIndex = indexForCollection(m_collections.value(collection.id())); q->beginRemoveRows(parentIndex, row, row); m_items.unref(item.id()); delete m_childEntities[collection.id()].takeAt(row); q->endRemoveRows(); } } void EntityTreeModelPrivate::monitoredItemChanged(const Akonadi::Item &item, const QSet &) { if (isHidden(item)) { return; } if ((m_itemPopulation == EntityTreeModel::LazyPopulation) && !m_populatedCols.contains(item.parentCollection().id())) { return; } auto itemIt = m_items.find(item.id()); if (itemIt == m_items.end()) { qCWarning(AKONADICORE_LOG) << "Got a stale 'changed' notification for an item which was already removed." << item.id() << item.remoteId(); return; } itemIt->value.apply(item); // Notifications about itemChange are always dispatched for real collection // and also all virtual collections the item belongs to. In order to preserve // the original storage collection when we need to have special handling for // notifications for virtual collections if (item.parentCollection().isVirtual()) { const Collection originalParent = itemIt->value.parentCollection(); itemIt->value.setParentCollection(originalParent); } const QModelIndexList indexes = indexesForItem(item); for (const QModelIndex &index : indexes) { if (index.isValid()) { dataChanged(index, index); } else { qCWarning(AKONADICORE_LOG) << "item has invalid index:" << item.id() << item.remoteId(); } } } void EntityTreeModelPrivate::monitoredItemMoved(const Akonadi::Item &item, const Akonadi::Collection &sourceCollection, const Akonadi::Collection &destCollection) { if (isHidden(item)) { return; } if (isHidden(sourceCollection)) { if (isHidden(destCollection)) { return; } monitoredItemAdded(item, destCollection); return; } else if (isHidden(destCollection)) { monitoredItemRemoved(item, sourceCollection); return; } else { monitoredItemRemoved(item, sourceCollection); monitoredItemAdded(item, destCollection); return; } // "Temporarily" commented out as it's likely the best course to // avoid the dreaded "reset storm" (or layoutChanged storm). The // whole itemMoved idea is great but not practical until all the // other proxy models play nicely with it, right now they just // transform moved signals in layout changed, which explodes into // a reset of the source model inside of the message list (ouch!) #if 0 if (!m_items.contains(item.id())) { qCWarning(AKONADICORE_LOG) << "Got a stale 'moved' notification for an item which was already removed." << item.id() << item.remoteId(); return; } Q_ASSERT(m_collections.contains(sourceCollection.id())); Q_ASSERT(m_collections.contains(destCollection.id())); const QModelIndex srcIndex = indexForCollection(sourceCollection); const QModelIndex destIndex = indexForCollection(destCollection); // Where should it go? Always append items and prepend collections and reorganize them with separate reactions to Attributes? const Item::Id itemId = item.id(); const int srcRow = indexOf(m_childEntities.value(sourceCollection.id()), itemId); const int destRow = q->rowCount(destIndex); Q_ASSERT(srcRow >= 0); Q_ASSERT(destRow >= 0); if (!q->beginMoveRows(srcIndex, srcRow, srcRow, destIndex, destRow)) { qCWarning(AKONADICORE_LOG) << "Invalid move"; return; } Q_ASSERT(m_childEntities.contains(sourceCollection.id())); Q_ASSERT(m_childEntities[sourceCollection.id()].size() > srcRow); Node *node = m_childEntities[sourceCollection.id()].takeAt(srcRow); m_items.insert(item.id(), item); node->parent = destCollection.id(); m_childEntities[destCollection.id()].append(node); q->endMoveRows(); #endif } void EntityTreeModelPrivate::monitoredItemLinked(const Akonadi::Item &item, const Akonadi::Collection &collection) { Q_Q(EntityTreeModel); if (isHidden(item)) { return; } const Collection::Id collectionId = collection.id(); const Item::Id itemId = item.id(); if (m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch && !m_collections.contains(collection.id())) { qCWarning(AKONADICORE_LOG) << "Got a stale 'linked' notification for an item whose collection was already removed." << item.id() << item.remoteId(); return; } Q_ASSERT(m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch ? m_collections.contains(collectionId) : true); if (m_mimeChecker.hasWantedMimeTypes() && !m_mimeChecker.isWantedItem(item)) { return; } //Adding items to not yet populated collections would block fetchMore, resullting in only new items showing up in the collection //This is only a problem with lazy population, otherwise fetchMore is not used at all if ((m_itemPopulation == EntityTreeModel::LazyPopulation) && !m_populatedCols.contains(collectionId)) { return; } QList &collectionEntities = m_childEntities[collectionId]; const int existingPosition = indexOf(collectionEntities, itemId); if (existingPosition > 0) { qCWarning(AKONADICORE_LOG) << "Item with id " << itemId << " already in virtual collection with id " << collectionId; return; } const int row = collectionEntities.size(); const QModelIndex parentIndex = indexForCollection(m_collections.value(collectionId)); q->beginInsertRows(parentIndex, row, row); m_items.ref(itemId, item); Node *node = new Node; node->id = itemId; node->parent = collectionId; node->type = Node::Item; collectionEntities.append(node); q->endInsertRows(); } void EntityTreeModelPrivate::monitoredItemUnlinked(const Akonadi::Item &item, const Akonadi::Collection &collection) { Q_Q(EntityTreeModel); if (isHidden(item)) { return; } if ((m_itemPopulation == EntityTreeModel::LazyPopulation) && !m_populatedCols.contains(item.parentCollection().id())) { return; } if (!m_items.contains(item.id())) { qCWarning(AKONADICORE_LOG) << "Got a stale 'unlinked' notification for an item which was already removed." << item.id() << item.remoteId(); return; } Q_ASSERT(m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch ? m_collections.contains(collection.id()) : true); const int row = indexOf(m_childEntities.value(collection.id()), item.id()); if (row < 0 || row >= m_childEntities[ collection.id() ].size()) { qCWarning(AKONADICORE_LOG) << "couldn't find index of unlinked item " << item.id() << collection.id() << row; Q_ASSERT(false); return; } const QModelIndex parentIndex = indexForCollection(m_collections.value(collection.id())); q->beginRemoveRows(parentIndex, row, row); delete m_childEntities[collection.id()].takeAt(row); m_items.unref(item.id()); q->endRemoveRows(); } void EntityTreeModelPrivate::collectionFetchJobDone(KJob *job) { m_pendingCollectionFetchJobs.remove(job); CollectionFetchJob *cJob = static_cast(job); if (job->error()) { qCWarning(AKONADICORE_LOG) << "Job error: " << job->errorString() << "for collection:" << cJob->collections() << endl; return; } if (!m_collectionTreeFetched && m_pendingCollectionFetchJobs.isEmpty()) { m_collectionTreeFetched = true; Q_EMIT q_ptr->collectionTreeFetched(m_collections | Views::values | Actions::toQVector); } qCDebug(DebugETM) << "Fetch job took " << jobTimeTracker.take(job).elapsed() << "msec"; qCDebug(DebugETM) << "was collection fetch job: collections:" << cJob->collections().size(); if (!cJob->collections().isEmpty()) { qCDebug(DebugETM) << "first fetched collection:" << cJob->collections().at(0).name(); } } void EntityTreeModelPrivate::itemFetchJobDone(KJob *job) { const Collection::Id collectionId = job->property(FetchCollectionId().constData()).value(); m_pendingCollectionRetrieveJobs.remove(collectionId); if (job->error()) { qCWarning(AKONADICORE_LOG) << "Job error: " << job->errorString() << "for collection:" << collectionId << endl; return; } if (!m_collections.contains(collectionId)) { qCWarning(AKONADICORE_LOG) << "Collection has been removed while fetching items"; return; } ItemFetchJob *iJob = static_cast(job); qCDebug(DebugETM) << "Fetch job took " << jobTimeTracker.take(job).elapsed() << "msec"; qCDebug(DebugETM) << "was item fetch job: items:" << iJob->count(); if (!iJob->count()) { m_collectionsWithoutItems.insert(collectionId); } else { m_collectionsWithoutItems.remove(collectionId); } m_populatedCols.insert(collectionId); Q_EMIT q_ptr->collectionPopulated(collectionId); // If collections are not in the model, there will be no valid index for them. if ((m_collectionFetchStrategy != EntityTreeModel::InvisibleCollectionFetch) && (m_collectionFetchStrategy != EntityTreeModel::FetchNoCollections) && !(!m_showRootCollection && collectionId == m_rootCollection.id())) { const QModelIndex index = indexForCollection(Collection(collectionId)); Q_ASSERT(index.isValid()); //To notify about the changed fetch and population state dataChanged(index, index); } } void EntityTreeModelPrivate::pasteJobDone(KJob *job) { if (job->error()) { QString errorMsg; if (qobject_cast(job)) { errorMsg = i18n("Could not copy item:"); } else if (qobject_cast(job)) { errorMsg = i18n("Could not copy collection:"); } else if (qobject_cast(job)) { errorMsg = i18n("Could not move item:"); } else if (qobject_cast(job)) { errorMsg = i18n("Could not move collection:"); } else if (qobject_cast(job)) { errorMsg = i18n("Could not link entity:"); } errorMsg += QLatin1Char(' ') + job->errorString(); QMessageBox::critical(nullptr, i18n("Error"), errorMsg); } } void EntityTreeModelPrivate::updateJobDone(KJob *job) { if (job->error()) { // TODO: handle job errors qCWarning(AKONADICORE_LOG) << "Job error:" << job->errorString(); } } void EntityTreeModelPrivate::rootFetchJobDone(KJob *job) { if (job->error()) { qCWarning(AKONADICORE_LOG) << job->errorString(); return; } CollectionFetchJob *collectionJob = qobject_cast(job); const Collection::List list = collectionJob->collections(); Q_ASSERT(list.size() == 1); m_rootCollection = list.first(); startFirstListJob(); } void EntityTreeModelPrivate::startFirstListJob() { Q_Q(EntityTreeModel); if (!m_collections.isEmpty()) { return; } // Even if the root collection is the invalid collection, we still need to start // the first list job with Collection::root. if (m_showRootCollection) { // Notify the outside that we're putting collection::root into the model. q->beginInsertRows(QModelIndex(), 0, 0); m_collections.insert(m_rootCollection.id(), m_rootCollection); delete m_rootNode; m_rootNode = new Node; m_rootNode->id = m_rootCollection.id(); m_rootNode->parent = -1; m_rootNode->type = Node::Collection; m_childEntities[-1].append(m_rootNode); q->endInsertRows(); } else { // Otherwise store it silently because it's not part of the usable model. delete m_rootNode; m_rootNode = new Node; m_needDeleteRootNode = true; m_rootNode->id = m_rootCollection.id(); m_rootNode->parent = -1; m_rootNode->type = Node::Collection; m_collections.insert(m_rootCollection.id(), m_rootCollection); } const bool noMimetypes = !m_mimeChecker.hasWantedMimeTypes(); const bool noResources = m_monitor->resourcesMonitored().isEmpty(); const bool multipleCollections = m_monitor->collectionsMonitored().size() > 1; const bool generalPopulation = !noMimetypes || noResources; const CollectionFetchJob::Type fetchType = getFetchType(m_collectionFetchStrategy); //Collections can only be monitored if no resources and no mimetypes are monitored if (multipleCollections && noMimetypes && noResources) { fetchCollections(m_monitor->collectionsMonitored(), CollectionFetchJob::Base); fetchCollections(m_monitor->collectionsMonitored(), fetchType); return; } qCDebug(DebugETM) << "GEN" << generalPopulation << noMimetypes << noResources; if (generalPopulation) { fetchCollections(m_rootCollection, fetchType); } // If the root collection is not collection::root, then it could have items, and they will need to be // retrieved now. // Only fetch items NOT if there is NoItemPopulation, or if there is Lazypopulation and the root is visible // (if the root is not visible the lazy population can not be triggered) if ((m_itemPopulation != EntityTreeModel::NoItemPopulation) && !((m_itemPopulation == EntityTreeModel::LazyPopulation) && m_showRootCollection)) { if (m_rootCollection != Collection::root()) { fetchItems(m_rootCollection); } } // Resources which are explicitly monitored won't have appeared yet if their mimetype didn't match. // We fetch the top level collections and examine them for whether to add them. // This fetches virtual collections into the tree. if (!m_monitor->resourcesMonitored().isEmpty()) { fetchTopLevelCollections(); } } void EntityTreeModelPrivate::fetchTopLevelCollections() const { Q_Q(const EntityTreeModel); CollectionFetchJob *job = new CollectionFetchJob(Collection::root(), CollectionFetchJob::FirstLevel, m_session); q->connect(job, SIGNAL(collectionsReceived(Akonadi::Collection::List)), q, SLOT(topLevelCollectionsFetched(Akonadi::Collection::List))); q->connect(job, SIGNAL(result(KJob*)), q, SLOT(collectionFetchJobDone(KJob*))); qCDebug(DebugETM) << "EntityTreeModelPrivate::fetchTopLevelCollections"; jobTimeTracker[job].start(); } void EntityTreeModelPrivate::topLevelCollectionsFetched(const Akonadi::Collection::List &list) { Q_Q(EntityTreeModel); for (const Collection &collection : list) { // These collections have been explicitly shown in the Monitor, // but hidden trumps that for now. This may change in the future if we figure out a use for it. if (isHidden(collection)) { continue; } if (m_monitor->resourcesMonitored().contains(collection.resource().toUtf8()) && !m_collections.contains(collection.id())) { const QModelIndex parentIndex = indexForCollection(collection.parentCollection()); // Prepending new collections. const int row = 0; q->beginInsertRows(parentIndex, row, row); m_collections.insert(collection.id(), collection); Node *node = new Node; node->id = collection.id(); Q_ASSERT(collection.parentCollection() == Collection::root()); node->parent = collection.parentCollection().id(); node->type = Node::Collection; m_childEntities[collection.parentCollection().id()].prepend(node); q->endInsertRows(); if (m_itemPopulation == EntityTreeModel::ImmediatePopulation) { fetchItems(collection); } Q_ASSERT(collection.isValid()); fetchCollections(collection, CollectionFetchJob::Recursive); } } } Akonadi::Collection::List EntityTreeModelPrivate::getParentCollections(const Item &item) const { Collection::List list; QHashIterator > iter(m_childEntities); while (iter.hasNext()) { iter.next(); int nodeIndex = indexOf(iter.value(), item.id()); if (nodeIndex != -1 && iter.value().at(nodeIndex)->type == Node::Item) { list << m_collections.value(iter.key()); } } return list; } void EntityTreeModelPrivate::ref(Collection::Id id) { m_monitor->d_ptr->ref(id); } bool EntityTreeModelPrivate::shouldPurge(Collection::Id id) { // reference counted collections should never be purged // they first have to be deref'ed until they reach 0. // if the collection is buffered, keep it. if (m_monitor->d_ptr->isMonitored(id)) { return false; } // otherwise we can safely purge this item return true; } bool EntityTreeModelPrivate::isMonitored(Collection::Id id) { return m_monitor->d_ptr->isMonitored(id); } bool EntityTreeModelPrivate::isBuffered(Collection::Id id) { return m_monitor->d_ptr->m_buffer.isBuffered(id); } void EntityTreeModelPrivate::deref(Collection::Id id) { const Collection::Id bumpedId = m_monitor->d_ptr->deref(id); if (bumpedId < 0) { return; } //The collection has already been removed, don't purge if (!m_collections.contains(bumpedId)) { return; } if (shouldPurge(bumpedId)) { purgeItems(bumpedId); } } QList::iterator EntityTreeModelPrivate::skipCollections(QList::iterator it, QList::iterator end, int *pos) { for (; it != end; ++it) { if ((*it)->type == Node::Item) { break; } ++(*pos); } return it; } QList::iterator EntityTreeModelPrivate::removeItems(QList::iterator it, QList::iterator end, int *pos, const Collection &collection) { Q_Q(EntityTreeModel); QList::iterator startIt = it; // figure out how many items we will delete int start = *pos; for (; it != end; ++it) { if ((*it)->type != Node::Item) { break; } ++(*pos); } it = startIt; const QModelIndex parentIndex = indexForCollection(collection); q->beginRemoveRows(parentIndex, start, (*pos) - 1); const int toDelete = (*pos) - start; Q_ASSERT(toDelete > 0); QList &es = m_childEntities[collection.id()]; //NOTE: .erase will invalidate all iterators besides "it"! for (int i = 0; i < toDelete; ++i) { Q_ASSERT(es.count(*it) == 1); // don't keep implicitly shared data alive Q_ASSERT(m_items.contains((*it)->id)); m_items.unref((*it)->id); // delete actual node delete *it; it = es.erase(it); } q->endRemoveRows(); return it; } void EntityTreeModelPrivate::purgeItems(Collection::Id id) { QList &childEntities = m_childEntities[id]; const Collection collection = m_collections.value(id); Q_ASSERT(collection.isValid()); QList::iterator begin = childEntities.begin(); QList::iterator end = childEntities.end(); int pos = 0; while ((begin = skipCollections(begin, end, &pos)) != end) { begin = removeItems(begin, end, &pos, collection); end = childEntities.end(); } m_populatedCols.remove(id); //if an empty collection is purged and we leave it in here, itemAdded will be ignored for the collection //and the collection is never populated by fetchMore (but maybe by statistics changed?) m_collectionsWithoutItems.remove(id); } void EntityTreeModelPrivate::dataChanged(const QModelIndex &top, const QModelIndex &bottom) { Q_Q(EntityTreeModel); QModelIndex rightIndex; Node *node = static_cast(bottom.internalPointer()); if (!node) { return; } if (node->type == Node::Collection) { rightIndex = bottom.sibling(bottom.row(), q->entityColumnCount(EntityTreeModel::CollectionTreeHeaders) - 1); } if (node->type == Node::Item) { rightIndex = bottom.sibling(bottom.row(), q->entityColumnCount(EntityTreeModel::ItemListHeaders) - 1); } Q_EMIT q->dataChanged(top, rightIndex); } QModelIndex EntityTreeModelPrivate::indexForCollection(const Collection &collection) const { Q_Q(const EntityTreeModel); if (!collection.isValid()) { return QModelIndex(); } if (m_collectionFetchStrategy == EntityTreeModel::InvisibleCollectionFetch) { return QModelIndex(); } // The id of the parent of Collection::root is not guaranteed to be -1 as assumed by startFirstListJob, // we ensure that we use -1 for the invalid Collection. Collection::Id parentId = -1; if ((collection == m_rootCollection)) { if (m_showRootCollection) { return q->createIndex(0, 0, static_cast(m_rootNode)); } return QModelIndex(); } if (collection == Collection::root()) { parentId = -1; } else if (collection.parentCollection().isValid()) { parentId = collection.parentCollection().id(); } else { QHash >::const_iterator it = m_childEntities.constBegin(); const QHash >::const_iterator end = m_childEntities.constEnd(); for (; it != end; ++it) { const int row = indexOf(it.value(), collection.id()); if (row < 0) { continue; } Node *node = it.value().at(row); return q->createIndex(row, 0, static_cast(node)); } return QModelIndex(); } const int row = indexOf(m_childEntities.value(parentId), collection.id()); if (row < 0) { return QModelIndex(); } Node *node = m_childEntities.value(parentId).at(row); return q->createIndex(row, 0, static_cast(node)); } QModelIndexList EntityTreeModelPrivate::indexesForItem(const Item &item) const { Q_Q(const EntityTreeModel); QModelIndexList indexes; if (m_collectionFetchStrategy == EntityTreeModel::FetchNoCollections) { Q_ASSERT(m_childEntities.contains(m_rootCollection.id())); QList nodeList = m_childEntities.value(m_rootCollection.id()); const int row = indexOf(nodeList, item.id()); Q_ASSERT(row >= 0); Q_ASSERT(row < nodeList.size()); Node *node = nodeList.at(row); indexes << q->createIndex(row, 0, static_cast(node)); return indexes; } const Collection::List collections = getParentCollections(item); indexes.reserve(collections.size()); for (const Collection &collection : collections) { const int row = indexOf(m_childEntities.value(collection.id()), item.id()); Q_ASSERT(row >= 0); Q_ASSERT(m_childEntities.contains(collection.id())); QList nodeList = m_childEntities.value(collection.id()); Q_ASSERT(row < nodeList.size()); Node *node = nodeList.at(row); indexes << q->createIndex(row, 0, static_cast(node)); } return indexes; } void EntityTreeModelPrivate::beginResetModel() { Q_Q(EntityTreeModel); q->beginResetModel(); } void EntityTreeModelPrivate::endResetModel() { Q_Q(EntityTreeModel); foreach (Akonadi::Job *job, m_session->findChildren()) { job->disconnect(q); } m_collections.clear(); m_collectionsWithoutItems.clear(); m_populatedCols.clear(); m_items.clear(); m_pendingCollectionFetchJobs.clear(); m_pendingCollectionRetrieveJobs.clear(); m_collectionTreeFetched = false; foreach (const QList &list, m_childEntities) { qDeleteAll(list); } m_childEntities.clear(); if (m_needDeleteRootNode) { m_needDeleteRootNode = false; delete m_rootNode; } m_rootNode = nullptr; q->endResetModel(); fillModel(); } void EntityTreeModelPrivate::monitoredItemsRetrieved(KJob *job) { if (job->error()) { qCWarning(AKONADICORE_LOG) << job->errorString(); return; } Q_Q(EntityTreeModel); ItemFetchJob *fetchJob = qobject_cast(job); Q_ASSERT(fetchJob); Item::List list = fetchJob->items(); q->beginResetModel(); foreach (const Item &item, list) { Node *node = new Node; node->id = item.id(); node->parent = m_rootCollection.id(); node->type = Node::Item; m_childEntities[-1].append(node); m_items.ref(item.id(), item); } q->endResetModel(); } void EntityTreeModelPrivate::fillModel() { Q_Q(EntityTreeModel); m_mimeChecker.setWantedMimeTypes(m_monitor->mimeTypesMonitored()); const Collection::List collections = m_monitor->collectionsMonitored(); if (collections.isEmpty() && m_monitor->numMimeTypesMonitored() == 0 && m_monitor->numResourcesMonitored() == 0 && m_monitor->numItemsMonitored() != 0) { m_rootCollection = Collection(-1); m_collectionTreeFetched = true; Q_EMIT q_ptr->collectionTreeFetched(collections); // there are no collections to fetch Item::List items; items.reserve(m_monitor->itemsMonitoredEx().size()); Q_FOREACH (Item::Id id, m_monitor->itemsMonitoredEx()) { items.append(Item(id)); } ItemFetchJob *itemFetch = new ItemFetchJob(items, m_session); itemFetch->setFetchScope(m_monitor->itemFetchScope()); itemFetch->fetchScope().setIgnoreRetrievalErrors(true); q->connect(itemFetch, SIGNAL(finished(KJob*)), q, SLOT(monitoredItemsRetrieved(KJob*))); return; } // In case there is only a single collection monitored, we can use this // collection as root of the node tree, in all other cases // Collection::root() is used if (collections.size() == 1) { m_rootCollection = collections.first(); } else { m_rootCollection = Collection::root(); } if (m_rootCollection == Collection::root()) { QTimer::singleShot(0, q, SLOT(startFirstListJob())); } else { Q_ASSERT(m_rootCollection.isValid()); CollectionFetchJob *rootFetchJob = new CollectionFetchJob(m_rootCollection, CollectionFetchJob::Base, m_session); q->connect(rootFetchJob, SIGNAL(result(KJob*)), SLOT(rootFetchJobDone(KJob*))); qCDebug(DebugETM) << ""; jobTimeTracker[rootFetchJob].start(); } } bool EntityTreeModelPrivate::canFetchMore(const QModelIndex &parent) const { const Item item = parent.data(EntityTreeModel::ItemRole).value(); if (m_collectionFetchStrategy == EntityTreeModel::InvisibleCollectionFetch) { return false; } if (item.isValid()) { // items can't have more rows. // TODO: Should I use this for fetching more of an item, ie more payload parts? return false; } else { // but collections can... const Collection::Id colId = parent.data(EntityTreeModel::CollectionIdRole).toULongLong(); // But the root collection can't... if (Collection::root().id() == colId) { return false; } // Collections which contain no items at all can't contain more if (m_collectionsWithoutItems.contains(colId)) { return false; } // Don't start the same job multiple times. if (m_pendingCollectionRetrieveJobs.contains(colId)) { return false; } // Can't fetch more if the collection's items have already been fetched if (m_populatedCols.contains(colId)) { return false; } foreach (Node *node, m_childEntities.value(colId)) { if (Node::Item == node->type) { // Only try to fetch more from a collection if we don't already have items in it. // Otherwise we'd spend all the time listing items in collections. return false; } } return true; } } QIcon EntityTreeModelPrivate::iconForName(const QString &name) const { if (m_iconThemeName != QIcon::themeName()) { m_iconThemeName = QIcon::themeName(); m_iconCache.clear(); } QIcon &icon = m_iconCache[name]; if (icon.isNull()) { icon = QIcon::fromTheme(name); } return icon; } diff --git a/src/server/connection.cpp b/src/server/connection.cpp index 205f7ae63..1df53c368 100644 --- a/src/server/connection.cpp +++ b/src/server/connection.cpp @@ -1,514 +1,513 @@ /*************************************************************************** * Copyright (C) 2006 by Till Adam * * Copyright (C) 2013 by Volker Krause * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU Library General Public License as * * published by the Free Software Foundation; either version 2 of the * * License, or (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU Library General Public * * License along with this program; if not, write to the * * Free Software Foundation, Inc., * * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * ***************************************************************************/ #include "connection.h" #include "akonadiserver_debug.h" #include #include #include #include "storage/datastore.h" #include "storage/dbdeadlockcatcher.h" #include "handler.h" #include "notificationmanager.h" #include "tracer.h" #include #ifndef Q_OS_WIN #include #endif #include #include #include using namespace Akonadi; using namespace Akonadi::Server; #define IDLE_TIMER_TIMEOUT 180000 // 3 min static QString connectionIdentifier(Connection *c) { - QString id; - id.sprintf("%p", static_cast(c)); + const QString id = QString::asprintf("%p", static_cast(c)); return id; } Connection::Connection(QObject *parent) : AkThread(connectionIdentifier(this), QThread::InheritPriority, parent) { } Connection::Connection(quintptr socketDescriptor, QObject *parent) : AkThread(connectionIdentifier(this), QThread::InheritPriority, parent) { m_socketDescriptor = socketDescriptor; m_identifier = connectionIdentifier(this); // same as objectName() const QSettings settings(Akonadi::StandardDirs::serverConfigFile(), QSettings::IniFormat); m_verifyCacheOnRetrieval = settings.value(QStringLiteral("Cache/VerifyOnRetrieval"), m_verifyCacheOnRetrieval).toBool(); } void Connection::init() { AkThread::init(); QLocalSocket *socket = new QLocalSocket(); if (!socket->setSocketDescriptor(m_socketDescriptor)) { qCWarning(AKONADISERVER_LOG) << "Connection(" << m_identifier << ")::run: failed to set socket descriptor: " << socket->error() << "(" << socket->errorString() << ")"; delete socket; return; } m_socket = socket; connect(socket, &QLocalSocket::disconnected, this, &Connection::slotSocketDisconnected); m_idleTimer = new QTimer(this); connect(m_idleTimer, &QTimer::timeout, this, &Connection::slotConnectionIdle); storageBackend()->notificationCollector()->setConnection(this); if (socket->state() == QLocalSocket::ConnectedState) { QTimer::singleShot(0, this, &Connection::handleIncomingData); } else { connect(socket, &QLocalSocket::connected, this, &Connection::handleIncomingData, Qt::QueuedConnection); } try { slotSendHello(); } catch (const ProtocolException &e) { qCWarning(AKONADISERVER_LOG) << "Protocol Exception sending \"hello\" on connection" << m_identifier << ":" << e.what(); m_socket->disconnectFromServer(); } } void Connection::quit() { if (QThread::currentThread()->loopLevel() > 1) { m_connectionClosing = true; Q_EMIT connectionClosing(); return; } Tracer::self()->endConnection(m_identifier, QString()); delete m_socket; m_socket = nullptr; if (m_idleTimer) { m_idleTimer->stop(); } delete m_idleTimer; AkThread::quit(); } void Connection::slotSendHello() { SchemaVersion version = SchemaVersion::retrieveAll().at(0); Protocol::HelloResponse hello; hello.setServerName(QStringLiteral("Akonadi")); hello.setMessage(QStringLiteral("Not Really IMAP server")); hello.setProtocolVersion(Protocol::version()); hello.setGeneration(version.generation()); sendResponse(0, std::move(hello)); } DataStore *Connection::storageBackend() { if (!m_backend) { m_backend = DataStore::self(); } return m_backend; } Connection::~Connection() { quitThread(); if (m_reportTime) { reportTime(); } } void Connection::slotConnectionIdle() { Q_ASSERT(m_currentHandler == nullptr); if (m_backend && m_backend->isOpened()) { if (m_backend->inTransaction()) { // This is a programming error, the timer should not have fired. // But it is safer to abort and leave the connection open, until // a later operation causes the idle timer to fire (than crash // the akonadi server). qCInfo(AKONADISERVER_LOG) << m_sessionId << "NOT Closing idle db connection; we are in transaction"; return; } m_backend->close(); } } void Connection::slotSocketDisconnected() { // If we have active handler, wait for it to finish, then we emit the signal // from slotNewDate() if (m_currentHandler) { return; } Q_EMIT disconnected(); } void Connection::parseStream(const Protocol::CommandPtr &cmd) { if (!m_currentHandler->parseStream()) { try { m_currentHandler->failureResponse("Error while handling a command"); } catch (...) { m_connectionClosing = true; } qCWarning(AKONADISERVER_LOG) << "Error while handling command" << cmd->type() << "on connection" << m_identifier; } } void Connection::handleIncomingData() { Q_FOREVER { if (m_connectionClosing || !m_socket || m_socket->state() != QLocalSocket::ConnectedState) { break; } // Blocks with event loop until some data arrive, allows us to still use QTimers // and similar while waiting for some data to arrive if (m_socket->bytesAvailable() < int(sizeof(qint64))) { QEventLoop loop; connect(m_socket, &QLocalSocket::readyRead, &loop, &QEventLoop::quit); connect(m_socket, &QLocalSocket::stateChanged, &loop, &QEventLoop::quit); connect(this, &Connection::connectionClosing, &loop, &QEventLoop::quit); loop.exec(); } if (m_connectionClosing || !m_socket || m_socket->state() != QLocalSocket::ConnectedState) { break; } m_idleTimer->stop(); // will only open() a previously idle backend. // Otherwise, a new backend could lazily be constructed by later calls. if (!storageBackend()->isOpened()) { m_backend->open(); } QString currentCommand; while (m_socket->bytesAvailable() >= int(sizeof(qint64))) { Protocol::DataStream stream(m_socket); qint64 tag = -1; stream >> tag; // TODO: Check tag is incremental sequence Protocol::CommandPtr cmd; try { cmd = Protocol::deserialize(m_socket); } catch (const Akonadi::ProtocolException &e) { qCWarning(AKONADISERVER_LOG) << "ProtocolException while deserializing incoming data on connection" << m_identifier << ":" << e.what(); setState(Server::LoggingOut); return; } catch (const std::exception &e) { qCWarning(AKONADISERVER_LOG) << "Unknown exception while deserializing incoming data on connection" << m_identifier << ":" << e.what(); setState(Server::LoggingOut); return; } if (cmd->type() == Protocol::Command::Invalid) { qCWarning(AKONADISERVER_LOG) << "Received an invalid command on connection" << m_identifier << ": resetting connection"; setState(Server::LoggingOut); return; } // Tag context and collection context is not persistent. context()->setTag(-1); context()->setCollection(Collection()); if (Tracer::self()->currentTracer() != QLatin1String("null")) { Tracer::self()->connectionInput(m_identifier, tag, cmd); } m_currentHandler = findHandlerForCommand(cmd->type()); if (!m_currentHandler) { qCWarning(AKONADISERVER_LOG) << "Invalid command: no such handler for" << cmd->type() << "on connection" << m_identifier; setState(Server::LoggingOut); return; } if (m_reportTime) { startTime(); } m_currentHandler->setConnection(this); m_currentHandler->setTag(tag); m_currentHandler->setCommand(cmd); try { DbDeadlockCatcher catcher([this, &cmd]() { parseStream(cmd); }); } catch (const Akonadi::Server::HandlerException &e) { if (m_currentHandler) { try { m_currentHandler->failureResponse(e.what()); } catch (...) { m_connectionClosing = true; } qCWarning(AKONADISERVER_LOG) << "Handler exception when handling command" << cmd->type() << "on connection" << m_identifier << ":" << e.what(); } } catch (const Akonadi::Server::Exception &e) { if (m_currentHandler) { try { m_currentHandler->failureResponse(QString::fromUtf8(e.type()) + QLatin1String(": ") + QString::fromUtf8(e.what())); } catch (...) { m_connectionClosing = true; } qCWarning(AKONADISERVER_LOG) << "General exception when handling command" << cmd->type() << "on connection" << m_identifier << ":" << e.what(); } } catch (const Akonadi::ProtocolException &e) { // No point trying to send anything back to client, the connection is // already messed up qCWarning(AKONADISERVER_LOG) << "Protocol exception when handling command" << cmd->type() << "on connection" << m_identifier << ":" << e.what(); m_connectionClosing = true; #if defined(Q_OS_LINUX) && !defined(_LIBCPP_VERSION) } catch (abi::__forced_unwind&) { // HACK: NPTL throws __forced_unwind during thread cancellation and // we *must* rethrow it otherwise the program aborts. Due to the issue // described in #376385 we might end up destroying (cancelling) the // thread from a nested loop executed inside parseStream() above, // so the exception raised in there gets caught by this try..catch // statement and it must be rethrown at all cost. Remove this hack // once the root problem is fixed. throw; #endif } catch (...) { qCCritical(AKONADISERVER_LOG) << "Unknown exception while handling command" << cmd->type() << "on connection" << m_identifier; if (m_currentHandler) { try { m_currentHandler->failureResponse("Unknown exception caught"); } catch (...) { m_connectionClosing = true; } } } if (m_reportTime) { stopTime(currentCommand); } m_currentHandler.reset(); if (!m_socket || m_socket->state() != QLocalSocket::ConnectedState) { Q_EMIT disconnected(); return; } if (m_connectionClosing) { break; } } // reset, arm the timer m_idleTimer->start(IDLE_TIMER_TIMEOUT); if (m_connectionClosing) { break; } } if (m_connectionClosing) { m_socket->disconnect(this); m_socket->close(); QTimer::singleShot(0, this, &Connection::quit); } } CommandContext *Connection::context() const { return const_cast(&m_context); } std::unique_ptr Connection::findHandlerForCommand(Protocol::Command::Type command) { auto handler = Handler::findHandlerForCommandAlwaysAllowed(command); if (handler) { return handler; } switch (m_connectionState) { case NonAuthenticated: handler = Handler::findHandlerForCommandNonAuthenticated(command); break; case Authenticated: handler = Handler::findHandlerForCommandAuthenticated(command); break; case Selected: break; case LoggingOut: break; } return handler; } qint64 Connection::currentTag() const { return m_currentHandler->tag(); } void Connection::setState(ConnectionState state) { if (state == m_connectionState) { return; } m_connectionState = state; switch (m_connectionState) { case NonAuthenticated: assert(0); // can't happen, it's only the initial state, we can't go back to it break; case Authenticated: break; case Selected: break; case LoggingOut: m_socket->disconnectFromServer(); break; } } void Connection::setSessionId(const QByteArray &id) { - m_identifier.sprintf("%s (%p)", id.data(), static_cast(this)); + m_identifier = QString::asprintf("%s (%p)", id.data(), static_cast(this)); Tracer::self()->beginConnection(m_identifier, QString()); //m_streamParser->setTracerIdentifier(m_identifier); m_sessionId = id; setObjectName(QString::fromLatin1(id)); // this races with the use of objectName() in QThreadPrivate::start //thread()->setObjectName(objectName() + QStringLiteral("-Thread")); storageBackend()->setSessionId(id); } QByteArray Connection::sessionId() const { return m_sessionId; } bool Connection::isOwnerResource(const PimItem &item) const { if (context()->resource().isValid() && item.collection().resourceId() == context()->resource().id()) { return true; } // fallback for older resources if (sessionId() == item.collection().resource().name().toUtf8()) { return true; } return false; } bool Connection::isOwnerResource(const Collection &collection) const { if (context()->resource().isValid() && collection.resourceId() == context()->resource().id()) { return true; } if (sessionId() == collection.resource().name().toUtf8()) { return true; } return false; } bool Connection::verifyCacheOnRetrieval() const { return m_verifyCacheOnRetrieval; } void Connection::startTime() { m_time.start(); } void Connection::stopTime(const QString &identifier) { int elapsed = m_time.elapsed(); m_totalTime += elapsed; m_totalTimeByHandler[identifier] += elapsed; m_executionsByHandler[identifier]++; qCDebug(AKONADISERVER_LOG) << identifier << " time : " << elapsed << " total: " << m_totalTime; } void Connection::reportTime() const { qCDebug(AKONADISERVER_LOG) << "===== Time report for " << m_identifier << " ====="; qCDebug(AKONADISERVER_LOG) << " total: " << m_totalTime; for (auto it = m_totalTimeByHandler.cbegin(), end = m_totalTimeByHandler.cend(); it != end; ++it) { const QString &handler = it.key(); qCDebug(AKONADISERVER_LOG) << "handler : " << handler << " time: " << m_totalTimeByHandler.value(handler) << " executions " << m_executionsByHandler.value(handler) << " avg: " << m_totalTimeByHandler.value(handler) / m_executionsByHandler.value(handler); } } void Connection::sendResponse(qint64 tag, const Protocol::CommandPtr &response) { if (Tracer::self()->currentTracer() != QLatin1String("null")) { Tracer::self()->connectionOutput(m_identifier, tag, response); } Protocol::DataStream stream(m_socket); stream << tag; Protocol::serialize(m_socket, response); if (!m_socket->waitForBytesWritten()) { if (m_socket->state() == QLocalSocket::ConnectedState) { throw ProtocolException("Server write timeout"); } else { // The client has disconnected before we managed to send our response, // which is not an error } } } Protocol::CommandPtr Connection::readCommand() { while (m_socket->bytesAvailable() < (int) sizeof(qint64)) { Protocol::DataStream::waitForData(m_socket, 10000); // 10 seconds, just in case client is busy } Protocol::DataStream stream(m_socket); qint64 tag; stream >> tag; // TODO: compare tag with m_currentHandler->tag() ? return Protocol::deserialize(m_socket); } diff --git a/src/server/search/searchtaskmanager.cpp b/src/server/search/searchtaskmanager.cpp index e3c65ae4d..d2e403058 100644 --- a/src/server/search/searchtaskmanager.cpp +++ b/src/server/search/searchtaskmanager.cpp @@ -1,326 +1,331 @@ /* Copyright (c) 2013, 2014 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "searchtaskmanager.h" #include "agentsearchinstance.h" #include "connection.h" #include "storage/selectquerybuilder.h" #include "dbusconnectionpool.h" #include "entities.h" #include "akonadiserver_search_debug.h" #include #include #include #include - +#if QT_VERSION >= QT_VERSION_CHECK(5, 15, 0) +#include +#endif using namespace Akonadi; using namespace Akonadi::Server; SearchTaskManager *SearchTaskManager::sInstance = nullptr; SearchTaskManager::SearchTaskManager() : AkThread(QStringLiteral("SearchTaskManager")) , mShouldStop(false) { sInstance = this; QTimer::singleShot(0, this, &SearchTaskManager::searchLoop); } SearchTaskManager::~SearchTaskManager() { QMutexLocker locker(&mLock); mShouldStop = true; mWait.wakeAll(); locker.unlock(); quitThread(); mInstancesLock.lock(); qDeleteAll(mInstances); mInstancesLock.unlock(); } SearchTaskManager *SearchTaskManager::instance() { Q_ASSERT(sInstance); return sInstance; } void SearchTaskManager::registerInstance(const QString &id) { QMutexLocker locker(&mInstancesLock); qCDebug(AKONADISERVER_SEARCH_LOG) << "SearchManager::registerInstance(" << id << ")"; AgentSearchInstance *instance = mInstances.value(id); if (instance) { return; // already registered } instance = new AgentSearchInstance(id); if (!instance->init()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Failed to initialize Search agent"; delete instance; return; } qCDebug(AKONADISERVER_SEARCH_LOG) << "Registering search instance " << id; mInstances.insert(id, instance); } void SearchTaskManager::unregisterInstance(const QString &id) { QMutexLocker locker(&mInstancesLock); QMap::Iterator it = mInstances.find(id); if (it != mInstances.end()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Unregistering search instance" << id; it.value()->deleteLater(); mInstances.erase(it); } } void SearchTaskManager::addTask(SearchTask *task) { QueryBuilder qb(Collection::tableName()); qb.addJoin(QueryBuilder::InnerJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName()); qb.addColumn(Collection::idFullColumnName()); qb.addColumn(Resource::nameFullColumnName()); Q_ASSERT(!task->collections.isEmpty()); QVariantList list; list.reserve(task->collections.size()); for (qint64 collection : qAsConst(task->collections)) { list << collection; } qb.addValueCondition(Collection::idFullColumnName(), Query::In, list); if (!qb.exec()) { throw SearchException(qb.query().lastError().text()); } QSqlQuery query = qb.query(); if (!query.next()) { return; } mInstancesLock.lock(); org::freedesktop::Akonadi::AgentManager agentManager(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), DBusConnectionPool::threadConnection()); do { const QString resourceId = query.value(1).toString(); if (!mInstances.contains(resourceId)) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Resource" << resourceId << "does not implement Search interface, skipping"; } else if (!agentManager.agentInstanceOnline(resourceId)) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Agent" << resourceId << "is offline, skipping"; } else if (agentManager.agentInstanceStatus(resourceId) > 2) { // 2 == Broken, 3 == Not Configured qCDebug(AKONADISERVER_SEARCH_LOG) << "Agent" << resourceId << "is broken or not configured"; } else { const qint64 collectionId = query.value(0).toLongLong(); qCDebug(AKONADISERVER_SEARCH_LOG) << "Enqueued search query (" << resourceId << ", " << collectionId << ")"; task->queries << qMakePair(resourceId, collectionId); } } while (query.next()); mInstancesLock.unlock(); QMutexLocker locker(&mLock); mTasklist.append(task); mWait.wakeAll(); } void SearchTaskManager::pushResults(const QByteArray &searchId, const QSet &ids, Connection *connection) { Q_UNUSED(searchId); qCDebug(AKONADISERVER_SEARCH_LOG) << ids.count() << "results for search" << searchId << "pushed from" << connection->context()->resource().name(); QMutexLocker locker(&mLock); ResourceTask *task = mRunningTasks.take(connection->context()->resource().name()); if (!task) { qCDebug(AKONADISERVER_SEARCH_LOG) << "No running task for" << connection->context()->resource().name() << " - maybe it has timed out?"; return; } if (task->parentTask->id != searchId) { qCDebug(AKONADISERVER_SEARCH_LOG) << "Received results for different search - maybe the original task has timed out?"; qCDebug(AKONADISERVER_SEARCH_LOG) << "Search is" << searchId << ", but task is" << task->parentTask->id; return; } task->results = ids; mPendingResults.append(task); mWait.wakeAll(); } bool SearchTaskManager::allResourceTasksCompleted(SearchTask *agentSearchTask) const { // Check for queries pending to be dispatched if (!agentSearchTask->queries.isEmpty()) { return false; } // Check for running queries QMap::const_iterator it = mRunningTasks.begin(); QMap::const_iterator end = mRunningTasks.end(); for (; it != end; ++it) { if (it.value()->parentTask == agentSearchTask) { return false; } } return true; } SearchTaskManager::TasksMap::Iterator SearchTaskManager::cancelRunningTask(TasksMap::Iterator &iter) { ResourceTask *task = iter.value(); SearchTask *parentTask = task->parentTask; QMutexLocker locker(&parentTask->sharedLock); //erase the task before allResourceTasksCompleted SearchTaskManager::TasksMap::Iterator it = mRunningTasks.erase(iter); // We're not clearing the results since we don't want to clear successful results from other resources parentTask->complete = allResourceTasksCompleted(parentTask); parentTask->notifier.wakeAll(); delete task; return it; } void SearchTaskManager::searchLoop() { qint64 timeout = ULONG_MAX; QMutexLocker locker(&mLock); Q_FOREVER { qCDebug(AKONADISERVER_SEARCH_LOG) << "Search loop is waiting, will wake again in" << timeout << "ms"; +#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) mWait.wait(&mLock, timeout); - +#else + mWait.wait(&mLock, QDeadlineTimer(QDeadlineTimer::Forever)); +#endif if (mShouldStop) { Q_FOREACH (SearchTask *task, mTasklist) { QMutexLocker locker(&task->sharedLock); task->queries.clear(); task->notifier.wakeAll(); } QMap::Iterator it = mRunningTasks.begin(); for (; it != mRunningTasks.end();) { if (mTasklist.contains(it.value()->parentTask)) { delete it.value(); it = mRunningTasks.erase(it); continue; } it = cancelRunningTask(it); } break; } // First notify about available results while (!mPendingResults.isEmpty()) { ResourceTask *finishedTask = mPendingResults.first(); mPendingResults.remove(0); qCDebug(AKONADISERVER_SEARCH_LOG) << "Pending results from" << finishedTask->resourceId << "for collection" << finishedTask->collectionId << "for search" << finishedTask->parentTask->id << "available!"; SearchTask *parentTask = finishedTask->parentTask; QMutexLocker locker(&parentTask->sharedLock); // We need to append, this agent search task is shared parentTask->pendingResults += finishedTask->results; parentTask->complete = allResourceTasksCompleted(parentTask); parentTask->notifier.wakeAll(); delete finishedTask; } // No check whether there are any tasks running longer than 1 minute and kill them QMap::Iterator it = mRunningTasks.begin(); const qint64 now = QDateTime::currentMSecsSinceEpoch(); for (; it != mRunningTasks.end();) { ResourceTask *task = it.value(); if (now - task->timestamp > 60 * 1000) { // Remove the task - and signal to parent task that it has "finished" without results qCDebug(AKONADISERVER_SEARCH_LOG) << "Resource task" << task->resourceId << "for search" << task->parentTask->id << "timed out!"; it = cancelRunningTask(it); } else { ++it; } } if (!mTasklist.isEmpty()) { SearchTask *task = mTasklist.first(); qCDebug(AKONADISERVER_SEARCH_LOG) << "Search task" << task->id << "available!"; if (task->queries.isEmpty()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "nothing to do for task"; QMutexLocker locker(&task->sharedLock); //After this the AgentSearchTask will be destroyed task->complete = true; task->notifier.wakeAll(); mTasklist.remove(0); continue; } QVector >::iterator it = task->queries.begin(); for (; it != task->queries.end();) { if (!mRunningTasks.contains(it->first)) { qCDebug(AKONADISERVER_SEARCH_LOG) << "\t Sending query for collection" << it->second << "to resource" << it->first; ResourceTask *rTask = new ResourceTask; rTask->resourceId = it->first; rTask->collectionId = it->second; rTask->parentTask = task; rTask->timestamp = QDateTime::currentMSecsSinceEpoch(); mRunningTasks.insert(it->first, rTask); mInstancesLock.lock(); AgentSearchInstance *instance = mInstances.value(it->first); if (!instance) { mInstancesLock.unlock(); // Resource disappeared in the meanwhile continue; } instance->search(task->id, task->query, it->second); mInstancesLock.unlock(); task->sharedLock.lock(); it = task->queries.erase(it); task->sharedLock.unlock(); } else { ++it; } } // Yay! We managed to dispatch all requests! if (task->queries.isEmpty()) { qCDebug(AKONADISERVER_SEARCH_LOG) << "All queries from task" << task->id << "dispatched!"; mTasklist.remove(0); } timeout = 60 * 1000; // check whether all tasks have finished within a minute } else { if (mRunningTasks.isEmpty()) { timeout = ULONG_MAX; } } } }