diff --git a/src/private/standarddirs.cpp b/src/private/standarddirs.cpp index 8fa454c89..45af611aa 100644 --- a/src/private/standarddirs.cpp +++ b/src/private/standarddirs.cpp @@ -1,198 +1,220 @@ /* Copyright (c) 2011 Volker Krause Copyright (c) 2018 Daniel Vrátil This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "standarddirs_p.h" #include "instance_p.h" #include "akonadi-prefix.h" +#include "akonadiprivate_debug.h" #include #include #include #include #include #include using namespace Akonadi; namespace { QString buildFullRelPath(const char *resource, const QString &relPath) { QString fullRelPath = QStringLiteral("/akonadi"); #ifdef Q_OS_WIN // On Windows all Generic*Location fall into ~/AppData/Local so we need to disambiguate // inside the "akonadi" folder whether it's data or config. fullRelPath += QLatin1Char('/') + QString::fromLocal8Bit(resource); #else Q_UNUSED(resource); #endif if (Akonadi::Instance::hasIdentifier()) { fullRelPath += QStringLiteral("/instance/") + Akonadi::Instance::identifier(); } if (!relPath.isEmpty()) { fullRelPath += QLatin1Char('/') + relPath; } return fullRelPath; } } QString StandardDirs::configFile(const QString &configFile, FileAccessMode openMode) { const QString savePath = StandardDirs::saveDir("config") + QLatin1Char('/') + configFile; if (openMode == WriteOnly) { return savePath; } auto path = QStandardPaths::locate(QStandardPaths::GenericConfigLocation, QStringLiteral("akonadi/") + configFile); // HACK: when using instance namespaces, ignore the non-namespaced file if (Akonadi::Instance::hasIdentifier() && path.startsWith(QStandardPaths::writableLocation(QStandardPaths::ConfigLocation))) { path.clear(); } if (path.isEmpty()) { return savePath; } else if (openMode == ReadOnly || path == savePath) { return path; } // file found in system paths and mode is ReadWrite, thus // we copy to the home path location and return this path QFile::copy(path, savePath); return savePath; } QString StandardDirs::serverConfigFile(FileAccessMode openMode) { return configFile(QStringLiteral("akonadiserverrc"), openMode); } QString StandardDirs::connectionConfigFile(FileAccessMode openMode) { return configFile(QStringLiteral("akonadiconnectionrc"), openMode); } QString StandardDirs::agentsConfigFile(FileAccessMode openMode) { return configFile(QStringLiteral("agentsrc"), openMode); } QString StandardDirs::agentConfigFile(const QString &identifier, FileAccessMode openMode) { return configFile(QStringLiteral("agent_config_") + identifier, openMode); } QString StandardDirs::saveDir(const char *resource, const QString &relPath) { const QString fullRelPath = buildFullRelPath(resource, relPath); + QString fullPath; if (qstrncmp(resource, "config", 6) == 0) { - return QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + fullRelPath; + fullPath = QStandardPaths::writableLocation(QStandardPaths::GenericConfigLocation) + fullRelPath; } else if (qstrncmp(resource, "data", 4) == 0) { - return QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + fullRelPath; + fullPath = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + fullRelPath; } else { qt_assert_x(__FUNCTION__, "Invalid resource type", __FILE__, __LINE__); return {}; } + + // ensure directory exists or is created + QFileInfo fileInfo(fullPath); + if (fileInfo.exists()) { + if (fileInfo.isDir()) { + return fullPath; + } else { + qCWarning(AKONADIPRIVATE_LOG) << "StandardDirs::saveDir: '" << fileInfo.absoluteFilePath() + << "' exists but is not a directory"; + } + } else { + if (!QDir::home().mkpath(fileInfo.absoluteFilePath())) { + qCWarning(AKONADIPRIVATE_LOG) << "StandardDirs::saveDir: failed to create directory '" + << fileInfo.absoluteFilePath() << "'"; + } else { + return fullPath; + } + } + + return {}; } QString StandardDirs::locateResourceFile(const char *resource, const QString &relPath) { const QString fullRelPath = buildFullRelPath(resource, relPath); QVector userLocations; QStandardPaths::StandardLocation genericLocation; QString fallback; if (qstrncmp(resource, "config", 6) == 0) { userLocations = { QStandardPaths::AppConfigLocation, QStandardPaths::ConfigLocation }; genericLocation = QStandardPaths::GenericConfigLocation; fallback = QDir::toNativeSeparators(QStringLiteral(AKONADIPREFIX "/" AKONADICONFIG)); } else if (qstrncmp(resource, "data", 4) == 0) { userLocations = { QStandardPaths::AppLocalDataLocation, QStandardPaths::AppDataLocation }; genericLocation = QStandardPaths::GenericDataLocation; fallback = QDir::toNativeSeparators(QStringLiteral(AKONADIPREFIX "/" AKONADIDATA)); } else { qt_assert_x(__FUNCTION__, "Invalid resource type", __FILE__, __LINE__); return {}; } const auto locateFile = [](QStandardPaths::StandardLocation location, const QString &relPath) -> QString { const auto path = QStandardPaths::locate(location, relPath); if (!path.isEmpty()) { QFileInfo file(path); if (file.exists() && file.isFile() && file.isReadable()) { return path; } } return {}; }; // Always honor instance in user-specific locations for (const auto location : qAsConst(userLocations)) { const auto path = locateFile(location, fullRelPath); if (!path.isEmpty()) { return path; } } // First try instance-specific path in generic locations auto path = locateFile(genericLocation, fullRelPath); if (!path.isEmpty()) { return path; } // Fallback to global instance path in generic locations path = locateFile(genericLocation, QStringLiteral("/akonadi/") + relPath); if (!path.isEmpty()) { return path; } QFile f(fallback + QStringLiteral("/akonadi/") + relPath); if (f.exists()) { return f.fileName(); } return {}; } QStringList StandardDirs::locateAllResourceDirs(const QString &relPath) { auto dirs = QStandardPaths::locateAll(QStandardPaths::GenericDataLocation, relPath, QStandardPaths::LocateDirectory); const auto fallback = QDir::toNativeSeparators(QStringLiteral(AKONADIPREFIX "/" AKONADIDATA "/") + relPath); if (!dirs.contains(fallback)) { if (QDir::root().exists(fallback)) { dirs.push_back(fallback); } } return dirs; } QString StandardDirs::findExecutable(const QString &executableName) { QString executable = QStandardPaths::findExecutable(executableName, {qApp->applicationDirPath()}); if (executable.isEmpty()) { executable = QStandardPaths::findExecutable(executableName); } return executable; } diff --git a/src/server/storage/querybuilder.cpp b/src/server/storage/querybuilder.cpp index dea6817ed..7cf691420 100644 --- a/src/server/storage/querybuilder.cpp +++ b/src/server/storage/querybuilder.cpp @@ -1,635 +1,642 @@ /* Copyright (c) 2007 - 2012 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "querybuilder.h" #include "akonadiserver_debug.h" #ifndef QUERYBUILDER_UNITTEST #include "storage/datastore.h" #include "storage/querycache.h" #include "storage/storagedebugger.h" #endif #include #include #include #include using namespace Akonadi::Server; static QLatin1String compareOperatorToString(Query::CompareOperator op) { switch (op) { case Query::Equals: return QLatin1String(" = "); case Query::NotEquals: return QLatin1String(" <> "); case Query::Is: return QLatin1String(" IS "); case Query::IsNot: return QLatin1String(" IS NOT "); case Query::Less: return QLatin1String(" < "); case Query::LessOrEqual: return QLatin1String(" <= "); case Query::Greater: return QLatin1String(" > "); case Query::GreaterOrEqual: return QLatin1String(" >= "); case Query::In: return QLatin1String(" IN "); case Query::NotIn: return QLatin1String(" NOT IN "); case Query::Like: return QLatin1String(" LIKE "); } Q_ASSERT_X(false, "QueryBuilder::compareOperatorToString()", "Unknown compare operator."); return QLatin1String(""); } static QLatin1String logicOperatorToString(Query::LogicOperator op) { switch (op) { case Query::And: return QLatin1String(" AND "); case Query::Or: return QLatin1String(" OR "); } Q_ASSERT_X(false, "QueryBuilder::logicOperatorToString()", "Unknown logic operator."); return QLatin1String(""); } static QLatin1String sortOrderToString(Query::SortOrder order) { switch (order) { case Query::Ascending: return QLatin1String(" ASC"); case Query::Descending: return QLatin1String(" DESC"); } Q_ASSERT_X(false, "QueryBuilder::sortOrderToString()", "Unknown sort order."); return QLatin1String(""); } static void appendJoined(QString *statement, const QStringList &strings, const QLatin1String &glue = QLatin1String(", ")) { for (int i = 0, c = strings.size(); i < c; ++i) { *statement += strings.at(i); if (i + 1 < c) { *statement += glue; } } } QueryBuilder::QueryBuilder(const QString &table, QueryBuilder::QueryType type) : mTable(table) #ifndef QUERYBUILDER_UNITTEST , mDatabaseType(DbType::type(DataStore::self()->database())) , mQuery(DataStore::self()->database()) #else , mDatabaseType(DbType::Unknown) #endif , mType(type) , mIdentificationColumn() , mLimit(-1) , mDistinct(false) { static const QString defaultIdColumn = QStringLiteral("id"); mIdentificationColumn = defaultIdColumn; } void QueryBuilder::setDatabaseType(DbType::Type type) { mDatabaseType = type; } void QueryBuilder::addJoin(JoinType joinType, const QString &table, const Query::Condition &condition) { Q_ASSERT((joinType == InnerJoin && (mType == Select || mType == Update)) || (joinType == LeftJoin && mType == Select)); if (mJoinedTables.contains(table)) { // InnerJoin is more restrictive than a LeftJoin, hence use that in doubt mJoins[table].first = qMin(joinType, mJoins.value(table).first); mJoins[table].second.addCondition(condition); } else { mJoins[table] = qMakePair(joinType, condition); mJoinedTables << table; } } void QueryBuilder::addJoin(JoinType joinType, const QString &table, const QString &col1, const QString &col2) { Query::Condition condition; condition.addColumnCondition(col1, Query::Equals, col2); addJoin(joinType, table, condition); } void QueryBuilder::addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addValueCondition(column, op, value); } void QueryBuilder::addColumnCondition(const QString &column, Query::CompareOperator op, const QString &column2, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addColumnCondition(column, op, column2); } QSqlQuery &QueryBuilder::query() { return mQuery; } void QueryBuilder::sqliteAdaptUpdateJoin(Query::Condition &condition) { // FIXME: This does not cover all cases by far. It however can handle most // (probably all) of the update-join queries we do in Akonadi and convert them // properly into a SQLite-compatible query. Better than nothing ;-) if (!condition.mSubConditions.isEmpty()) { for (int i = condition.mSubConditions.count() - 1; i >= 0; --i) { sqliteAdaptUpdateJoin(condition.mSubConditions[i]); } return; } QString table; if (condition.mColumn.contains(QLatin1Char('.'))) { table = condition.mColumn.left(condition.mColumn.indexOf(QLatin1Char('.'))); } else { return; } if (!mJoinedTables.contains(table)) { return; } const QPair joinCondition = mJoins.value(table); QueryBuilder qb(table, Select); qb.addColumn(condition.mColumn); qb.addCondition(joinCondition.second); // Convert the subquery to string condition.mColumn.reserve(1024); condition.mColumn.resize(0); condition.mColumn += QLatin1String("( "); qb.buildQuery(&condition.mColumn); condition.mColumn += QLatin1String(" )"); } void QueryBuilder::buildQuery(QString *statement) { // we add the ON conditions of Inner Joins in a Update query here // but don't want to change the mRootCondition on each exec(). Query::Condition whereCondition = mRootCondition[WhereCondition]; switch (mType) { case Select: // Enable forward-only on all SELECT queries, since we never need to // iterate backwards. This is a memory optimization. mQuery.setForwardOnly(true); *statement += QLatin1String("SELECT "); if (mDistinct) { *statement += QLatin1String("DISTINCT "); } Q_ASSERT_X(mColumns.count() > 0, "QueryBuilder::exec()", "No columns specified"); appendJoined(statement, mColumns); *statement += QLatin1String(" FROM "); *statement += mTable; for (const QString &joinedTable : qAsConst(mJoinedTables)) { const QPair &join = mJoins.value(joinedTable); switch (join.first) { case LeftJoin: *statement += QLatin1String(" LEFT JOIN "); break; case InnerJoin: *statement += QLatin1String(" INNER JOIN "); break; } *statement += joinedTable; *statement += QLatin1String(" ON "); buildWhereCondition(statement, join.second); } break; case Insert: { *statement += QLatin1String("INSERT INTO "); *statement += mTable; *statement += QLatin1String(" ("); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { *statement += mColumnValues.at(i).first; if (i + 1 < c) { *statement += QLatin1String(", "); } } *statement += QLatin1String(") VALUES ("); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { bindValue(statement, mColumnValues.at(i).second); if (i + 1 < c) { *statement += QLatin1String(", "); } } *statement += QLatin1Char(')'); if (mDatabaseType == DbType::PostgreSQL && !mIdentificationColumn.isEmpty()) { *statement += QLatin1String(" RETURNING ") + mIdentificationColumn; } break; } case Update: { // put the ON condition into the WHERE part of the UPDATE query if (mDatabaseType != DbType::Sqlite) { for (const QString &table : qAsConst(mJoinedTables)) { const QPair< JoinType, Query::Condition > &join = mJoins.value(table); Q_ASSERT(join.first == InnerJoin); whereCondition.addCondition(join.second); } } else { // Note: this will modify the whereCondition sqliteAdaptUpdateJoin(whereCondition); } *statement += QLatin1String("UPDATE "); *statement += mTable; if (mDatabaseType == DbType::MySQL && !mJoinedTables.isEmpty()) { // for mysql we list all tables directly *statement += QLatin1String(", "); appendJoined(statement, mJoinedTables); } *statement += QLatin1String(" SET "); Q_ASSERT_X(mColumnValues.count() >= 1, "QueryBuilder::exec()", "At least one column needs to be changed"); for (int i = 0, c = mColumnValues.size(); i < c; ++i) { const QPair &p = mColumnValues.at(i); *statement += p.first; *statement += QLatin1String(" = "); bindValue(statement, p.second); if (i + 1 < c) { *statement += QLatin1String(", "); } } if (mDatabaseType == DbType::PostgreSQL && !mJoinedTables.isEmpty()) { // PSQL have this syntax // FROM t1 JOIN t2 JOIN ... *statement += QLatin1String(" FROM "); appendJoined(statement, mJoinedTables, QLatin1String(" JOIN ")); } break; } case Delete: *statement += QLatin1String("DELETE FROM "); *statement += mTable; break; default: Q_ASSERT_X(false, "QueryBuilder::exec()", "Unknown enum value"); } if (!whereCondition.isEmpty()) { *statement += QLatin1String(" WHERE "); buildWhereCondition(statement, whereCondition); } if (!mGroupColumns.isEmpty()) { *statement += QLatin1String(" GROUP BY "); appendJoined(statement, mGroupColumns); } if (!mRootCondition[HavingCondition].isEmpty()) { *statement += QLatin1String(" HAVING "); buildWhereCondition(statement, mRootCondition[HavingCondition]); } if (!mSortColumns.isEmpty()) { Q_ASSERT_X(mType == Select, "QueryBuilder::exec()", "Order statements are only valid for SELECT queries"); *statement += QLatin1String(" ORDER BY "); for (int i = 0, c = mSortColumns.size(); i < c; ++i) { const QPair &order = mSortColumns.at(i); *statement += order.first; *statement += sortOrderToString(order.second); if (i + 1 < c) { *statement += QLatin1String(", "); } } } if (mLimit > 0) { *statement += QLatin1Literal(" LIMIT ") + QString::number(mLimit); } if (mType == Select && mForUpdate) { if (mDatabaseType == DbType::Sqlite) { // SQLite does not support SELECT ... FOR UPDATE syntax, because it does // table-level locking } else { *statement += QLatin1Literal(" FOR UPDATE"); } } } bool QueryBuilder::retryLastTransaction(bool rollback) { #ifndef QUERYBUILDER_UNITTEST mQuery = DataStore::self()->retryLastTransaction(rollback); return !mQuery.lastError().isValid(); #else Q_UNUSED(rollback); return true; #endif } bool QueryBuilder::exec() { QString statement; statement.reserve(1024); buildQuery(&statement); #ifndef QUERYBUILDER_UNITTEST auto query = QueryCache::query(statement); if (query.has_value()) { mQuery = std::move(*query); } else { mQuery.clear(); - mQuery.prepare(statement); + if (!mQuery.prepare(statement)) { + qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR while PREPARING QUERY:"; + qCCritical(AKONADISERVER_LOG) << " Error code:" << mQuery.lastError().nativeErrorCode(); + qCCritical(AKONADISERVER_LOG) << " DB error: " << mQuery.lastError().databaseText(); + qCCritical(AKONADISERVER_LOG) << " Error text:" << mQuery.lastError().text(); + qCCritical(AKONADISERVER_LOG) << " Query:" << statement; + return false; + } QueryCache::insert(statement, mQuery); } //too heavy debug info but worths to have from time to time //qCDebug(AKONADISERVER_LOG) << "Executing query" << statement; bool isBatch = false; for (int i = 0; i < mBindValues.count(); ++i) { mQuery.bindValue(QLatin1Char(':') + QString::number(i), mBindValues[i]); if (!isBatch && static_cast(mBindValues[i].type()) == QMetaType::QVariantList) { isBatch = true; } //qCDebug(AKONADISERVER_LOG) << QString::fromLatin1( ":%1" ).arg( i ) << mBindValues[i]; } bool ret; if (StorageDebugger::instance()->isSQLDebuggingEnabled()) { QTime t; t.start(); if (isBatch) { ret = mQuery.execBatch(); } else { ret = mQuery.exec(); } StorageDebugger::instance()->queryExecuted(reinterpret_cast(DataStore::self()), mQuery, t.elapsed()); } else { StorageDebugger::instance()->incSequence(); if (isBatch) { ret = mQuery.execBatch(); } else { ret = mQuery.exec(); } } // Add the query to DataStore so that we can replay it in case transaction deadlocks. // The method does nothing when this query is not executed within a transaction. // We don't care whether the query was successful or not. In case of error, the caller // will rollback the transaction anyway, and all cached queries will be removed. DataStore::self()->addQueryToTransaction(statement, mBindValues, isBatch); if (!ret) { // Handle transaction deadlocks and timeouts by attempting to replay the transaction. if (mDatabaseType == DbType::PostgreSQL) { const QString dbError = mQuery.lastError().databaseText(); if (dbError.contains(QLatin1String("40P01" /* deadlock_detected */))) { qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } } else if (mDatabaseType == DbType::MySQL) { const QString lastErrorStr = mQuery.lastError().nativeErrorCode(); const int error = lastErrorStr.isEmpty() ? -1 : lastErrorStr.toInt(); if (error == 1213 /* ER_LOCK_DEADLOCK */) { qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } else if (error == 1205 /* ER_LOCK_WAIT_TIMEOUT */) { qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(); } } else if (mDatabaseType == DbType::Sqlite && !DbType::isSystemSQLite(DataStore::self()->database())) { const QString lastErrorStr = mQuery.lastError().nativeErrorCode(); const int error = lastErrorStr.isEmpty() ? -1 : lastErrorStr.toInt(); if (error == 6 /* SQLITE_LOCKED */) { qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction deadlock, retrying transaction"; qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(true); } else if (error == 5 /* SQLITE_BUSY */) { qCWarning(AKONADISERVER_LOG) << "QueryBuilder::exec(): database reported transaction timeout, retrying transaction"; qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text(); return retryLastTransaction(true); } } else if (mDatabaseType == DbType::Sqlite) { // We can't have a transaction deadlock in SQLite when using driver shipped // with Qt, because it does not support concurrent transactions and DataStore // serializes them through a global lock. } qCCritical(AKONADISERVER_LOG) << "DATABASE ERROR:"; qCCritical(AKONADISERVER_LOG) << " Error code:" << mQuery.lastError().nativeErrorCode(); qCCritical(AKONADISERVER_LOG) << " DB error: " << mQuery.lastError().databaseText(); qCCritical(AKONADISERVER_LOG) << " Error text:" << mQuery.lastError().text(); qCCritical(AKONADISERVER_LOG) << " Values:" << mQuery.boundValues(); qCCritical(AKONADISERVER_LOG) << " Query:" << statement; return false; } #else mStatement = statement; #endif return true; } void QueryBuilder::addColumns(const QStringList &cols) { mColumns << cols; } void QueryBuilder::addColumn(const QString &col) { mColumns << col; } void QueryBuilder::addColumn(const Query::Case &caseStmt) { QString query; buildCaseStatement(&query, caseStmt); mColumns.append(query); } void QueryBuilder::addAggregation(const QString &col, const QString &aggregate) { mColumns.append(aggregate + QLatin1Char('(') + col + QLatin1Char(')')); } void QueryBuilder::addAggregation(const Query::Case &caseStmt, const QString &aggregate) { QString query(aggregate + QLatin1Char('(')); buildCaseStatement(&query, caseStmt); query += QLatin1Char(')'); mColumns.append(query); } void QueryBuilder::bindValue(QString *query, const QVariant &value) { mBindValues << value; *query += QLatin1Char(':') + QString::number(mBindValues.count() - 1); } void QueryBuilder::buildWhereCondition(QString *query, const Query::Condition &cond) { if (!cond.isEmpty()) { *query += QLatin1String("( "); const QLatin1String glue = logicOperatorToString(cond.mCombineOp); const Query::Condition::List &subConditions = cond.subConditions(); for (int i = 0, c = subConditions.size(); i < c; ++i) { buildWhereCondition(query, subConditions.at(i)); if (i + 1 < c) { *query += glue; } } *query += QLatin1String(" )"); } else { *query += cond.mColumn; *query += compareOperatorToString(cond.mCompareOp); if (cond.mComparedColumn.isEmpty()) { if (cond.mComparedValue.isValid()) { if (cond.mComparedValue.canConvert(QVariant::List)) { *query += QLatin1String("( "); const QVariantList &entries = cond.mComparedValue.toList(); Q_ASSERT_X(!entries.isEmpty(), "QueryBuilder::buildWhereCondition()", "No values given for IN condition."); for (int i = 0, c = entries.size(); i < c; ++i) { bindValue(query, entries.at(i)); if (i + 1 < c) { *query += QLatin1String(", "); } } *query += QLatin1String(" )"); } else { bindValue(query, cond.mComparedValue); } } else { *query += QLatin1String("NULL"); } } else { *query += cond.mComparedColumn; } } } void QueryBuilder::buildCaseStatement(QString *query, const Query::Case &caseStmt) { *query += QLatin1String("CASE "); Q_FOREACH (const auto &whenThen, caseStmt.mWhenThen) { *query += QLatin1String("WHEN "); buildWhereCondition(query, whenThen.first); // When *query += QLatin1String(" THEN ") + whenThen.second; // then } if (!caseStmt.mElse.isEmpty()) { *query += QLatin1String(" ELSE ") + caseStmt.mElse; } *query += QLatin1String(" END"); } void QueryBuilder::setSubQueryMode(Query::LogicOperator op, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].setSubQueryMode(op); } void QueryBuilder::addCondition(const Query::Condition &condition, ConditionType type) { Q_ASSERT(type == WhereCondition || (type == HavingCondition && mType == Select)); mRootCondition[type].addCondition(condition); } void QueryBuilder::addSortColumn(const QString &column, Query::SortOrder order) { mSortColumns << qMakePair(column, order); } void QueryBuilder::addGroupColumn(const QString &column) { Q_ASSERT(mType == Select); mGroupColumns << column; } void QueryBuilder::addGroupColumns(const QStringList &columns) { Q_ASSERT(mType == Select); mGroupColumns += columns; } void QueryBuilder::setColumnValue(const QString &column, const QVariant &value) { mColumnValues << qMakePair(column, value); } void QueryBuilder::setDistinct(bool distinct) { mDistinct = distinct; } void QueryBuilder::setLimit(int limit) { mLimit = limit; } void QueryBuilder::setIdentificationColumn(const QString &column) { mIdentificationColumn = column; } qint64 QueryBuilder::insertId() { if (mDatabaseType == DbType::PostgreSQL) { query().next(); if (mIdentificationColumn.isEmpty()) { return 0; // FIXME: Does this make sense? } return query().record().value(mIdentificationColumn).toLongLong(); } else { const QVariant v = query().lastInsertId(); if (!v.isValid()) { return -1; } bool ok; const qint64 insertId = v.toLongLong(&ok); if (!ok) { return -1; } return insertId; } return -1; } void QueryBuilder::setForUpdate(bool forUpdate) { mForUpdate = forUpdate; } diff --git a/src/server/storagejanitor.cpp b/src/server/storagejanitor.cpp index 3ca6f4823..e4a67ac2a 100644 --- a/src/server/storagejanitor.cpp +++ b/src/server/storagejanitor.cpp @@ -1,848 +1,848 @@ /* Copyright (c) 2011 Volker Krause This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "storagejanitor.h" #include "storage/queryhelper.h" #include "storage/transaction.h" #include "storage/datastore.h" #include "storage/selectquerybuilder.h" #include "storage/parthelper.h" #include "storage/dbconfig.h" #include "storage/collectionstatistics.h" #include "search/searchrequest.h" #include "search/searchmanager.h" #include "resourcemanager.h" #include "entities.h" #include "dbusconnectionpool.h" #include "agentmanagerinterface.h" #include "akonadiserver_debug.h" #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace Akonadi; using namespace Akonadi::Server; StorageJanitor::StorageJanitor(QObject *parent) : AkThread(QStringLiteral("StorageJanitor"), QThread::IdlePriority, parent) , m_lostFoundCollectionId(-1) { } StorageJanitor::~StorageJanitor() { quitThread(); } void StorageJanitor::init() { AkThread::init(); QDBusConnection conn = DBusConnectionPool::threadConnection(); conn.registerService(DBus::serviceName(DBus::StorageJanitor)); conn.registerObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), this, QDBusConnection::ExportScriptableSlots | QDBusConnection::ExportScriptableSignals); } void StorageJanitor::quit() { QDBusConnection conn = DBusConnectionPool::threadConnection(); conn.unregisterObject(QStringLiteral(AKONADI_DBUS_STORAGEJANITOR_PATH), QDBusConnection::UnregisterTree); conn.unregisterService(DBus::serviceName(DBus::StorageJanitor)); conn.disconnectFromBus(conn.name()); // Make sure all children are deleted within context of this thread qDeleteAll(children()); AkThread::quit(); } void StorageJanitor::check() // implementation of `akonadictl fsck` { m_lostFoundCollectionId = -1; // start with a fresh one each time inform("Looking for resources in the DB not matching a configured resource..."); findOrphanedResources(); inform("Looking for collections not belonging to a valid resource..."); findOrphanedCollections(); inform("Checking collection tree consistency..."); const Collection::List cols = Collection::retrieveAll(); std::for_each(cols.begin(), cols.end(), [this](const Collection & col) { checkPathToRoot(col); }); inform("Looking for items not belonging to a valid collection..."); findOrphanedItems(); inform("Looking for item parts not belonging to a valid item..."); findOrphanedParts(); inform("Looking for item flags not belonging to a valid item..."); findOrphanedPimItemFlags(); inform("Looking for overlapping external parts..."); findOverlappingParts(); inform("Verifying external parts..."); verifyExternalParts(); inform("Checking size treshold changes..."); checkSizeTreshold(); inform("Looking for dirty objects..."); findDirtyObjects(); inform("Looking for rid-duplicates not matching the content mime-type of the parent collection"); findRIDDuplicates(); inform("Migrating parts to new cache hierarchy..."); migrateToLevelledCacheHierarchy(); inform("Checking search index consistency..."); findOrphanSearchIndexEntries(); inform("Flushing collection statistics memory cache..."); CollectionStatistics::self()->expireCache(); /* TODO some ideas for further checks: * the collection tree is non-cyclic * content type constraints of collections are not violated * find unused flags * find unused mimetypes * check for dead entries in relation tables * check if part size matches file size */ inform("Consistency check done."); Q_EMIT done(); } qint64 StorageJanitor::lostAndFoundCollection() { if (m_lostFoundCollectionId > 0) { return m_lostFoundCollectionId; } Transaction transaction(DataStore::self(), QStringLiteral("JANITOR LOST+FOUND")); Resource lfRes = Resource::retrieveByName(QStringLiteral("akonadi_lost+found_resource")); if (!lfRes.isValid()) { lfRes.setName(QStringLiteral("akonadi_lost+found_resource")); if (!lfRes.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found resource!"; } } Collection lfRoot; SelectQueryBuilder qb; qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::Equals, lfRes.id()); qb.addValueCondition(Collection::parentIdFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { qCCritical(AKONADISERVER_LOG) << "Failed to query top level collections"; return -1; } const Collection::List cols = qb.result(); if (cols.size() > 1) { qCCritical(AKONADISERVER_LOG) << "More than one top-level lost+found collection!?"; } else if (cols.size() == 1) { lfRoot = cols.first(); } else { lfRoot.setName(QStringLiteral("lost+found")); lfRoot.setResourceId(lfRes.id()); lfRoot.setCachePolicyLocalParts(QStringLiteral("ALL")); lfRoot.setCachePolicyCacheTimeout(-1); lfRoot.setCachePolicyInherit(false); if (!lfRoot.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found root."; } DataStore::self()->notificationCollector()->collectionAdded(lfRoot, lfRes.name().toUtf8()); } Collection lfCol; lfCol.setName(QDateTime::currentDateTime().toString(QStringLiteral("yyyy-MM-dd hh:mm:ss"))); lfCol.setResourceId(lfRes.id()); lfCol.setParentId(lfRoot.id()); if (!lfCol.insert()) { qCCritical(AKONADISERVER_LOG) << "Failed to create lost+found collection!"; } Q_FOREACH (const MimeType &mt, MimeType::retrieveAll()) { lfCol.addMimeType(mt); } DataStore::self()->notificationCollector()->collectionAdded(lfCol, lfRes.name().toUtf8()); transaction.commit(); m_lostFoundCollectionId = lfCol.id(); return m_lostFoundCollectionId; } void StorageJanitor::findOrphanedResources() { SelectQueryBuilder qbres; OrgFreedesktopAkonadiAgentManagerInterface iface( DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus(), this); if (!iface.isValid()) { inform(QStringLiteral("ERROR: Couldn't talk to %1").arg(DBus::Control)); return; } const QStringList knownResources = iface.agentInstances(); if (knownResources.isEmpty()) { inform(QStringLiteral("ERROR: no known resources. This must be a mistake?")); return; } qbres.addValueCondition(Resource::nameFullColumnName(), Query::NotIn, QVariant(knownResources)); qbres.addValueCondition(Resource::idFullColumnName(), Query::NotEquals, 1); // skip akonadi_search_resource if (!qbres.exec()) { inform("Failed to query known resources, skipping test"); return; } //qCDebug(AKONADISERVER_LOG) << "SQL:" << qbres.query().lastQuery(); const Resource::List orphanResources = qbres.result(); const int orphanResourcesSize(orphanResources.size()); if (orphanResourcesSize > 0) { QStringList resourceNames; resourceNames.reserve(orphanResourcesSize); for (const Resource &resource : orphanResources) { resourceNames.append(resource.name()); } inform(QStringLiteral("Found %1 orphan resources: %2").arg(orphanResourcesSize). arg(resourceNames.join(QLatin1Char(',')))); for (const QString &resourceName : qAsConst(resourceNames)) { inform(QStringLiteral("Removing resource %1").arg(resourceName)); ResourceManager::self()->removeResourceInstance(resourceName); } } } void StorageJanitor::findOrphanedCollections() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName()); qb.addValueCondition(Resource::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned collections, skipping test"); return; } const Collection::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan collections.")); // TODO: attach to lost+found resource } } void StorageJanitor::checkPathToRoot(const Collection &col) { if (col.parentId() == 0) { return; } const Collection parent = col.parent(); if (!parent.isValid()) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") has no valid parent.")); // TODO fix that by attaching to a top-level lost+found folder return; } if (col.resourceId() != parent.resourceId()) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") belongs to a different resource than its parent.")); // can/should we actually fix that? } checkPathToRoot(parent); } void StorageJanitor::findOrphanedItems() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName()); qb.addValueCondition(Collection::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned items, skipping test"); return; } const PimItem::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan items.")); // Attach to lost+found collection Transaction transaction(DataStore::self(), QStringLiteral("JANITOR ORPHANS")); QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update); qint64 col = lostAndFoundCollection(); if (col == -1) { return; } - qb.setColumnValue(PimItem::collectionIdFullColumnName(), col); + qb.setColumnValue(PimItem::collectionIdColumn(), col); QVector imapIds; imapIds.reserve(orphans.count()); for (const PimItem &item : qAsConst(orphans)) { imapIds.append(item.id()); } ImapSet set; set.add(imapIds); QueryHelper::setToQuery(set, PimItem::idFullColumnName(), qb); if (qb.exec() && transaction.commit()) { inform(QLatin1Literal("Moved orphan items to collection ") + QString::number(col)); } else { inform(QLatin1Literal("Error moving orphan items to collection ") + QString::number(col) + QLatin1Literal(" : ") + qb.query().lastError().text()); } } } void StorageJanitor::findOrphanedParts() { SelectQueryBuilder qb; qb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName()); qb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant()); if (!qb.exec()) { inform("Failed to query orphaned parts, skipping test"); return; } const Part::List orphans = qb.result(); if (!orphans.isEmpty()) { inform(QLatin1Literal("Found ") + QString::number(orphans.size()) + QLatin1Literal(" orphan parts.")); // TODO: create lost+found items for those? delete? } } void StorageJanitor:: findOrphanedPimItemFlags() { QueryBuilder sqb(PimItemFlagRelation::tableName(), QueryBuilder::Select); sqb.addColumn(PimItemFlagRelation::leftFullColumnName()); sqb.addJoin(QueryBuilder::LeftJoin, PimItem::tableName(), PimItemFlagRelation::leftFullColumnName(), PimItem::idFullColumnName()); sqb.addValueCondition(PimItem::idFullColumnName(), Query::Is, QVariant()); if (!sqb.exec()) { inform("Failed to query orphaned item flags, skipping test"); return; } QVector imapIds; int count = 0; while (sqb.query().next()) { ++count; imapIds.append(sqb.query().value(0).toInt()); } sqb.query().finish(); if (count > 0) { ImapSet set; set.add(imapIds); QueryBuilder qb(PimItemFlagRelation::tableName(), QueryBuilder::Delete); QueryHelper::setToQuery(set, PimItemFlagRelation::leftFullColumnName(), qb); if (!qb.exec()) { qCCritical(AKONADISERVER_LOG) << "Error:" << qb.query().lastError().text(); return; } inform(QLatin1Literal("Found and deleted ") + QString::number(count) + QLatin1Literal(" orphan pim item flags.")); } } void StorageJanitor::findOverlappingParts() { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataColumn()); qb.addColumn(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(") as cnt")); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant()); qb.addGroupColumn(Part::dataColumn()); qb.addValueCondition(QLatin1Literal("count(") + Part::idColumn() + QLatin1Literal(")"), Query::Greater, 1, QueryBuilder::HavingCondition); if (!qb.exec()) { inform("Failed to query overlapping parts, skipping test"); return; } int count = 0; while (qb.query().next()) { ++count; inform(QLatin1Literal("Found overlapping part data: ") + qb.query().value(0).toString()); // TODO: uh oh, this is bad, how do we recover from that? } qb.query().finish(); if (count > 0) { inform(QLatin1Literal("Found ") + QString::number(count) + QLatin1Literal(" overlapping parts - bad.")); } } void StorageJanitor::verifyExternalParts() { QSet existingFiles; QSet usedFiles; // list all files const QString dataDir = StandardDirs::saveDir("data", QStringLiteral("file_db_data")); QDirIterator it(dataDir, QDir::Files, QDirIterator::Subdirectories); while (it.hasNext()) { existingFiles.insert(it.next()); } existingFiles.remove(dataDir + QDir::separator() + QLatin1Char('.')); existingFiles.remove(dataDir + QDir::separator() + QLatin1String("..")); inform(QLatin1Literal("Found ") + QString::number(existingFiles.size()) + QLatin1Literal(" external files.")); // list all parts from the db which claim to have an associated file QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::dataColumn()); qb.addColumn(Part::pimItemIdColumn()); qb.addColumn(Part::idColumn()); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); qb.addValueCondition(Part::dataColumn(), Query::IsNot, QVariant()); if (!qb.exec()) { inform("Failed to query existing parts, skipping test"); return; } while (qb.query().next()) { const auto filename = qb.query().value(0).toByteArray(); const Entity::Id pimItemId = qb.query().value(1).value(); const Entity::Id partId = qb.query().value(2).value(); QString partPath; if (!filename.isEmpty()) { partPath = ExternalPartStorage::resolveAbsolutePath(filename); } else { partPath = ExternalPartStorage::resolveAbsolutePath(ExternalPartStorage::nameForPartId(partId)); } if (existingFiles.contains(partPath)) { usedFiles.insert(partPath); } else { inform(QLatin1Literal("Cleaning up missing external file: ") + partPath + QLatin1Literal(" for item: ") + QString::number(pimItemId) + QLatin1Literal(" on part: ") + QString::number(partId)); Part part; part.setId(partId); part.setPimItemId(pimItemId); part.setData(QByteArray()); part.setDatasize(0); part.setStorage(Part::Internal); part.update(); } } qb.query().finish(); inform(QLatin1Literal("Found ") + QString::number(usedFiles.size()) + QLatin1Literal(" external parts.")); // see what's left and move it to lost+found const QSet unreferencedFiles = existingFiles - usedFiles; if (!unreferencedFiles.isEmpty()) { const QString lfDir = StandardDirs::saveDir("data", QStringLiteral("file_lost+found")); for (const QString &file : unreferencedFiles) { inform(QLatin1Literal("Found unreferenced external file: ") + file); const QFileInfo f(file); QFile::rename(file, lfDir + QDir::separator() + f.fileName()); } inform(QStringLiteral("Moved %1 unreferenced files to lost+found.").arg(unreferencedFiles.size())); } else { inform("Found no unreferenced external files."); } } void StorageJanitor::findDirtyObjects() { SelectQueryBuilder cqb; cqb.setSubQueryMode(Query::Or); cqb.addValueCondition(Collection::remoteIdColumn(), Query::Is, QVariant()); cqb.addValueCondition(Collection::remoteIdColumn(), Query::Equals, QString()); if (!cqb.exec()) { inform("Failed to query collections without RID, skipping test"); return; } const Collection::List ridLessCols = cqb.result(); for (const Collection &col : ridLessCols) { inform(QLatin1Literal("Collection \"") + col.name() + QLatin1Literal("\" (id: ") + QString::number(col.id()) + QLatin1Literal(") has no RID.")); } inform(QLatin1Literal("Found ") + QString::number(ridLessCols.size()) + QLatin1Literal(" collections without RID.")); SelectQueryBuilder iqb1; iqb1.setSubQueryMode(Query::Or); iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Is, QVariant()); iqb1.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, QString()); if (!iqb1.exec()) { inform("Failed to query items without RID, skipping test"); return; } const PimItem::List ridLessItems = iqb1.result(); for (const PimItem &item : ridLessItems) { inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" in collection \"") + QString::number(item.collectionId()) + QLatin1Literal("\" has no RID.")); } inform(QLatin1Literal("Found ") + QString::number(ridLessItems.size()) + QLatin1Literal(" items without RID.")); SelectQueryBuilder iqb2; iqb2.addValueCondition(PimItem::dirtyColumn(), Query::Equals, true); iqb2.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant()); iqb2.addSortColumn(PimItem::idFullColumnName()); if (!iqb2.exec()) { inform("Failed to query dirty items, skipping test"); return; } const PimItem::List dirtyItems = iqb2.result(); for (const PimItem &item : dirtyItems) { inform(QLatin1Literal("Item \"") + QString::number(item.id()) + QLatin1Literal("\" has RID and is dirty.")); } inform(QLatin1Literal("Found ") + QString::number(dirtyItems.size()) + QLatin1Literal(" dirty items.")); } void StorageJanitor::findRIDDuplicates() { QueryBuilder qb(Collection::tableName(), QueryBuilder::Select); qb.addColumn(Collection::idColumn()); qb.addColumn(Collection::nameColumn()); qb.exec(); while (qb.query().next()) { const Collection::Id colId = qb.query().value(0).value(); const QString name = qb.query().value(1).toString(); inform(QStringLiteral("Checking ") + name); QueryBuilder duplicates(PimItem::tableName(), QueryBuilder::Select); duplicates.addColumn(PimItem::remoteIdColumn()); duplicates.addColumn(QStringLiteral("count(") + PimItem::idColumn() + QStringLiteral(") as cnt")); duplicates.addValueCondition(PimItem::remoteIdColumn(), Query::IsNot, QVariant()); duplicates.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); duplicates.addGroupColumn(PimItem::remoteIdColumn()); duplicates.addValueCondition(QStringLiteral("count(") + PimItem::idColumn() + QLatin1Char(')'), Query::Greater, 1, QueryBuilder::HavingCondition); duplicates.exec(); Akonadi::Server::Collection col = Akonadi::Server::Collection::retrieveById(colId); const QVector contentMimeTypes = col.mimeTypes(); QVariantList contentMimeTypesVariantList; contentMimeTypesVariantList.reserve(contentMimeTypes.count()); for (const Akonadi::Server::MimeType &mimeType : contentMimeTypes) { contentMimeTypesVariantList << mimeType.id(); } while (duplicates.query().next()) { const QString rid = duplicates.query().value(0).toString(); Query::Condition condition(Query::And); condition.addValueCondition(PimItem::remoteIdColumn(), Query::Equals, rid); condition.addValueCondition(PimItem::mimeTypeIdColumn(), Query::NotIn, contentMimeTypesVariantList); condition.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); QueryBuilder items(PimItem::tableName(), QueryBuilder::Select); items.addColumn(PimItem::idColumn()); items.addCondition(condition); if (!items.exec()) { inform(QStringLiteral("Error while deleting duplicates: ") + items.query().lastError().text()); continue; } QVariantList itemsIds; while (items.query().next()) { itemsIds.push_back(items.query().value(0)); } items.query().finish(); if (itemsIds.isEmpty()) { // the mimetype filter may have dropped some entries from the // duplicates query continue; } inform(QStringLiteral("Found duplicates ") + rid); SelectQueryBuilder parts; parts.addValueCondition(Part::pimItemIdFullColumnName(), Query::In, QVariant::fromValue(itemsIds)); parts.addValueCondition(Part::storageFullColumnName(), Query::Equals, (int) Part::External); if (parts.exec()) { const auto partsList = parts.result(); for (const auto &part : partsList) { bool exists = false; const auto filename = ExternalPartStorage::resolveAbsolutePath(part.data(), &exists); if (exists) { QFile::remove(filename); } } } items = QueryBuilder(PimItem::tableName(), QueryBuilder::Delete); items.addCondition(condition); if (!items.exec()) { inform(QStringLiteral("Error while deleting duplicates ") + items.query().lastError().text()); } } duplicates.query().finish(); } qb.query().finish(); } void StorageJanitor::vacuum() { const DbType::Type dbType = DbType::type(DataStore::self()->database()); if (dbType == DbType::MySQL || dbType == DbType::PostgreSQL) { inform("vacuuming database, that'll take some time and require a lot of temporary disk space..."); Q_FOREACH (const QString &table, allDatabaseTables()) { inform(QStringLiteral("optimizing table %1...").arg(table)); QString queryStr; if (dbType == DbType::MySQL) { queryStr = QLatin1Literal("OPTIMIZE TABLE ") + table; } else if (dbType == DbType::PostgreSQL) { queryStr = QLatin1Literal("VACUUM FULL ANALYZE ") + table; } else { continue; } QSqlQuery q(DataStore::self()->database()); if (!q.exec(queryStr)) { qCCritical(AKONADISERVER_LOG) << "failed to optimize table" << table << ":" << q.lastError().text(); } } inform("vacuum done"); } else { inform("Vacuum not supported for this database backend."); } Q_EMIT done(); } void StorageJanitor::checkSizeTreshold() { { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idFullColumnName()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::Internal); qb.addValueCondition(Part::datasizeFullColumnName(), Query::Greater, DbConfig::configuredDatabase()->sizeThreshold()); if (!qb.exec()) { inform("Failed to query parts larger than treshold, skipping test"); return; } QSqlQuery query = qb.query(); inform(QStringLiteral("Found %1 parts to be moved to external files").arg(query.size())); while (query.next()) { Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD")); Part part = Part::retrieveById(query.value(0).toLongLong()); const QByteArray name = ExternalPartStorage::nameForPartId(part.id()); const QString partPath = ExternalPartStorage::resolveAbsolutePath(name); QFile f(partPath); if (f.exists()) { qCDebug(AKONADISERVER_LOG) << "External payload file" << name << "already exists"; // That however is not a critical issue, since the part is not external, // so we can safely overwrite it } if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) { qCCritical(AKONADISERVER_LOG) << "Failed to open file" << name << "for writing"; continue; } if (f.write(part.data()) != part.datasize()) { qCCritical(AKONADISERVER_LOG) << "Failed to write data to payload file" << name; f.remove(); continue; } part.setData(name); part.setStorage(Part::External); if (!part.update() || !transaction.commit()) { qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id(); f.remove(); continue; } inform(QStringLiteral("Moved part %1 from database into external file %2").arg(part.id()).arg(QString::fromLatin1(name))); } query.finish(); } { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idFullColumnName()); qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External); qb.addValueCondition(Part::datasizeFullColumnName(), Query::Less, DbConfig::configuredDatabase()->sizeThreshold()); if (!qb.exec()) { inform("Failed to query parts smaller than treshold, skipping test"); return; } QSqlQuery query = qb.query(); inform(QStringLiteral("Found %1 parts to be moved to database").arg(query.size())); while (query.next()) { Transaction transaction(DataStore::self(), QStringLiteral("JANITOR CHECK SIZE THRESHOLD 2")); Part part = Part::retrieveById(query.value(0).toLongLong()); const QString partPath = ExternalPartStorage::resolveAbsolutePath(part.data()); QFile f(partPath); if (!f.exists()) { qCCritical(AKONADISERVER_LOG) << "Part file" << part.data() << "does not exist"; continue; } if (!f.open(QIODevice::ReadOnly)) { qCCritical(AKONADISERVER_LOG) << "Failed to open part file" << part.data() << "for reading"; continue; } part.setStorage(Part::Internal); part.setData(f.readAll()); if (part.data().size() != part.datasize()) { qCCritical(AKONADISERVER_LOG) << "Sizes of" << part.id() << "data don't match"; continue; } if (!part.update() || !transaction.commit()) { qCCritical(AKONADISERVER_LOG) << "Failed to update database entry of part" << part.id(); continue; } f.close(); f.remove(); inform(QStringLiteral("Moved part %1 from external file into database").arg(part.id())); } query.finish(); } } void StorageJanitor::migrateToLevelledCacheHierarchy() { QueryBuilder qb(Part::tableName(), QueryBuilder::Select); qb.addColumn(Part::idColumn()); qb.addColumn(Part::dataColumn()); qb.addValueCondition(Part::storageColumn(), Query::Equals, Part::External); if (!qb.exec()) { inform("Failed to query external payload parts, skipping test"); return; } QSqlQuery query = qb.query(); while (query.next()) { const qint64 id = query.value(0).toLongLong(); const QByteArray data = query.value(1).toByteArray(); const QString fileName = QString::fromUtf8(data); bool oldExists = false, newExists = false; // Resolve the current path const QString currentPath = ExternalPartStorage::resolveAbsolutePath(fileName, &oldExists); // Resolve the new path with legacy fallback disabled, so that it always // returns the new levelled-cache path, even when the old one exists const QString newPath = ExternalPartStorage::resolveAbsolutePath(fileName, &newExists, false); if (!oldExists) { qCCritical(AKONADISERVER_LOG) << "Old payload part does not exist, skipping part" << fileName; continue; } if (currentPath != newPath) { if (newExists) { qCCritical(AKONADISERVER_LOG) << "Part is in legacy location, but the destination file already exists, skipping part" << fileName; continue; } QFile f(currentPath); if (!f.rename(newPath)) { qCCritical(AKONADISERVER_LOG) << "Failed to move part from" << currentPath << " to " << newPath << ":" << f.errorString(); continue; } inform(QStringLiteral("Migrated part %1 to new levelled cache").arg(id)); } } query.finish(); } void StorageJanitor::findOrphanSearchIndexEntries() { QueryBuilder qb(Collection::tableName(), QueryBuilder::Select); qb.addSortColumn(Collection::idColumn(), Query::Ascending); qb.addColumn(Collection::idColumn()); qb.addColumn(Collection::isVirtualColumn()); if (!qb.exec()) { inform("Failed to query collections, skipping test"); return; } QDBusInterface iface(DBus::agentServiceName(QStringLiteral("akonadi_indexing_agent"), DBus::Agent), QStringLiteral("/"), QStringLiteral("org.freedesktop.Akonadi.Indexer"), DBusConnectionPool::threadConnection()); if (!iface.isValid()) { inform("Akonadi Indexing Agent is not running, skipping test"); return; } QSqlQuery query = qb.query(); while (query.next()) { const qint64 colId = query.value(0).toLongLong(); // Skip virtual collections, they are not indexed if (query.value(1).toBool()) { inform(QStringLiteral("Skipping virtual Collection %1").arg(colId)); continue; } inform(QStringLiteral("Checking Collection %1 search index...").arg(colId)); SearchRequest req("StorageJanitor"); req.setStoreResults(true); req.setCollections({ colId }); req.setRemoteSearch(false); req.setQuery(QStringLiteral("{ }")); // empty query to match all QStringList mts; Collection col; col.setId(colId); const auto colMts = col.mimeTypes(); if (colMts.isEmpty()) { // No mimetypes means we don't know which search store to look into, // skip it. continue; } mts.reserve(colMts.count()); for (const auto &mt : colMts) { mts << mt.name(); } req.setMimeTypes(mts); req.exec(); auto searchResults = req.results(); QueryBuilder iqb(PimItem::tableName(), QueryBuilder::Select); iqb.addColumn(PimItem::idColumn()); iqb.addValueCondition(PimItem::collectionIdColumn(), Query::Equals, colId); if (!iqb.exec()) { inform(QStringLiteral("Failed to query items in collection %1").arg(colId)); continue; } QSqlQuery itemQuery = iqb.query(); while (itemQuery.next()) { searchResults.remove(itemQuery.value(0).toLongLong()); } itemQuery.finish(); if (!searchResults.isEmpty()) { inform(QStringLiteral("Collection %1 search index contains %2 orphan items. Scheduling reindexing").arg(colId).arg(searchResults.count())); iface.call(QDBus::NoBlock, QStringLiteral("reindexCollection"), colId); } } query.finish(); } void StorageJanitor::inform(const char *msg) { inform(QLatin1String(msg)); } void StorageJanitor::inform(const QString &msg) { qCDebug(AKONADISERVER_LOG) << msg; Q_EMIT information(msg); }