diff --git a/src/engine/database.cpp b/src/engine/database.cpp index b670f7a6..3bc29e24 100644 --- a/src/engine/database.cpp +++ b/src/engine/database.cpp @@ -1,237 +1,240 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * Copyright (C) 2016 Christoph Cullmann * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "database.h" #include "transaction.h" #include "postingdb.h" #include "documentdb.h" #include "documenturldb.h" #include "documentiddb.h" #include "positiondb.h" #include "documenttimedb.h" #include "documentdatadb.h" #include "mtimedb.h" #include "document.h" #include "enginequery.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "phraseanditerator.h" #include "writetransaction.h" #include "idutils.h" #include "fsutils.h" +#include "enginedebug.h" + #include #include #include #include using namespace Baloo; Database::Database(const QString& path) : m_path(path) , m_env(nullptr) { } Database::~Database() { // try only to close if we did open the DB successfully if (m_env) { mdb_env_close(m_env); m_env = nullptr; } } bool Database::open(OpenMode mode) { QMutexLocker locker(&m_mutex); // nop if already open! if (m_env) { return true; } QDir dir(m_path); if (!dir.exists()) { dir.mkpath(QStringLiteral(".")); dir.refresh(); } QFileInfo indexInfo(dir, QStringLiteral("index")); if ((mode != CreateDatabase) && !indexInfo.exists()) { return false; } if (mode == CreateDatabase) { if (!QFileInfo(dir.absolutePath()).permission(QFile::WriteOwner)) { - qCritical() << m_path << "does not have write permissions. Aborting"; + qCCritical(ENGINE) << m_path << "does not have write permissions. Aborting"; return false; } if (!indexInfo.exists()) { FSUtils::disableCoW(m_path); } } int rc = mdb_env_create(&m_env); if (rc) { m_env = nullptr; return false; } /** * maximal number of allowed named databases, must match number of databases we create below * each additional one leads to overhead */ mdb_env_set_maxdbs(m_env, 12); /** * size limit for database == size limit of mmap * use 1 GB on 32-bit, use 256 GB on 64-bit */ const size_t maximalSizeInBytes = size_t((sizeof(size_t) == 4) ? 1 : 256) * size_t(1024) * size_t(1024) * size_t(1024); mdb_env_set_mapsize(m_env, maximalSizeInBytes); // The directory needs to be created before opening the environment QByteArray arr = QFile::encodeName(indexInfo.absoluteFilePath()); rc = mdb_env_open(m_env, arr.constData(), MDB_NOSUBDIR | MDB_NOMEMINIT | ((mode == ReadOnlyDatabase) ? MDB_RDONLY : 0), 0664); if (rc) { mdb_env_close(m_env); m_env = nullptr; return false; } rc = mdb_reader_check(m_env, nullptr); - Q_ASSERT_X(rc == 0, "Database::open reader_check", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "Database::open reader_check" << mdb_strerror(rc); mdb_env_close(m_env); m_env = nullptr; return false; } // // Individual Databases // MDB_txn* txn; if (mode != CreateDatabase) { int rc = mdb_txn_begin(m_env, nullptr, MDB_RDONLY, &txn); - Q_ASSERT_X(rc == 0, "Database::transaction ro begin", mdb_strerror(rc)); if (rc) { + qCWarning(ENGINE) << "Database::transaction ro begin" << mdb_strerror(rc); mdb_env_close(m_env); m_env = nullptr; return false; } m_dbis.postingDbi = PostingDB::open(txn); m_dbis.positionDBi = PositionDB::open(txn); m_dbis.docTermsDbi = DocumentDB::open("docterms", txn); m_dbis.docFilenameTermsDbi = DocumentDB::open("docfilenameterms", txn); m_dbis.docXattrTermsDbi = DocumentDB::open("docxatrrterms", txn); m_dbis.idTreeDbi = IdTreeDB::open(txn); m_dbis.idFilenameDbi = IdFilenameDB::open(txn); m_dbis.docTimeDbi = DocumentTimeDB::open(txn); m_dbis.docDataDbi = DocumentDataDB::open(txn); m_dbis.contentIndexingDbi = DocumentIdDB::open("indexingleveldb", txn); m_dbis.failedIdDbi = DocumentIdDB::open("failediddb", txn); m_dbis.mtimeDbi = MTimeDB::open(txn); - Q_ASSERT(m_dbis.isValid()); if (!m_dbis.isValid()) { + qCWarning(ENGINE) << "dbis is invalid"; mdb_txn_abort(txn); mdb_env_close(m_env); m_env = nullptr; return false; } rc = mdb_txn_commit(txn); - Q_ASSERT_X(rc == 0, "Database::transaction ro commit", mdb_strerror(rc)); if (rc) { + qCWarning(ENGINE) << "Database::transaction ro commit" << mdb_strerror(rc); mdb_env_close(m_env); m_env = nullptr; return false; } } else { int rc = mdb_txn_begin(m_env, nullptr, 0, &txn); - Q_ASSERT_X(rc == 0, "Database::transaction begin", mdb_strerror(rc)); if (rc) { + qCWarning(ENGINE) << "Database::transaction begin" << mdb_strerror(rc); mdb_env_close(m_env); m_env = nullptr; return false; } m_dbis.postingDbi = PostingDB::create(txn); m_dbis.positionDBi = PositionDB::create(txn); m_dbis.docTermsDbi = DocumentDB::create("docterms", txn); m_dbis.docFilenameTermsDbi = DocumentDB::create("docfilenameterms", txn); m_dbis.docXattrTermsDbi = DocumentDB::create("docxatrrterms", txn); m_dbis.idTreeDbi = IdTreeDB::create(txn); m_dbis.idFilenameDbi = IdFilenameDB::create(txn); m_dbis.docTimeDbi = DocumentTimeDB::create(txn); m_dbis.docDataDbi = DocumentDataDB::create(txn); m_dbis.contentIndexingDbi = DocumentIdDB::create("indexingleveldb", txn); m_dbis.failedIdDbi = DocumentIdDB::create("failediddb", txn); m_dbis.mtimeDbi = MTimeDB::create(txn); - Q_ASSERT(m_dbis.isValid()); - if (!m_dbis.isValid()) { + if (!m_dbis.isValid()) + qCWarning(ENGINE) << "dbis is invalid";{ mdb_txn_abort(txn); mdb_env_close(m_env); m_env = nullptr; return false; } rc = mdb_txn_commit(txn); - Q_ASSERT_X(rc == 0, "Database::transaction commit", mdb_strerror(rc)); if (rc) { + qCWarning(ENGINE) << "Database::transaction commit" << mdb_strerror(rc); mdb_env_close(m_env); m_env = nullptr; return false; } } Q_ASSERT(m_env); return true; } bool Database::isOpen() const { QMutexLocker locker(&m_mutex); return m_env != nullptr; } QString Database::path() const { QMutexLocker locker(&m_mutex); return m_path; } diff --git a/src/engine/documentdatadb.cpp b/src/engine/documentdatadb.cpp index 6eddb3ff..35a91f8f 100644 --- a/src/engine/documentdatadb.cpp +++ b/src/engine/documentdatadb.cpp @@ -1,149 +1,158 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "documentdatadb.h" +#include "enginedebug.h" using namespace Baloo; DocumentDataDB::DocumentDataDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } DocumentDataDB::~DocumentDataDB() { } MDB_dbi DocumentDataDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "documentdatadb", MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "DocumentUrlDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentDataDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi DocumentDataDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "documentdatadb", MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "DocumentDataDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "DocumentUrlDB::create", mdb_strerror(rc)); return dbi; } void DocumentDataDB::put(quint64 docId, const QByteArray& url) { Q_ASSERT(docId > 0); Q_ASSERT(!url.isEmpty()); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); MDB_val val; val.mv_size = url.size(); val.mv_data = static_cast(const_cast(url.constData())); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "DocumentDataDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentDataDB::put" << mdb_strerror(rc); + } } QByteArray DocumentDataDB::get(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentDataDB::get" << docId << mdb_strerror(rc); + } return QByteArray(); } - Q_ASSERT_X(rc == 0, "DocumentDataDB::get", mdb_strerror(rc)); return QByteArray(static_cast(val.mv_data), val.mv_size); } void DocumentDataDB::del(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentDataDB::del" << docId << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "DocumentUrlDB::del", mdb_strerror(rc)); } bool DocumentDataDB::contains(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentDataDB::contains" << docId << mdb_strerror(rc); + } return false; } - Q_ASSERT_X(rc == 0, "DocumentDataDB::contains", mdb_strerror(rc)); return true; } QMap DocumentDataDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "DocumentDataDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "DocumentDataDB::toTestMap", mdb_strerror(rc)); const quint64 id = *(static_cast(key.mv_data)); const QByteArray ba(static_cast(val.mv_data), val.mv_size); map.insert(id, ba); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/documentdb.cpp b/src/engine/documentdb.cpp index 75819016..fceda60d 100644 --- a/src/engine/documentdb.cpp +++ b/src/engine/documentdb.cpp @@ -1,171 +1,179 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "documentdb.h" #include "doctermscodec.h" - -#include +#include "enginedebug.h" using namespace Baloo; DocumentDB::DocumentDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } DocumentDB::~DocumentDB() { } MDB_dbi DocumentDB::create(const char* name, MDB_txn* txn) { - MDB_dbi dbi; - int rc = mdb_dbi_open(txn, name, MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "DocumentDB::create", mdb_strerror(rc)); + MDB_dbi dbi = 0; + const int rc = mdb_dbi_open(txn, name, MDB_CREATE | MDB_INTEGERKEY, &dbi); + if (rc) { + qCWarning(ENGINE) << "DocumentDB::create" << name << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi DocumentDB::open(const char* name, MDB_txn* txn) { - MDB_dbi dbi; - int rc = mdb_dbi_open(txn, name, MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + MDB_dbi dbi = 0; + const int rc = mdb_dbi_open(txn, name, MDB_INTEGERKEY, &dbi); + if (rc) { + qCWarning(ENGINE) << "DocumentDB::open" << name << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "DocumentDB::open", mdb_strerror(rc)); return dbi; } void DocumentDB::put(quint64 docId, const QVector& list) { Q_ASSERT(docId > 0); Q_ASSERT(!list.isEmpty()); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); DocTermsCodec codec; QByteArray arr = codec.encode(list); MDB_val val; val.mv_size = arr.size(); val.mv_data = static_cast(arr.data()); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "DocumentDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentDB::put" << mdb_strerror(rc); + } } QVector DocumentDB::get(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "DocumentDB::get" << docId << mdb_strerror(rc); return QVector(); } - Q_ASSERT_X(rc == 0, "DocumentDB::get", mdb_strerror(rc)); QByteArray arr = QByteArray::fromRawData(static_cast(val.mv_data), val.mv_size); DocTermsCodec codec; auto result = codec.decode(arr); if (result.isEmpty()) { - qDebug() << "Document Terms DB contains corrupt data for " << docId; + qCDebug(ENGINE) << "Document Terms DB contains corrupt data for " << docId; } return result; } void DocumentDB::del(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentDB::del" << docId << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "DocumentDB::del", mdb_strerror(rc)); } bool DocumentDB::contains(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); MDB_val val; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentDB::contains" << docId << mdb_strerror(rc); + } return false; } - Q_ASSERT_X(rc == 0, "DocumentDB::contains", mdb_strerror(rc)); return true; } uint DocumentDB::size() { MDB_stat stat; int rc = mdb_stat(m_txn, m_dbi, &stat); - Q_ASSERT_X(rc == 0, "DocumentDB::size", mdb_strerror(rc)); + if (rc) { + qCDebug(ENGINE) << "DocumentDB::size" << mdb_strerror(rc); + return 0; + } return stat.ms_entries; } QMap> DocumentDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap> map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); const quint64 id = *(static_cast(key.mv_data)); const QVector vec = DocTermsCodec().decode(QByteArray(static_cast(val.mv_data), val.mv_size)); map.insert(id, vec); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/documentiddb.cpp b/src/engine/documentiddb.cpp index e7ce9973..6e6dfe7f 100644 --- a/src/engine/documentiddb.cpp +++ b/src/engine/documentiddb.cpp @@ -1,165 +1,173 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "documentiddb.h" - -#include +#include "enginedebug.h" using namespace Baloo; DocumentIdDB::DocumentIdDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } DocumentIdDB::~DocumentIdDB() { } MDB_dbi DocumentIdDB::create(const char* name, MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, name, MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "DocumentIdDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentIdDB::create" << name << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi DocumentIdDB::open(const char* name, MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, name, MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "DocumentIdDB::open" << name << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "DocumentIdDB::create", mdb_strerror(rc)); return dbi; } void DocumentIdDB::put(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); MDB_val val; val.mv_size = 0; val.mv_data = nullptr; int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "DocumentIdDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentIdDB::put" << mdb_strerror(rc); + } } bool DocumentIdDB::contains(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentIdDB::contains" << docId << mdb_strerror(rc); + } return false; } - Q_ASSERT_X(rc == 0, "DocumentIdDB::contains", mdb_strerror(rc)); return true; } void DocumentIdDB::del(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentIdDB::del" << docId << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "DocumentIdDB::del", mdb_strerror(rc)); } QVector DocumentIdDB::fetchItems(int size) { Q_ASSERT(size > 0); MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); QVector vec; for (int i = 0; i < size; i++) { - MDB_val key; + MDB_val key{0, nullptr}; int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "DocumentIdDB::fetchItems" << size << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "DocumentIdDB::fetchItems", mdb_strerror(rc)); quint64 id = *(static_cast(key.mv_data)); vec << id; } mdb_cursor_close(cursor); return vec; } uint DocumentIdDB::size() { MDB_stat stat; int rc = mdb_stat(m_txn, m_dbi, &stat); - Q_ASSERT_X(rc == 0, "DocumentIdDB::size", mdb_strerror(rc)); + if (rc) { + qCDebug(ENGINE) << "DocumentIdDB::size" << mdb_strerror(rc); + return 0; + } return stat.ms_entries; } QVector DocumentIdDB::toTestVector() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); - MDB_val key = {0, nullptr}; - MDB_val val; + MDB_val key{0, nullptr}; + MDB_val val{0, nullptr}; QVector vec; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "DocumentTimeDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::toTestMap", mdb_strerror(rc)); const quint64 id = *(static_cast(key.mv_data)); vec << id; } mdb_cursor_close(cursor); return vec; } diff --git a/src/engine/documenttimedb.cpp b/src/engine/documenttimedb.cpp index a8a5e8e4..6ee9b96b 100644 --- a/src/engine/documenttimedb.cpp +++ b/src/engine/documenttimedb.cpp @@ -1,148 +1,157 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "documenttimedb.h" +#include "enginedebug.h" using namespace Baloo; DocumentTimeDB::DocumentTimeDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } DocumentTimeDB::~DocumentTimeDB() { } MDB_dbi DocumentTimeDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "documenttimedb", MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "DocumentTimeDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentTimeDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi DocumentTimeDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "documenttimedb", MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "DocumentTimeDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::create", mdb_strerror(rc)); return dbi; } void DocumentTimeDB::put(quint64 docId, const TimeInfo& info) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = &docId; MDB_val val; val.mv_size = sizeof(TimeInfo); val.mv_data = static_cast(const_cast(&info)); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "DocumentTimeDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "DocumentTimeDB::put" << docId << mdb_strerror(rc); + } } DocumentTimeDB::TimeInfo DocumentTimeDB::get(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = &docId; - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentTimeDB::get" << docId << mdb_strerror(rc); + } return TimeInfo(); } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::get", mdb_strerror(rc)); return *(static_cast(val.mv_data)); } void DocumentTimeDB::del(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentTimeDB::del" << docId << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::del", mdb_strerror(rc)); } bool DocumentTimeDB::contains(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); MDB_val val; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "DocumentTimeDB::contains" << docId << mdb_strerror(rc); + } return false; } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::contains", mdb_strerror(rc)); return true; } QMap DocumentTimeDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "DocumentTimeDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "DocumentTimeDB::toTestMap", mdb_strerror(rc)); const quint64 id = *(static_cast(key.mv_data)); const TimeInfo ti = *(static_cast(val.mv_data)); map.insert(id, ti); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/documenturldb.cpp b/src/engine/documenturldb.cpp index fb35edd0..7c3c4f95 100644 --- a/src/engine/documenturldb.cpp +++ b/src/engine/documenturldb.cpp @@ -1,228 +1,227 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "documenturldb.h" #include "idutils.h" #include "postingiterator.h" #include #include -#include using namespace Baloo; DocumentUrlDB::DocumentUrlDB(MDB_dbi idTreeDb, MDB_dbi idFilenameDb, MDB_txn* txn) : m_txn(txn) , m_idFilenameDbi(idFilenameDb) , m_idTreeDbi(idTreeDb) { } DocumentUrlDB::~DocumentUrlDB() { } bool DocumentUrlDB::put(quint64 docId, const QByteArray& url) { if (!docId || url.isEmpty() || url.endsWith('/')) { return false; } IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); typedef QPair IdNamePath; QByteArray arr = url; // // id and parent // { quint64 id = filePathToId(arr); if (!id) { return false; } Q_ASSERT(id == docId); int pos = arr.lastIndexOf('/'); QByteArray name = arr.mid(pos + 1); arr.resize(pos); quint64 parentId = filePathToId(arr); add(id, parentId, name); if (idFilenameDb.contains(parentId)) return true; } // // The rest of the path // QVector list; while (!arr.isEmpty()) { quint64 id = filePathToId(arr); Q_ASSERT(id); int pos = arr.lastIndexOf('/'); QByteArray name = arr.mid(pos + 1); list.prepend(qMakePair(id, name)); arr.resize(pos); } for (int i = 0; i < list.size(); i++) { quint64 id = list[i].first; QByteArray name = list[i].second; // Update the IdTree quint64 parentId = 0; if (i) { parentId = list[i-1].first; } add(id, parentId, name); } return true; } void DocumentUrlDB::add(quint64 id, quint64 parentId, const QByteArray& name) { if (!id || name.isEmpty()) { return; } IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); IdTreeDB idTreeDb(m_idTreeDbi, m_txn); QVector subDocs = idTreeDb.get(parentId); // insert if not there sortedIdInsert(subDocs, id); idTreeDb.put(parentId, subDocs); // Update the IdFileName IdFilenameDB::FilePath path; path.parentId = parentId; path.name = name; idFilenameDb.put(id, path); } QByteArray DocumentUrlDB::get(quint64 docId) const { if (!docId) { return QByteArray(); } IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); auto path = idFilenameDb.get(docId); if (path.name.isEmpty()) { return QByteArray(); } QByteArray ret = path.name; quint64 id = path.parentId; // arbitrary path depth limit - we have to deal with // possibly corrupted DBs out in the wild int depth_limit = 512; while (id) { auto p = idFilenameDb.get(id); if (p.name.isEmpty()) { return QByteArray(); } if (!depth_limit--) { return QByteArray(); } ret = p.name + '/' + ret; id = p.parentId; } return '/' + ret; } QVector DocumentUrlDB::getChildren(quint64 docId) const { IdTreeDB idTreeDb(m_idTreeDbi, m_txn); return idTreeDb.get(docId); } void DocumentUrlDB::rename(quint64 docId, const QByteArray& newFileName) { if (!docId || newFileName.isEmpty()) { return; } IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); auto path = idFilenameDb.get(docId); path.name = newFileName; idFilenameDb.put(docId, path); } quint64 DocumentUrlDB::getId(quint64 docId, const QByteArray& fileName) const { if (fileName.isEmpty()) { return 0; } IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); IdTreeDB idTreeDb(m_idTreeDbi, m_txn); QVector subFiles = idTreeDb.get(docId); for (quint64 id : subFiles) { IdFilenameDB::FilePath path = idFilenameDb.get(id); if (path.name == fileName) { return id; } } return 0; } QMap DocumentUrlDB::toTestMap() const { IdTreeDB idTreeDb(m_idTreeDbi, m_txn); QMap> idTreeMap = idTreeDb.toTestMap(); QSet allIds; for (auto it = idTreeMap.cbegin(); it != idTreeMap.cend(); it++) { allIds.insert(it.key()); for (quint64 id : it.value()) { allIds.insert(id); } } QMap map; for (quint64 id : allIds) { if (id) { QByteArray path = get(id); //FIXME: this prevents sanitizing // reactivate Q_ASSERT(!path.isEmpty()); map.insert(id, path); } } return map; } diff --git a/src/engine/documenturldb.h b/src/engine/documenturldb.h index eb42e1cd..bd96c26f 100644 --- a/src/engine/documenturldb.h +++ b/src/engine/documenturldb.h @@ -1,158 +1,159 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_DOCUMENTURLDB_H #define BALOO_DOCUMENTURLDB_H #include "idtreedb.h" #include "idfilenamedb.h" #include #include namespace Baloo { class UrlTest; class PostingIterator; class BALOO_ENGINE_EXPORT DocumentUrlDB { public: explicit DocumentUrlDB(MDB_dbi idTreeDb, MDB_dbi idFileNameDb, MDB_txn* txn); ~DocumentUrlDB(); /** * Returns true if added * Returns false is the file no longer exists and could not be added */ bool put(quint64 docId, const QByteArray& url); QByteArray get(quint64 docId) const; QVector getChildren(quint64 docId) const; /** * \arg shouldDeleteFolder This function is called on any empty folder, and is used to * determine if that empty folder should be deleted. */ template void del(quint64 docId, Functor shouldDeleteFolder) { replace(docId, QByteArray(), shouldDeleteFolder); } /** * If \p url is empty then the docId is deleted */ template void replace(quint64 docId, const QByteArray& url, Functor shouldDeleteFolder); void rename(quint64 docId, const QByteArray& newFileName); quint64 getId(quint64 docId, const QByteArray& fileName) const; PostingIterator* iter(quint64 docId) { IdTreeDB db(m_idTreeDbi, m_txn); return db.iter(docId); } QMap toTestMap() const; private: void add(quint64 id, quint64 parentId, const QByteArray& name); MDB_txn* m_txn; MDB_dbi m_idFilenameDbi; MDB_dbi m_idTreeDbi; friend class UrlTest; }; template void DocumentUrlDB::replace(quint64 docId, const QByteArray& url, Functor shouldDeleteFolder) { Q_ASSERT(docId > 0); IdFilenameDB idFilenameDb(m_idFilenameDbi, m_txn); IdTreeDB idTreeDb(m_idTreeDbi, m_txn); // FIXME: Maybe this can be combined into one? auto path = idFilenameDb.get(docId); if (path.name.isEmpty()) { return; } idFilenameDb.del(docId); QVector subDocs = idTreeDb.get(path.parentId); subDocs.removeOne(docId); if (!subDocs.isEmpty()) { idTreeDb.put(path.parentId, subDocs); } else { idTreeDb.del(path.parentId); // // Delete every parent directory which only has 1 child // quint64 id = path.parentId; while (id) { auto path = idFilenameDb.get(id); // FIXME: Prevents database cleaning // Q_ASSERT(!path.name.isEmpty()); QVector subDocs = idTreeDb.get(path.parentId); if (subDocs.size() == 1 && shouldDeleteFolder(id)) { idTreeDb.del(path.parentId); idFilenameDb.del(id); } else { break; } id = path.parentId; } } if (url.isEmpty()) { auto subDocs = idTreeDb.get(docId); if (!subDocs.isEmpty()) { // Check if subdocs actually exist or is it a curruption for (auto const& docId : subDocs) { auto filePath = idFilenameDb.get(docId); auto fileName = QFile::decodeName(filePath.name); if (QFile::exists(fileName)) { - Q_ASSERT_X(idTreeDb.get(docId).isEmpty(), - "DocumentUrlDB::del", - "This folder still has sub-files in its cache. It cannot be deleted"); + if (!idTreeDb.get(docId).isEmpty()) { + qWarning() << "DocumentUrlDB::del" + << "This folder still has sub-files in its cache. It cannot be deleted"; + } } else { /* * FIXME: this is not an ideal solution we need to figure out how such currptions are * creeping in or at least if we detect some figure out a proper cleaning mechanism */ qWarning() << "Database has corrupted entries baloo may misbehave, please recreate the DB by running $ balooctl disable && balooctl enable"; } } } return; } put(docId, url); } } #endif // BALOO_DOCUMENTURLDB_H diff --git a/src/engine/fsutils.cpp b/src/engine/fsutils.cpp index 3f42f7f0..bf9bfc2a 100644 --- a/src/engine/fsutils.cpp +++ b/src/engine/fsutils.cpp @@ -1,85 +1,84 @@ /* * Copyright (C) 2010 Tobias Koenig * Copyright (C) 2014 Daniel Vrátil * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public License * along with this library; see the file COPYING.LIB. If not, write to * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. * */ #include "fsutils.h" - -#include +#include "enginedebug.h" #ifdef Q_OS_LINUX #include #include #include #include #endif using namespace Baloo; void FSUtils::disableCoW(const QString &path) { #ifndef Q_OS_LINUX Q_UNUSED(path); #else // from linux/fs.h, so that Baloo does not depend on Linux header files #ifndef FS_IOC_GETFLAGS #define FS_IOC_GETFLAGS _IOR('f', 1, long) #endif #ifndef FS_IOC_SETFLAGS #define FS_IOC_SETFLAGS _IOW('f', 2, long) #endif // Disable COW on file #ifndef FS_NOCOW_FL #define FS_NOCOW_FL 0x00800000 #endif ulong flags = 0; const int fd = open(qPrintable(path), O_RDONLY); if (fd == -1) { - qWarning() << "Failed to open" << path << "to modify flags (" << errno << ")"; + qCWarning(ENGINE) << "Failed to open" << path << "to modify flags (" << errno << ")"; return; } if (ioctl(fd, FS_IOC_GETFLAGS, &flags) == -1) { const int errno_ioctl = errno; // ignore ENOTTY, filesystem does not support attrs (and likely neither supports COW) if (errno_ioctl != ENOTTY) { - qWarning() << "ioctl error: failed to get file flags (" << errno_ioctl << ")"; + qCWarning(ENGINE) << "ioctl error: failed to get file flags (" << errno_ioctl << ")"; } close(fd); return; } if (!(flags & FS_NOCOW_FL)) { flags |= FS_NOCOW_FL; if (ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1) { const int errno_ioctl = errno; // ignore EOPNOTSUPP, returned on filesystems not supporting COW if (errno_ioctl != EOPNOTSUPP) { - qWarning() << "ioctl error: failed to set file flags (" << errno_ioctl << ")"; + qCWarning(ENGINE) << "ioctl error: failed to set file flags (" << errno_ioctl << ")"; } close(fd); return; } } close(fd); #endif } diff --git a/src/engine/idfilenamedb.cpp b/src/engine/idfilenamedb.cpp index f03faae7..91ab5711 100644 --- a/src/engine/idfilenamedb.cpp +++ b/src/engine/idfilenamedb.cpp @@ -1,158 +1,171 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "idfilenamedb.h" +#include "enginedebug.h" using namespace Baloo; IdFilenameDB::IdFilenameDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } IdFilenameDB::~IdFilenameDB() { } MDB_dbi IdFilenameDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "idfilename", MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "IdFilenameDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "IdFilenameDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi IdFilenameDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "idfilename", MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "IdFilenameDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "IdFilenameDB::open", mdb_strerror(rc)); return dbi; } void IdFilenameDB::put(quint64 docId, const FilePath& path) { Q_ASSERT(docId > 0); Q_ASSERT(!path.name.isEmpty()); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); QByteArray data(8 + path.name.size(), Qt::Uninitialized); memcpy(data.data(), &path.parentId, 8); memcpy(data.data() + 8, path.name.data(), path.name.size()); MDB_val val; val.mv_size = data.size(); val.mv_data = static_cast(data.data()); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "IdFilenameDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "IdFilenameDB::put" << mdb_strerror(rc); + } } IdFilenameDB::FilePath IdFilenameDB::get(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); FilePath path; - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "IdfilenameDB::get" << docId << mdb_strerror(rc); + } return path; } - Q_ASSERT_X(rc == 0, "IdfilenameDB::get", mdb_strerror(rc)); path.parentId = static_cast(val.mv_data)[0]; path.name = QByteArray(static_cast(val.mv_data) + 8, val.mv_size - 8); return path; } bool IdFilenameDB::contains(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "IdfilenameDB::contains" << docId << mdb_strerror(rc); + } return false; } - Q_ASSERT_X(rc == 0, "IdfilenameDB::contains", mdb_strerror(rc)); return true; } void IdFilenameDB::del(quint64 docId) { Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - Q_ASSERT_X(rc == 0, "IdfilenameDB::del", mdb_strerror(rc)); + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "IdFilenameDB::del" << mdb_strerror(rc); + } } QMap IdFilenameDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "IdFilenameDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "IdFilenameDB::toTestMap", mdb_strerror(rc)); + const quint64 id = *(static_cast(key.mv_data)); FilePath path; path.parentId = static_cast(val.mv_data)[0]; path.name = QByteArray(static_cast(val.mv_data) + 8, val.mv_size - 8); map.insert(id, path); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/idtreedb.cpp b/src/engine/idtreedb.cpp index 5ba289d9..1a2c99a8 100644 --- a/src/engine/idtreedb.cpp +++ b/src/engine/idtreedb.cpp @@ -1,187 +1,196 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "idtreedb.h" +#include "enginedebug.h" #include "postingiterator.h" -#include #include using namespace Baloo; IdTreeDB::IdTreeDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } MDB_dbi IdTreeDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "idtree", MDB_CREATE | MDB_INTEGERKEY, &dbi); - Q_ASSERT_X(rc == 0, "IdTreeDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "IdTreeDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi IdTreeDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "idtree", MDB_INTEGERKEY, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "IdTreeDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "IdTreeDB::open", mdb_strerror(rc)); return dbi; } void IdTreeDB::put(quint64 docId, const QVector subDocIds) { Q_ASSERT(!subDocIds.isEmpty()); Q_ASSERT(!subDocIds.contains(0)); MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); MDB_val val; val.mv_size = subDocIds.size() * sizeof(quint64); val.mv_data = static_cast(const_cast(subDocIds.constData())); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "IdTreeDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "IdTreeDB::put" << mdb_strerror(rc); + } } QVector IdTreeDB::get(quint64 docId) { MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "IdTreeDB::get" << docId << mdb_strerror(rc); + } return QVector(); } - Q_ASSERT_X(rc == 0, "IdTreeeDB::get", mdb_strerror(rc)); // FIXME: This still makes a copy of the data. Perhaps we can avoid that? QVector list(val.mv_size / sizeof(quint64)); memcpy(list.data(), val.mv_data, val.mv_size); return list; } void IdTreeDB::del(quint64 docId) { MDB_val key; key.mv_size = sizeof(quint64); key.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - Q_ASSERT_X(rc == 0, "IdTreeDB::del", mdb_strerror(rc)); + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "IdTreeDB::del" << mdb_strerror(rc); + } } // // Iter // class IdTreePostingIterator : public PostingIterator { public: IdTreePostingIterator(const IdTreeDB& db, const QVector list) : m_db(db), m_pos(-1), m_idList(list) {} quint64 docId() const override { if (m_pos >= 0 && m_pos < m_resultList.size()) return m_resultList[m_pos]; return 0; } quint64 next() override { if (m_resultList.isEmpty() && m_idList.isEmpty()) { return 0; } if (m_resultList.isEmpty()) { while (!m_idList.isEmpty()) { quint64 id = m_idList.takeLast(); m_idList << m_db.get(id); m_resultList << id; } std::sort(m_resultList.begin(), m_resultList.end()); m_pos = 0; } else { if (m_pos < m_resultList.size()) m_pos++; else m_resultList.clear(); } if (m_pos < m_resultList.size()) return m_resultList[m_pos]; else return 0; } private: IdTreeDB m_db; int m_pos; QVector m_idList; QVector m_resultList; }; PostingIterator* IdTreeDB::iter(quint64 docId) { Q_ASSERT(docId > 0); QVector list = {docId}; return new IdTreePostingIterator(*this, list); } QMap> IdTreeDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap> map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); const quint64 id = *(static_cast(key.mv_data)); QVector list(val.mv_size / sizeof(quint64)); memcpy(list.data(), val.mv_data, val.mv_size); map.insert(id, list); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/mtimedb.cpp b/src/engine/mtimedb.cpp index c5b5159f..5685b85c 100644 --- a/src/engine/mtimedb.cpp +++ b/src/engine/mtimedb.cpp @@ -1,260 +1,268 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "mtimedb.h" +#include "enginedebug.h" #include "vectorpostingiterator.h" #include using namespace Baloo; MTimeDB::MTimeDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } MTimeDB::~MTimeDB() { } MDB_dbi MTimeDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "mtimedb", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &dbi); - Q_ASSERT_X(rc == 0, "MTimeDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "MTimeDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi MTimeDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "mtimedb", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "MTimeDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "MTimeDB::open", mdb_strerror(rc)); return dbi; } void MTimeDB::put(quint32 mtime, quint64 docId) { Q_ASSERT(mtime > 0); Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint32); key.mv_data = static_cast(&mtime); MDB_val val; val.mv_size = sizeof(quint64); val.mv_data = static_cast(&docId); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "MTimeDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "MTimeDB::put" << mdb_strerror(rc); + } } QVector MTimeDB::get(quint32 mtime) { Q_ASSERT(mtime > 0); MDB_val key; key.mv_size = sizeof(quint32); key.mv_data = static_cast(&mtime); QVector values; MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "MTimeDB::get" << mtime << mdb_strerror(rc); + } mdb_cursor_close(cursor); return values; } - Q_ASSERT_X(rc == 0, "MTimeDB::get", mdb_strerror(rc)); values << *static_cast(val.mv_data); while (1) { rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT_DUP); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "MTimeDB::get (loop)" << mtime << mdb_strerror(rc); + } break; } - Q_ASSERT_X(rc == 0, "MTimeDB::get while", mdb_strerror(rc)); - values << *static_cast(val.mv_data); } mdb_cursor_close(cursor); std::sort(values.begin(), values.end()); values.erase(std::unique(values.begin(), values.end()), values.end()); return values; } void MTimeDB::del(quint32 mtime, quint64 docId) { Q_ASSERT(mtime > 0); Q_ASSERT(docId > 0); MDB_val key; key.mv_size = sizeof(quint32); key.mv_data = static_cast(&mtime); MDB_val val; val.mv_size = sizeof(quint64); val.mv_data = static_cast(&docId); int rc = mdb_del(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "MTimeDB::del" << mtime << docId << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "DocumentDB::del", mdb_strerror(rc)); } // // Posting Iterator // PostingIterator* MTimeDB::iter(quint32 mtime, MTimeDB::Comparator com) { if (com == Equal) { return new VectorPostingIterator(get(mtime)); } MDB_val key; key.mv_size = sizeof(quint32); key.mv_data = &mtime; MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::iter" << mtime << mdb_strerror(rc); mdb_cursor_close(cursor); return nullptr; } - Q_ASSERT_X(rc == 0, "MTimeDB::iter", mdb_strerror(rc)); QVector results; results << *static_cast(val.mv_data); if (com == GreaterEqual) { while (1) { rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::iter (loop)" << mtime << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "MTimeDB::iter >=", mdb_strerror(rc)); results << *static_cast(val.mv_data); } } else { while (1) { rc = mdb_cursor_get(cursor, &key, &val, MDB_PREV); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::iter (loop)" << mtime << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "MTimeDB::iter >=", mdb_strerror(rc)); quint64 id = *static_cast(val.mv_data); results.push_front(id); } } mdb_cursor_close(cursor); std::sort(results.begin(), results.end()); results.erase(std::unique(results.begin(), results.end()), results.end()); return new VectorPostingIterator(results); } PostingIterator* MTimeDB::iterRange(quint32 beginTime, quint32 endTime) { Q_ASSERT(beginTime); Q_ASSERT(endTime); MDB_val key; key.mv_size = sizeof(quint32); key.mv_data = &beginTime; MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::iterRange" << beginTime << endTime << mdb_strerror(rc); mdb_cursor_close(cursor); return nullptr; } - Q_ASSERT_X(rc == 0, "MTimeDB::iterRange", mdb_strerror(rc)); QVector results; results << *static_cast(val.mv_data); while (1) { rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::iterRange" << beginTime << endTime << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "MTimeDB::iter >=", mdb_strerror(rc)); quint32 time = *static_cast(key.mv_data); if (time > endTime) { break; } results << *static_cast(val.mv_data); } mdb_cursor_close(cursor); std::sort(results.begin(), results.end()); results.erase(std::unique(results.begin(), results.end()), results.end()); return new VectorPostingIterator(results); } QMap MTimeDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "MTimeDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "MTimeDB::toTestMap", mdb_strerror(rc)); const quint32 time = *(static_cast(key.mv_data)); const quint64 id = *(static_cast(val.mv_data)); map.insert(time, id); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/phraseanditerator.cpp b/src/engine/phraseanditerator.cpp index 2c20b11a..ddcb3973 100644 --- a/src/engine/phraseanditerator.cpp +++ b/src/engine/phraseanditerator.cpp @@ -1,123 +1,121 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "phraseanditerator.h" -#include - using namespace Baloo; PhraseAndIterator::PhraseAndIterator(const QVector& iterators) : m_iterators(iterators) , m_docId(0) { if (m_iterators.contains(nullptr)) { qDeleteAll(m_iterators); m_iterators.clear(); } } PhraseAndIterator::~PhraseAndIterator() { qDeleteAll(m_iterators); } quint64 PhraseAndIterator::docId() const { return m_docId; } bool PhraseAndIterator::checkIfPositionsMatch() { QVector< QVector > positionList; positionList.reserve(m_iterators.size()); // All the iterators should have the same value for (int i = 0; i < m_iterators.size(); i++) { PostingIterator* iter = m_iterators[i]; Q_ASSERT(iter->docId() == m_docId); QVector pi = iter->positions(); for (int j = 0; j < pi.size(); j++) { pi[j] -= i; } positionList << pi; } // Intersect all these positions QVector vec = positionList[0]; for (int l = 1; l < positionList.size(); l++) { QVector newVec = positionList[l]; int i = 0; int j = 0; QVector finalVec; while (i < vec.size() && j < newVec.size()) { if (vec[i] == newVec[j]) { finalVec << vec[i]; i++; j++; } else if (vec[i] < newVec[j]) { i++; } else { j++; } } vec = finalVec; } return !vec.isEmpty(); } quint64 PhraseAndIterator::next() { if (m_iterators.isEmpty()) { m_docId = 0; return 0; } if (m_iterators[0]->next() == 0) { m_docId = 0; return 0; } m_docId = m_iterators[0]->docId(); for (int i = 1; i < m_iterators.size(); i++) { PostingIterator* iter = m_iterators[i]; if (iter->docId() == 0 && iter->next() == 0) { m_docId = 0; return 0; } iter->skipTo(m_docId); if (m_docId != iter->docId()) { return next(); } } if (checkIfPositionsMatch()) return m_docId; else return next(); } diff --git a/src/engine/positiondb.cpp b/src/engine/positiondb.cpp index d1049178..6b8f2b57 100644 --- a/src/engine/positiondb.cpp +++ b/src/engine/positiondb.cpp @@ -1,201 +1,206 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ +#include "enginedebug.h" #include "positiondb.h" #include "positioncodec.h" #include "positioninfo.h" #include "postingiterator.h" -#include - using namespace Baloo; PositionDB::PositionDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } PositionDB::~PositionDB() { } MDB_dbi PositionDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "positiondb", MDB_CREATE, &dbi); - Q_ASSERT_X(rc == 0, "PositionDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "PositionDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi PositionDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "positiondb", 0, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "PositionDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "PositionDB::open", mdb_strerror(rc)); return dbi; } void PositionDB::put(const QByteArray& term, const QVector& list) { Q_ASSERT(!term.isEmpty()); Q_ASSERT(!list.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); PositionCodec codec; QByteArray data = codec.encode(list); MDB_val val; val.mv_size = data.size(); val.mv_data = static_cast(data.data()); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "PositionDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "PositionDB::put" << mdb_strerror(rc); + } } QVector PositionDB::get(const QByteArray& term) { Q_ASSERT(!term.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "PositionDB::get" << term << mdb_strerror(rc); + } return QVector(); } - Q_ASSERT_X(rc == 0, "PositionDB::get", mdb_strerror(rc)); QByteArray data = QByteArray::fromRawData(static_cast(val.mv_data), val.mv_size); PositionCodec codec; return codec.decode(data); } void PositionDB::del(const QByteArray& term) { Q_ASSERT(!term.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "PositionDB::del" << term << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "PositionDB::del", mdb_strerror(rc)); } // // Query // class DBPositionIterator : public PostingIterator { public: DBPositionIterator(char* data, uint size) : m_pos(-1) { PositionCodec codec; m_vec = codec.decode(QByteArray(static_cast(data), size)); } quint64 next() override { m_pos++; if (m_pos >= m_vec.size()) { return 0; } return m_vec[m_pos].docId; } quint64 docId() const override { if (m_pos < 0 || m_pos >= m_vec.size()) { return 0; } return m_vec[m_pos].docId; } QVector positions() override { if (m_pos < 0 || m_pos >= m_vec.size()) { return QVector(); } return m_vec[m_pos].positions; } private: QVector m_vec; int m_pos; }; PostingIterator* PositionDB::iter(const QByteArray& term) { Q_ASSERT(!term.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "PositionDB::iter" << term << mdb_strerror(rc); return nullptr; } - Q_ASSERT_X(rc == 0, "PositionDB::iter", mdb_strerror(rc)); return new DBPositionIterator(static_cast(val.mv_data), val.mv_size); } QMap> PositionDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap> map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); const QByteArray ba(static_cast(key.mv_data), key.mv_size); const QVector vinfo = PositionCodec().decode(QByteArray(static_cast(val.mv_data), val.mv_size)); map.insert(ba, vinfo); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/postingdb.cpp b/src/engine/postingdb.cpp index c965223b..d16fd297 100644 --- a/src/engine/postingdb.cpp +++ b/src/engine/postingdb.cpp @@ -1,294 +1,298 @@ /* This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ +#include "enginedebug.h" #include "postingdb.h" #include "orpostingiterator.h" #include "postingcodec.h" -#include - using namespace Baloo; PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn) : m_txn(txn) , m_dbi(dbi) { Q_ASSERT(txn != nullptr); Q_ASSERT(dbi != 0); } PostingDB::~PostingDB() { } MDB_dbi PostingDB::create(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi); - Q_ASSERT_X(rc == 0, "PostingDB::create", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "PostingDB::create" << mdb_strerror(rc); + return 0; + } return dbi; } MDB_dbi PostingDB::open(MDB_txn* txn) { - MDB_dbi dbi; + MDB_dbi dbi = 0; int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCWarning(ENGINE) << "PostingDB::open" << mdb_strerror(rc); return 0; } - Q_ASSERT_X(rc == 0, "PostingDB::open", mdb_strerror(rc)); return dbi; } void PostingDB::put(const QByteArray& term, const PostingList& list) { Q_ASSERT(!term.isEmpty()); Q_ASSERT(!list.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); PostingCodec codec; QByteArray arr = codec.encode(list); MDB_val val; val.mv_size = arr.size(); val.mv_data = static_cast(arr.data()); int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); - Q_ASSERT_X(rc == 0, "PostingDB::put", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "PostingDB::put" << mdb_strerror(rc); + } } PostingList PostingDB::get(const QByteArray& term) { Q_ASSERT(!term.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); - MDB_val val; + MDB_val val{0, nullptr}; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "PostingDB::get" << term << mdb_strerror(rc); + } return PostingList(); } - Q_ASSERT_X(rc == 0, "PostingDB::get", mdb_strerror(rc)); QByteArray arr = QByteArray::fromRawData(static_cast(val.mv_data), val.mv_size); PostingCodec codec; return codec.decode(arr); } void PostingDB::del(const QByteArray& term) { Q_ASSERT(!term.isEmpty()); MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); int rc = mdb_del(m_txn, m_dbi, &key, nullptr); - if (rc == MDB_NOTFOUND) { - return; + if (rc != 0 && rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "PostingDB::del" << term << mdb_strerror(rc); } - Q_ASSERT_X(rc == 0, "PostingDB::del", mdb_strerror(rc)); } QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term) { MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); QVector terms; int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE); - while (rc != MDB_NOTFOUND) { - Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); - + while (rc == 0) { const QByteArray arr(static_cast(key.mv_data), key.mv_size); if (!arr.startsWith(term)) { break; } terms << arr; rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); } - Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); + if (rc != MDB_NOTFOUND) { + qCDebug(ENGINE) << "PostingDB::fetchTermsStartingWith" << mdb_strerror(rc); + } mdb_cursor_close(cursor); return terms; } class DBPostingIterator : public PostingIterator { public: DBPostingIterator(void* data, uint size); quint64 docId() const override; quint64 next() override; private: const QVector m_vec; int m_pos; }; PostingIterator* PostingDB::iter(const QByteArray& term) { MDB_val key; key.mv_size = term.size(); key.mv_data = static_cast(const_cast(term.constData())); MDB_val val; int rc = mdb_get(m_txn, m_dbi, &key, &val); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "PostingDB::iter" << term << mdb_strerror(rc); return nullptr; } - Q_ASSERT_X(rc == 0, "PostingDB::iter", mdb_strerror(rc)); return new DBPostingIterator(val.mv_data, val.mv_size); } // // Posting Iterator // DBPostingIterator::DBPostingIterator(void* data, uint size) : m_vec(PostingCodec().decode(QByteArray(static_cast(data), size))) , m_pos(-1) { } quint64 DBPostingIterator::docId() const { if (m_pos < 0 || m_pos >= m_vec.size()) { return 0; } return m_vec[m_pos]; } quint64 DBPostingIterator::next() { if (m_pos >= m_vec.size() - 1) { m_pos = m_vec.size(); return 0; } m_pos++; return m_vec[m_pos]; } template PostingIterator* PostingDB::iter(const QByteArray& prefix, Validator validate) { Q_ASSERT(!prefix.isEmpty()); MDB_val key; key.mv_size = prefix.size(); key.mv_data = static_cast(const_cast(prefix.constData())); MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); QVector termIterators; MDB_val val; int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); - while (rc != MDB_NOTFOUND) { - Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); - + while (rc == 0) { const QByteArray arr(static_cast(key.mv_data), key.mv_size); if (!arr.startsWith(prefix)) { break; } if (validate(arr)) { termIterators << new DBPostingIterator(val.mv_data, val.mv_size); } rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); } - if (rc != MDB_NOTFOUND) { - Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); + + if (rc != 0 && rc != MDB_NOTFOUND) { + qCWarning(ENGINE) << "PostingDB::regexpIter" << mdb_strerror(rc); } mdb_cursor_close(cursor); if (termIterators.isEmpty()) { return nullptr; } return new OrPostingIterator(termIterators); } PostingIterator* PostingDB::prefixIter(const QByteArray& prefix) { auto validate = [] (const QByteArray& arr) { Q_UNUSED(arr); return true; }; return iter(prefix, validate); } PostingIterator* PostingDB::regexpIter(const QRegularExpression& regexp, const QByteArray& prefix) { int prefixLen = prefix.length(); auto validate = [®exp, prefixLen] (const QByteArray& arr) { QString term = QString::fromUtf8(arr.mid(prefixLen)); return regexp.match(term).hasMatch(); }; return iter(prefix, validate); } PostingIterator* PostingDB::compIter(const QByteArray& prefix, qlonglong comVal, PostingDB::Comparator com) { int prefixLen = prefix.length(); auto validate = [prefixLen, comVal, com] (const QByteArray& arr) { bool ok = false; auto val = QByteArray::fromRawData(arr.constData() + prefixLen, arr.length() - prefixLen).toLongLong(&ok); return ok && ((com == LessEqual && val <= comVal) || (com == GreaterEqual && val >= comVal)); }; return iter(prefix, validate); } QMap PostingDB::toTestMap() const { MDB_cursor* cursor; mdb_cursor_open(m_txn, m_dbi, &cursor); MDB_val key = {0, nullptr}; MDB_val val; QMap map; while (1) { int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); - if (rc == MDB_NOTFOUND) { + if (rc) { + qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc); break; } - Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); const QByteArray ba(static_cast(key.mv_data), key.mv_size); const PostingList plist = PostingCodec().decode(QByteArray(static_cast(val.mv_data), val.mv_size)); map.insert(ba, plist); } mdb_cursor_close(cursor); return map; } diff --git a/src/engine/transaction.cpp b/src/engine/transaction.cpp index fd4a85c1..d56dbae5 100644 --- a/src/engine/transaction.cpp +++ b/src/engine/transaction.cpp @@ -1,606 +1,640 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "transaction.h" #include "postingdb.h" #include "documentdb.h" #include "documenturldb.h" #include "documentiddb.h" #include "positiondb.h" #include "documentdatadb.h" #include "mtimedb.h" #include "document.h" #include "enginequery.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "phraseanditerator.h" #include "writetransaction.h" #include "idutils.h" #include "database.h" #include "databasesize.h" +#include "enginedebug.h" + #include #include using namespace Baloo; Transaction::Transaction(const Database& db, Transaction::TransactionType type) : m_dbis(db.m_dbis) , m_env(db.m_env) , m_writeTrans(nullptr) { uint flags = type == ReadOnly ? MDB_RDONLY : 0; int rc = mdb_txn_begin(db.m_env, nullptr, flags, &m_txn); - Q_ASSERT_X(rc == 0, "Transaction", mdb_strerror(rc)); + if (rc) { + qCDebug(ENGINE) << "Transaction" << mdb_strerror(rc); + return; + } if (type == ReadWrite) { m_writeTrans = new WriteTransaction(m_dbis, m_txn); } } Transaction::Transaction(Database* db, Transaction::TransactionType type) : Transaction(*db, type) { } Transaction::~Transaction() { - if (m_writeTrans) - qWarning() << "Closing an active WriteTransaction without calling abort/commit"; + if (m_writeTrans) { + qWarning(ENGINE) << "Closing an active WriteTransaction without calling abort/commit"; + } if (m_txn) { - abort(); + abortTransaction(); } } bool Transaction::hasDocument(quint64 id) const { Q_ASSERT(id > 0); IdFilenameDB idFilenameDb(m_dbis.idFilenameDbi, m_txn); return idFilenameDb.contains(id); } bool Transaction::inPhaseOne(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.contains(id); } bool Transaction::hasFailed(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); return failedIdDb.contains(id); } QByteArray Transaction::documentUrl(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.get(id); } quint64 Transaction::documentId(const QByteArray& path) const { Q_ASSERT(m_txn); Q_ASSERT(!path.isEmpty()); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); QList li = path.split('/'); quint64 parentId = 0; for (const QByteArray& fileName : li) { if (fileName.isEmpty()) { continue; } parentId = docUrlDb.getId(parentId, fileName); if (!parentId) { return 0; } } return parentId; } QVector Transaction::childrenDocumentId(quint64 parentId) const { DocumentUrlDB docUrlDB(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDB.getChildren(parentId); } DocumentTimeDB::TimeInfo Transaction::documentTimeInfo(quint64 id) const { Q_ASSERT(m_txn); DocumentTimeDB docTimeDb(m_dbis.docTimeDbi, m_txn); return docTimeDb.get(id); } QByteArray Transaction::documentData(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentDataDB docDataDb(m_dbis.docDataDbi, m_txn); return docDataDb.get(id); } bool Transaction::hasChanges() const { Q_ASSERT(m_txn); - Q_ASSERT(m_writeTrans); + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return false; + } + return m_writeTrans->hasChanges(); } QVector Transaction::fetchPhaseOneIds(int size) const { Q_ASSERT(m_txn); Q_ASSERT(size > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.fetchItems(size); } QVector Transaction::fetchTermsStartingWith(const QByteArray& term) const { Q_ASSERT(term.size() > 0); PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.fetchTermsStartingWith(term); } uint Transaction::phaseOneSize() const { Q_ASSERT(m_txn); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.size(); } uint Transaction::size() const { Q_ASSERT(m_txn); DocumentDB docTermsDb(m_dbis.docTermsDbi, m_txn); return docTermsDb.size(); } // // Write Operations // void Transaction::setPhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.put(id); } void Transaction::removePhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.del(id); } void Transaction::addFailed(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); failedIdDb.put(id); } void Transaction::addDocument(const Document& doc) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); - Q_ASSERT(m_writeTrans); + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return; + } m_writeTrans->addDocument(doc); } void Transaction::removeDocument(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); - Q_ASSERT(m_writeTrans); + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return; + } m_writeTrans->removeDocument(id); } void Transaction::removeRecursively(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); - Q_ASSERT(m_writeTrans); + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return; + } m_writeTrans->removeRecursively(id); } void Transaction::replaceDocument(const Document& doc, DocumentOperations operations) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); - Q_ASSERT_X(hasDocument(doc.id()), "Transaction::replaceDocument", "Document does not exist"); + if (!hasDocument(doc.id())) { + qCDebug(ENGINE) << "Transaction::replaceDocument" << "Document does not exist"; + } + + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return; + } m_writeTrans->replaceDocument(doc, operations); } void Transaction::commit() { Q_ASSERT(m_txn); - Q_ASSERT(m_writeTrans); + if (!m_writeTrans) { + qCWarning(ENGINE) << "m_writeTrans is null"; + return; + } m_writeTrans->commit(); delete m_writeTrans; m_writeTrans = nullptr; int rc = mdb_txn_commit(m_txn); - Q_ASSERT_X(rc == 0, "Transaction::commit", mdb_strerror(rc)); + if (rc) { + qCWarning(ENGINE) << "Transaction::commit" << mdb_strerror(rc); + } m_txn = nullptr; } -void Transaction::abort() +void Transaction::abortTransaction() { Q_ASSERT(m_txn); mdb_txn_abort(m_txn); m_txn = nullptr; delete m_writeTrans; m_writeTrans = nullptr; } // // Queries // PostingIterator* Transaction::postingIterator(const EngineQuery& query) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); PositionDB positionDb(m_dbis.positionDBi, m_txn); if (query.leaf()) { if (query.op() == EngineQuery::Equal) { return postingDb.iter(query.term()); } else if (query.op() == EngineQuery::StartsWith) { return postingDb.prefixIter(query.term()); } else { Q_ASSERT(0); } } if (query.subQueries().isEmpty()) { return nullptr; } QVector vec; vec.reserve(query.subQueries().size()); if (query.op() == EngineQuery::Phrase) { const auto subQueries = query.subQueries(); for (const EngineQuery& q : subQueries) { - Q_ASSERT_X(q.leaf(), "Transaction::toPostingIterator", "Phrase queries must contain leaf queries"); + if (!q.leaf()) { + qCDebug(ENGINE) << "Transaction::toPostingIterator" << "Phrase queries must contain leaf queries"; + continue; + } vec << positionDb.iter(q.term()); } return new PhraseAndIterator(vec); } const auto subQueries = query.subQueries(); for (const EngineQuery& q : subQueries) { auto iterator = postingIterator(q); if (iterator) { vec << iterator; } else if (query.op() == EngineQuery::And) { return nullptr; } } if (vec.empty()) { return nullptr; } else if (vec.size() == 1) { return vec.takeFirst(); } if (query.op() == EngineQuery::And) { return new AndPostingIterator(vec); } else if (query.op() == EngineQuery::Or) { return new OrPostingIterator(vec); } Q_ASSERT(0); return nullptr; } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, qlonglong value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::mTimeIter(quint32 mtime, MTimeDB::Comparator com) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iter(mtime, com); } PostingIterator* Transaction::mTimeRangeIter(quint32 beginTime, quint32 endTime) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iterRange(beginTime, endTime); } PostingIterator* Transaction::docUrlIter(quint64 id) const { DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.iter(id); } QVector Transaction::exec(const EngineQuery& query, int limit) const { Q_ASSERT(m_txn); QVector results; PostingIterator* it = postingIterator(query); if (!it) { return results; } while (it->next() && limit) { results << it->docId(); limit--; } return results; } // // Introspection // QVector Transaction::documentTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); return documentTermsDB.get(docId); } QVector Transaction::documentFileNameTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); return documentFileNameTermsDB.get(docId); } QVector Transaction::documentXattrTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); return documentXattrTermsDB.get(docId); } // // File Size // static size_t dbiSize(MDB_txn* txn, MDB_dbi dbi) { MDB_stat stat; mdb_stat(txn, dbi, &stat); return (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages) * stat.ms_psize; } DatabaseSize Transaction::dbSize() { DatabaseSize dbSize; dbSize.postingDb = dbiSize(m_txn, m_dbis.postingDbi); dbSize.positionDb = dbiSize(m_txn, m_dbis.positionDBi); dbSize.docTerms = dbiSize(m_txn, m_dbis.docTermsDbi); dbSize.docFilenameTerms = dbiSize(m_txn, m_dbis.docFilenameTermsDbi); dbSize.docXattrTerms = dbiSize(m_txn, m_dbis.docXattrTermsDbi); dbSize.idTree = dbiSize(m_txn, m_dbis.idTreeDbi); dbSize.idFilename = dbiSize(m_txn, m_dbis.idFilenameDbi); dbSize.docTime = dbiSize(m_txn, m_dbis.docTimeDbi); dbSize.docData = dbiSize(m_txn, m_dbis.docDataDbi); dbSize.contentIndexingIds = dbiSize(m_txn, m_dbis.contentIndexingDbi); dbSize.failedIds = dbiSize(m_txn, m_dbis.failedIdDbi); dbSize.mtimeDb = dbiSize(m_txn, m_dbis.mtimeDbi); dbSize.expectedSize = dbSize.positionDb + dbSize.positionDb + dbSize.docTerms + dbSize.docFilenameTerms + dbSize.docXattrTerms + dbSize.idTree + dbSize.idFilename + dbSize.docTime + dbSize.docData + dbSize.contentIndexingIds + dbSize.failedIds + dbSize.mtimeDb; MDB_envinfo info; mdb_env_info(m_env, &info); dbSize.actualSize = info.me_last_pgno * 4096; // TODO: separate page size return dbSize; } // // Debugging // void Transaction::checkFsTree() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "Total Document IDs: " << allIds.size() << endl; int count = 0; for (quint64 id: qAsConst(allIds)) { QByteArray url = docUrlDb.get(id); if (url.isEmpty()) { auto terms = documentTermsDB.get(id); auto fileNameTerms = documentFileNameTermsDB.get(id); auto xAttrTerms = documentXattrTermsDB.get(id); // Lets reverse enginer the terms QList newTerms; QMapIterator it(map); while (it.hasNext()) { it.next(); if (it.value().contains(id)) { newTerms << it.key(); } } out << "Missing filePath for " << id << endl; out << "\tPostingDB Terms: "; for (const QByteArray& term : qAsConst(newTerms)) { out << term << " "; } out << endl; out << "\tDocumentTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; out << "\tFileNameTermsDB: "; for (const QByteArray& term : fileNameTerms) { out << term << " "; } out << endl; out << "\tXAttrTermsDB: "; for (const QByteArray& term : xAttrTerms) { out << term << " "; } out << endl; count++; } else if (!QFileInfo::exists(QString::fromUtf8(url))) { out << "FilePath " << url << " for " << id << " does not exist"<< endl; count++; } } out << "Invalid Entries: " << count << " (" << count * 100.0 / allIds.size() << "%)" << endl; } void Transaction::checkTermsDbinPostingDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); // Iterate over each document, and fetch all terms // check if each term maps to its own id in the posting db auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "PostingDB check .." << endl; for (quint64 id : qAsConst(allIds)) { QVector terms = documentTermsDB.get(id); terms += documentXattrTermsDB.get(id); terms += documentFileNameTermsDB.get(id); for (const QByteArray& term : qAsConst(terms)) { PostingList plist = postingDb.get(term); if (!plist.contains(id)) { out << id << " is missing term " << term << endl; } } } } void Transaction::checkPostingDbinTermsDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); QMap map = postingDb.toTestMap(); QMapIterator it(map); QTextStream out(stdout); out << "DocumentTermsDB check .." << endl; while (it.hasNext()) { it.next(); const QByteArray& term = it.key(); const PostingList& list = it.value(); for (quint64 id : list) { if (documentTermsDB.get(id).contains(term)) { continue; } if (documentFileNameTermsDB.get(id).contains(term)) { continue; } if (documentXattrTermsDB.get(id).contains(term)) { continue; } out << id << " is missing " << QString::fromUtf8(term) << " from document terms db" << endl; } } } diff --git a/src/engine/transaction.h b/src/engine/transaction.h index bc56238e..8874a5e5 100644 --- a/src/engine/transaction.h +++ b/src/engine/transaction.h @@ -1,139 +1,139 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_TRANSACTION_H #define BALOO_TRANSACTION_H #include "databasedbis.h" #include "mtimedb.h" #include "postingdb.h" #include "writetransaction.h" #include "documenttimedb.h" #include #include namespace Baloo { class Database; class Document; class PostingIterator; class EngineQuery; class DatabaseSize; class DBState; class BALOO_ENGINE_EXPORT Transaction { public: enum TransactionType { ReadOnly, ReadWrite }; Transaction(const Database& db, TransactionType type); Transaction(Database* db, TransactionType type); ~Transaction(); // // Getters // bool hasDocument(quint64 id) const; bool inPhaseOne(quint64 id) const; bool hasFailed(quint64 id) const; QByteArray documentUrl(quint64 id) const; /** * This method is not cheap, and does not stat the filesystem in order to convert the path * \p path into an id. */ quint64 documentId(const QByteArray& path) const; QVector childrenDocumentId(quint64 parentId) const; QByteArray documentData(quint64 id) const; DocumentTimeDB::TimeInfo documentTimeInfo(quint64 id) const; QVector exec(const EngineQuery& query, int limit = -1) const; PostingIterator* postingIterator(const EngineQuery& query) const; PostingIterator* postingCompIterator(const QByteArray& prefix, qlonglong value, PostingDB::Comparator com) const; PostingIterator* mTimeIter(quint32 mtime, MTimeDB::Comparator com) const; PostingIterator* mTimeRangeIter(quint32 beginTime, quint32 endTime) const; PostingIterator* docUrlIter(quint64 id) const; QVector fetchPhaseOneIds(int size) const; uint phaseOneSize() const; uint size() const; QVector fetchTermsStartingWith(const QByteArray& term) const; // // Introspecing document data // QVector documentTerms(quint64 docId) const; QVector documentFileNameTerms(quint64 docId) const; QVector documentXattrTerms(quint64 docId) const; DatabaseSize dbSize(); // // Transaction handling // void commit(); - void abort(); + void abortTransaction(); bool hasChanges() const; // // Write Methods // void addDocument(const Document& doc); void removeDocument(quint64 id); void removeRecursively(quint64 parentId); void addFailed(quint64 id); template void removeRecursively(quint64 id, Functor shouldDelete) { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); m_writeTrans->removeRecursively(id, shouldDelete); } void replaceDocument(const Document& doc, DocumentOperations operations); void setPhaseOne(quint64 id); void removePhaseOne(quint64 id); // Debugging void checkFsTree(); void checkTermsDbinPostingDb(); void checkPostingDbinTermsDb(); private: Transaction(const Transaction& rhs) = delete; const DatabaseDbis& m_dbis; - MDB_txn* m_txn; - MDB_env* m_env; - WriteTransaction* m_writeTrans; + MDB_txn *m_txn = nullptr; + MDB_env *m_env = nullptr; + WriteTransaction *m_writeTrans = nullptr; friend class DatabaseSanitizerImpl; friend class DBState; // for testing }; } #endif // BALOO_TRANSACTION_H diff --git a/src/file/extractorprocess.cpp b/src/file/extractorprocess.cpp index c52c7e04..c18442f4 100644 --- a/src/file/extractorprocess.cpp +++ b/src/file/extractorprocess.cpp @@ -1,97 +1,98 @@ /* * Copyright (C) 2015 Vishesh Handa * Copyright (C) 2015 Pinak Ahuja * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "extractorprocess.h" +#include "baloodebug.h" + #include -#include #include using namespace Baloo; ExtractorProcess::ExtractorProcess(QObject* parent) : QObject(parent) , m_extractorPath(QStandardPaths::findExecutable(QStringLiteral("baloo_file_extractor"))) , m_extractorProcess(this) { connect(&m_extractorProcess, &QProcess::readyRead, this, &ExtractorProcess::slotIndexingFile); connect(&m_extractorProcess, QOverload::of(&QProcess::finished), [=](int exitCode, QProcess::ExitStatus exitStatus) { if (exitStatus == QProcess::CrashExit) { Q_EMIT failed(); } }); m_extractorProcess.setProgram(m_extractorPath); m_extractorProcess.setProcessChannelMode(QProcess::ForwardedErrorChannel); m_extractorProcess.start(); } ExtractorProcess::~ExtractorProcess() { m_extractorProcess.close(); } void ExtractorProcess::start() { m_extractorProcess.start(QIODevice::Unbuffered | QIODevice::ReadWrite); m_extractorProcess.waitForStarted(); m_extractorProcess.setReadChannel(QProcess::StandardOutput); } void ExtractorProcess::index(const QVector& fileIds) { Q_ASSERT(m_extractorProcess.state() == QProcess::Running); Q_ASSERT(!fileIds.isEmpty()); QDataStream batch(&m_extractorProcess); batch << fileIds; } void ExtractorProcess::slotIndexingFile() { while (m_extractorProcess.canReadLine()) { QString line = m_extractorProcess.readLine().trimmed(); if (line.isEmpty()) { continue; } char command = line[0].toLatin1(); QString arg = line.mid(2); switch (command) { case 'S': Q_EMIT startedIndexingFile(arg); break; case 'F': Q_EMIT finishedIndexingFile(arg); break; case 'B': Q_EMIT done(); break; default: - qCritical() << "Got unknown result from extractor" << command << arg; + qCritical(BALOO) << "Got unknown result from extractor" << command << arg; } } } diff --git a/src/file/fileindexscheduler.cpp b/src/file/fileindexscheduler.cpp index c6c7fc88..01ecba0c 100644 --- a/src/file/fileindexscheduler.cpp +++ b/src/file/fileindexscheduler.cpp @@ -1,247 +1,247 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "fileindexscheduler.h" +#include "baloodebug.h" #include "firstrunindexer.h" #include "newfileindexer.h" #include "modifiedfileindexer.h" #include "xattrindexer.h" #include "filecontentindexer.h" #include "filecontentindexerprovider.h" #include "unindexedfileindexer.h" #include "indexcleaner.h" #include "fileindexerconfig.h" #include #include -#include #include using namespace Baloo; FileIndexScheduler::FileIndexScheduler(Database* db, FileIndexerConfig* config, QObject* parent) : QObject(parent) , m_db(db) , m_config(config) , m_provider(db) , m_contentIndexer(nullptr) , m_indexerState(Idle) , m_timeEstimator(config, this) , m_checkUnindexedFiles(false) , m_checkStaleIndexEntries(false) , m_isGoingIdle(false) { Q_ASSERT(db); Q_ASSERT(config); m_threadPool.setMaxThreadCount(1); connect(&m_powerMonitor, &PowerStateMonitor::powerManagementStatusChanged, this, &FileIndexScheduler::powerManagementStatusChanged); m_contentIndexer = new FileContentIndexer(m_config, &m_provider, this); m_contentIndexer->setAutoDelete(false); connect(m_contentIndexer, &FileContentIndexer::done, this, &FileIndexScheduler::runnerFinished); connect(m_contentIndexer, &FileContentIndexer::newBatchTime, &m_timeEstimator, &TimeEstimator::handleNewBatchTime); QDBusConnection::sessionBus().registerObject(QStringLiteral("/scheduler"), this, QDBusConnection::ExportScriptableContents); } FileIndexScheduler::~FileIndexScheduler() { m_threadPool.waitForDone(0); // wait 0 msecs } void FileIndexScheduler::scheduleIndexing() { if (!m_isGoingIdle && m_indexerState != Idle) { return; } m_isGoingIdle = false; if (m_config->isInitialRun()) { auto runnable = new FirstRunIndexer(m_db, m_config, m_config->includeFolders()); connect(runnable, &FirstRunIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_indexerState = FirstRun; Q_EMIT stateChanged(m_indexerState); return; } if (!m_newFiles.isEmpty()) { auto runnable = new NewFileIndexer(m_db, m_config, m_newFiles); connect(runnable, &NewFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_newFiles.clear(); m_indexerState = NewFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_modifiedFiles.isEmpty()) { auto runnable = new ModifiedFileIndexer(m_db, m_config, m_modifiedFiles); connect(runnable, &ModifiedFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_modifiedFiles.clear(); m_indexerState = ModifiedFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_xattrFiles.isEmpty()) { auto runnable = new XAttrIndexer(m_db, m_config, m_xattrFiles); connect(runnable, &XAttrIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_xattrFiles.clear(); m_indexerState = XAttrFiles; Q_EMIT stateChanged(m_indexerState); return; } if (m_provider.size() && !m_powerMonitor.isOnBattery()) { m_threadPool.start(m_contentIndexer); m_indexerState = ContentIndexing; Q_EMIT stateChanged(m_indexerState); return; } if (m_checkUnindexedFiles) { auto runnable = new UnindexedFileIndexer(m_db, m_config); connect(runnable, &UnindexedFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_checkUnindexedFiles = false; m_indexerState = UnindexedFileCheck; Q_EMIT stateChanged(m_indexerState); return; } if (m_checkStaleIndexEntries) { auto runnable = new IndexCleaner(m_db, m_config); connect(runnable, &IndexCleaner::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_checkStaleIndexEntries = false; m_indexerState = StaleIndexEntriesClean; Q_EMIT stateChanged(m_indexerState); return; } m_indexerState = Idle; Q_EMIT stateChanged(m_indexerState); } static void removeStartsWith(QStringList& list, const QString& dir) { const auto tail = std::remove_if(list.begin(), list.end(), [&dir](const QString& file) { return file.startsWith(dir); }); list.erase(tail, list.end()); } void FileIndexScheduler::handleFileRemoved(const QString& file) { if (!file.endsWith(QLatin1Char('/'))) { m_newFiles.removeOne(file); m_modifiedFiles.removeOne(file); m_xattrFiles.removeOne(file); } else { removeStartsWith(m_newFiles, file); removeStartsWith(m_modifiedFiles, file); removeStartsWith(m_xattrFiles, file); } } void FileIndexScheduler::powerManagementStatusChanged(bool isOnBattery) { - qDebug() << "Power state changed"; + qCDebug(BALOO) << "Power state changed"; if (isOnBattery && m_indexerState == ContentIndexing) { - qDebug() << "On battery stopping content indexer"; + qCDebug(BALOO) << "On battery, stopping content indexer"; m_contentIndexer->quit(); //TODO: Maybe we can add a special state for suspended due to being on battery. m_indexerState = Idle; Q_EMIT stateChanged(m_indexerState); } else if (!isOnBattery) { scheduleIndexing(); } } void FileIndexScheduler::setSuspend(bool suspend) { if (suspend) { - qDebug() << "Suspending"; + qCDebug(BALOO) << "Suspending"; if (m_indexerState == ContentIndexing) { m_contentIndexer->quit(); } m_indexerState = Suspended; Q_EMIT stateChanged(m_indexerState); } else { - qDebug() << "Resuming"; + qCDebug(BALOO) << "Resuming"; m_indexerState = Idle; // No need to emit here we'll be emitting in scheduling scheduleIndexing(); } } uint FileIndexScheduler::getRemainingTime() { if (m_indexerState != ContentIndexing) { return 0; } return m_timeEstimator.calculateTimeLeft(m_provider.size()); } void FileIndexScheduler::scheduleCheckUnindexedFiles() { m_checkUnindexedFiles = true; } void FileIndexScheduler::checkUnindexedFiles() { m_checkUnindexedFiles = true; scheduleIndexing(); } void FileIndexScheduler::scheduleCheckStaleIndexEntries() { m_checkStaleIndexEntries = true; } void FileIndexScheduler::checkStaleIndexEntries() { m_checkStaleIndexEntries = true; scheduleIndexing(); } uint FileIndexScheduler::getBatchSize() { return m_config->maxUncomittedFiles(); } diff --git a/src/file/indexcleaner.cpp b/src/file/indexcleaner.cpp index 20eabe15..7d9cf10b 100644 --- a/src/file/indexcleaner.cpp +++ b/src/file/indexcleaner.cpp @@ -1,82 +1,83 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "indexcleaner.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include "idutils.h" -#include +#include "baloodebug.h" + #include #include using namespace Baloo; IndexCleaner::IndexCleaner(Database* db, FileIndexerConfig* config) : m_db(db) , m_config(config) { Q_ASSERT(db); Q_ASSERT(config); } void IndexCleaner::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); auto shouldDelete = [&](quint64 id) { if (!id) { return false; } QString url = tr.documentUrl(id); if (!QFile::exists(url)) { - qDebug() << "not exists: " << url; + qCDebug(BALOO) << "not exists: " << url; return true; } if (!m_config->shouldBeIndexed(url)) { - qDebug() << "should not be indexed: " << url; + qCDebug(BALOO) << "should not be indexed: " << url; return true; } // FIXME: This mimetype is not completely accurate! QString mimetype = mimeDb.mimeTypeForFile(url, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { - qDebug() << "mimetype should not be indexed: " << url << mimetype; + qCDebug(BALOO) << "mimetype should not be indexed: " << url << mimetype; return true; } return false; }; const auto includeFolders = m_config->includeFolders(); for (const QString& folder : includeFolders) { quint64 id = filePathToId(QFile::encodeName(folder)); tr.removeRecursively(id, shouldDelete); } tr.commit(); Q_EMIT done(); } diff --git a/src/file/main.cpp b/src/file/main.cpp index 52c42e98..604dbc96 100644 --- a/src/file/main.cpp +++ b/src/file/main.cpp @@ -1,105 +1,104 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013-2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include #include #include -#include #include #include #include "global.h" #include "database.h" #include "fileindexerconfig.h" #include "priority.h" #include "migrator.h" #include "mainhub.h" #include #include int main(int argc, char** argv) { lowerIOPriority(); lowerSchedulingPriority(); lowerPriority(); KAboutData aboutData(QStringLiteral("baloo"), i18n("Baloo File Indexing Daemon"), PROJECT_VERSION); aboutData.addAuthor(i18n("Vishesh Handa"), i18n("Maintainer"), QStringLiteral("vhanda@kde.org"), QStringLiteral("http://vhanda.in")); QCoreApplication app(argc, argv); KAboutData::setApplicationData(aboutData); Baloo::FileIndexerConfig indexerConfig; if (!indexerConfig.indexingEnabled()) { std::cout << "Baloo File Indexing has been disabled" << std::endl; return 0; } if (!QDBusConnection::sessionBus().registerService(QStringLiteral("org.kde.baloo"))) { qWarning() << "Failed to register via dbus. Another instance is running"; return 1; } // Crash Handling KCrash::setFlags(KCrash::AutoRestart); const QString path = Baloo::fileIndexDbPath(); Baloo::Migrator migrator(path, &indexerConfig); if (migrator.migrationRequired()) { migrator.migrate(); } if (!QFile::exists(path + "/index")) { indexerConfig.setInitialRun(true); } // HACK: Untill we start using lmdb with robust mutex support. We're just going to remove // the lock manually in the baloo_file process. QFile::remove(path + "/index-lock"); Baloo::Database *db = Baloo::globalDatabaseInstance(); /** * try to open, if that fails, try to unlink the index db and retry */ if (!db->open(Baloo::Database::CreateDatabase)) { // delete old stuff, set to initial run! qWarning() << "Failed to create database, removing corrupted database."; QFile::remove(path + "/index"); QFile::remove(path + "/index-lock"); indexerConfig.setInitialRun(true); // try to create now after cleanup, if still no works => fail if (!db->open(Baloo::Database::CreateDatabase)) { qWarning() << "Failed to create database after deleting corrupted one."; return 1; } } Baloo::MainHub hub(db, &indexerConfig); return app.exec(); } diff --git a/src/file/metadatamover.cpp b/src/file/metadatamover.cpp index c5d2a891..663daff9 100644 --- a/src/file/metadatamover.cpp +++ b/src/file/metadatamover.cpp @@ -1,198 +1,198 @@ /* This file is part of the KDE Project Copyright (c) 2009-2011 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "metadatamover.h" #include "database.h" #include "transaction.h" #include "basicindexingjob.h" #include "idutils.h" #include "mainhub.h" #include "baloodebug.h" #include "baloowatcherapplication_interface.h" #include #include #include using namespace Baloo; MetadataMover::MetadataMover(Database* db, QObject* parent) : QObject(parent) , m_db(db) { m_serviceWatcher.setConnection(QDBusConnection::sessionBus()); m_serviceWatcher.setWatchMode(QDBusServiceWatcher::WatchForUnregistration); connect(&m_serviceWatcher, &QDBusServiceWatcher::serviceUnregistered, this, &MetadataMover::watcherServiceUnregistered); } MetadataMover::~MetadataMover() { } bool MetadataMover::hasWatcher() const { return !m_watcherApplications.isEmpty(); } static void buildRecursiveList(quint64 parentId, QList &fileList, Transaction &tr) { fileList.push_back(QFile::decodeName(tr.documentUrl(parentId))); const auto childrenIds = tr.childrenDocumentId(parentId); for (const auto oneChildren : childrenIds) { buildRecursiveList(oneChildren, fileList, tr); } } void MetadataMover::moveFileMetadata(const QString& from, const QString& to) { // qCDebug(BALOO) << from << to; Q_ASSERT(!from.isEmpty() && from != QLatin1String("/")); Q_ASSERT(!to.isEmpty() && to != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); quint64 id = tr.documentId(QFile::encodeName(from)); QList filesList; qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << (hasWatcher() ? "has watcher" : "has no watcher"); qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "id" << id; if (id && hasWatcher()) { buildRecursiveList(id, filesList, tr); } // We do NOT get deleted messages for overwritten files! Thus, we // have to remove all metadata for overwritten files first. removeMetadata(&tr, to); // and finally update the old statements updateMetadata(&tr, from, to); tr.commit(); if (hasWatcher()) { qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "notifyWatchers" << filesList; notifyWatchers(from, to, filesList); } } void MetadataMover::removeFileMetadata(const QString& file) { Q_ASSERT(!file.isEmpty() && file != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); removeMetadata(&tr, file); tr.commit(); } void MetadataMover::registerBalooWatcher(const QString &service) { int firstSlash = service.indexOf('/'); if (firstSlash == -1) { return; } QString dbusServiceName = service.left(firstSlash); QString dbusPath = service.mid(firstSlash); m_serviceWatcher.addWatchedService(dbusServiceName); m_watcherApplications.insert(dbusServiceName, new org::kde::BalooWatcherApplication(dbusServiceName, dbusPath, QDBusConnection::sessionBus(), this)); qCDebug(BALOO) << "MetadataMover::registerBalooWatcher" << service << dbusServiceName << dbusPath; } void MetadataMover::removeMetadata(Transaction* tr, const QString& url) { Q_ASSERT(!url.isEmpty()); quint64 id = tr->documentId(QFile::encodeName(url)); if (!id) { Q_EMIT fileRemoved(url); return; } bool isDir = url.endsWith('/'); if (!isDir) { tr->removeDocument(id); } else { tr->removeRecursively(id); } Q_EMIT fileRemoved(url); } void MetadataMover::updateMetadata(Transaction* tr, const QString& from, const QString& to) { qCDebug(BALOO) << from << "->" << to; Q_ASSERT(!from.isEmpty() && !to.isEmpty()); Q_ASSERT(from[from.size()-1] != QLatin1Char('/')); Q_ASSERT(to[to.size()-1] != QLatin1Char('/')); QByteArray toPath = QFile::encodeName(to); quint64 id = filePathToId(toPath); if (!id) { qWarning() << "File moved to path which now no longer exists -" << to; return; } if (!tr->hasDocument(id)) { // // If we have no metadata yet we need to tell the file indexer so it can // create the metadata in case the target folder is configured to be indexed. // qCDebug(BALOO) << "Moved without data"; Q_EMIT movedWithoutData(to); return; } BasicIndexingJob job(toPath, QString(), BasicIndexingJob::NoLevel); job.index(); tr->replaceDocument(job.document(), DocumentUrl | FileNameTerms); // Possible scenarios // 1. file moves to the same device - id is preserved // 2. file moves to a different device - id is not preserved } void MetadataMover::notifyWatchers(const QString &from, const QString &to, const QList &filesList) { Q_FOREACH(org::kde::BalooWatcherApplication *watcherApplication, m_watcherApplications) { qCDebug(BALOO) << "MetadataMover::notifyWatchers" << watcherApplication->service() << watcherApplication->objectName() << watcherApplication->path(); watcherApplication->renamedFiles(from, to, filesList); } } void MetadataMover::watcherServiceUnregistered(const QString &serviceName) { auto itService = m_watcherApplications.find(serviceName); if (itService == m_watcherApplications.end()) { return; } - qDebug() << "MetadataMover::watcherServiceUnregistered" << itService.key(); + qCDebug(BALOO) << "MetadataMover::watcherServiceUnregistered" << itService.key(); delete itService.value(); m_watcherApplications.erase(itService); }