diff --git a/autotests/unit/file/CMakeLists.txt b/autotests/unit/file/CMakeLists.txt index c8703bea..1c75c14c 100644 --- a/autotests/unit/file/CMakeLists.txt +++ b/autotests/unit/file/CMakeLists.txt @@ -1,51 +1,42 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") ecm_add_test(kinotifytest.cpp TEST_NAME "kinotifytest" LINK_LIBRARIES Qt5::Test baloofilecommon ) endif() -set (metadatamovertest_SOURCES - metadatamovertest.cpp -) - -qt5_add_dbus_adaptor(metadatamovertest_SOURCES - ${CMAKE_SOURCE_DIR}/src/file/org.kde.BalooWatcherApplication.xml - metadatamovertest.h MetadataMoverTestDBusSpy) - -ecm_add_test(${metadatamovertest_SOURCES} TEST_NAME metadatamovertest LINK_LIBRARIES Qt5::Test baloofilecommon KF5::Baloo) - MACRO(BALOO_FILE_AUTO_TESTS) FOREACH(_testname ${ARGN}) ecm_add_test(${_testname}.cpp TEST_NAME ${_testname} LINK_LIBRARIES Qt5::Test baloofilecommon KF5::Baloo) ENDFOREACH() ENDMACRO() baloo_file_auto_tests( pendingfilequeuetest fileindexerconfigtest basicindexingjobtest regularexpcachebenchmark filtereddiriteratortest unindexedfileiteratortest + metadatamovertest fileinfotest ) # # File Watch # set(fileWatch_SRC filewatchtest.cpp ../lib/xattrdetector.cpp) ecm_add_test(${fileWatch_SRC} TEST_NAME "filewatchtest" LINK_LIBRARIES Qt5::Test Qt5::DBus KF5::Baloo baloofilecommon ) # # Property Serialization # ecm_add_test(propertyserializationtest.cpp TEST_NAME "propertyserializationtest" LINK_LIBRARIES Qt5::Test KF5::FileMetaData baloofilecommon ) diff --git a/autotests/unit/file/filewatchtest.cpp b/autotests/unit/file/filewatchtest.cpp index acf838e8..c153f3a2 100644 --- a/autotests/unit/file/filewatchtest.cpp +++ b/autotests/unit/file/filewatchtest.cpp @@ -1,181 +1,150 @@ /* * * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filewatch.h" #include "fileindexerconfigutils.h" #include "database.h" #include "fileindexerconfig.h" #include "pendingfilequeue.h" -#include "mainhub.h" #include "../lib/xattrdetector.h" #include "../lib/baloo_xattr_p.h" #include #include #include namespace Baloo { class FileWatchTest : public QObject { Q_OBJECT private Q_SLOTS: - void init() - { - m_tempDir = new QTemporaryDir(); - m_db = new Database(m_tempDir->path()); - m_db->open(Database::CreateDatabase); - m_dbusInterface = new MainHub(m_db, &m_config, false); - } - void testFileCreation(); - - void cleanupTestCase() - { - delete m_dbusInterface; - m_dbusInterface = nullptr; - - delete m_db; - m_db = nullptr; - - delete m_tempDir; - m_tempDir = nullptr; - } - -private: - - FileIndexerConfig m_config; - - Database* m_db; - - QTemporaryDir* m_tempDir; - - Baloo::MainHub *m_dbusInterface; }; } using namespace Baloo; namespace { bool createFile(const QString& fileUrl) { QFile f1(fileUrl); f1.open(QIODevice::WriteOnly); f1.close(); return QFile::exists(fileUrl); } void modifyFile(const QString& fileUrl) { QFile f1(fileUrl); f1.open(QIODevice::Append | QIODevice::Text); QTextStream stream(&f1); stream << "1"; } } void FileWatchTest::testFileCreation() { QTemporaryDir includeDir; QStringList includeFolders; includeFolders << includeDir.path(); QStringList excludeFolders; Test::writeIndexerConfig(includeFolders, excludeFolders); QTemporaryDir dbDir; Database db(dbDir.path()); db.open(Baloo::Database::CreateDatabase); FileIndexerConfig config; - FileWatch fileWatch(&db, &config, m_dbusInterface); + FileWatch fileWatch(&db, &config); fileWatch.m_pendingFileQueue->setMaximumTimeout(0); fileWatch.m_pendingFileQueue->setMinimumTimeout(0); fileWatch.m_pendingFileQueue->setTrackingTime(0); QSignalSpy spy(&fileWatch, SIGNAL(installedWatches())); QVERIFY(spy.isValid()); fileWatch.watchIndexedFolders(); QVERIFY(spy.count() || spy.wait()); QSignalSpy spyIndexNew(&fileWatch, SIGNAL(indexNewFile(QString))); QSignalSpy spyIndexModified(&fileWatch, SIGNAL(indexModifiedFile(QString))); QSignalSpy spyIndexXattr(&fileWatch, SIGNAL(indexXAttr(QString))); QVERIFY(spyIndexNew.isValid()); QVERIFY(spyIndexModified.isValid()); QVERIFY(spyIndexXattr.isValid()); // Create a file and see if it is indexed QString fileUrl(includeDir.path() + "/t1"); QVERIFY(createFile(fileUrl)); QVERIFY(spyIndexNew.wait()); QCOMPARE(spyIndexNew.count(), 1); QCOMPARE(spyIndexModified.count(), 0); QCOMPARE(spyIndexXattr.count(), 0); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); // // Modify the file // modifyFile(fileUrl); QVERIFY(spyIndexModified.wait()); QCOMPARE(spyIndexNew.count(), 0); QCOMPARE(spyIndexModified.count(), 1); QCOMPARE(spyIndexXattr.count(), 0); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); // // Set an Xattr // XattrDetector detector; if (!detector.isSupported(dbDir.path())) { qWarning() << "Xattr not supported on this filesystem"; return; } const QString userComment(QStringLiteral("UserComment")); QVERIFY(baloo_setxattr(fileUrl, QLatin1String("user.xdg.comment"), userComment) != -1); QVERIFY(spyIndexXattr.wait()); QCOMPARE(spyIndexNew.count(), 0); QCOMPARE(spyIndexModified.count(), 0); QCOMPARE(spyIndexXattr.count(), 1); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); } QTEST_MAIN(FileWatchTest) #include "filewatchtest.moc" diff --git a/autotests/unit/file/metadatamovertest.cpp b/autotests/unit/file/metadatamovertest.cpp index d20b899b..18925772 100644 --- a/autotests/unit/file/metadatamovertest.cpp +++ b/autotests/unit/file/metadatamovertest.cpp @@ -1,322 +1,213 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013-2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ -#include "metadatamovertest.h" #include "metadatamover.h" -#include "baloowatcherapplicationadaptor.h" #include "database.h" #include "transaction.h" #include "document.h" #include "basicindexingjob.h" -#include "mainhub.h" #include #include #include #include #include -#include using namespace Baloo; class MetadataMoverTest : public QObject { Q_OBJECT public: MetadataMoverTest(QObject* parent = nullptr); private Q_SLOTS: void init(); void cleanupTestCase(); void testRemoveFile(); void testRenameFile(); void testMoveFile(); void testMoveFolder(); private: quint64 insertUrl(const QString& url); - FileIndexerConfig m_config; - Database* m_db; - QTemporaryDir* m_tempDir; }; MetadataMoverTest::MetadataMoverTest(QObject* parent) : QObject(parent) , m_db(nullptr) , m_tempDir(nullptr) { } void MetadataMoverTest::init() { m_tempDir = new QTemporaryDir(); m_db = new Database(m_tempDir->path()); m_db->open(Database::CreateDatabase); } void MetadataMoverTest::cleanupTestCase() { delete m_db; m_db = nullptr; delete m_tempDir; m_tempDir = nullptr; } quint64 MetadataMoverTest::insertUrl(const QString& url) { BasicIndexingJob job(url, QStringLiteral("text/plain")); job.index(); Transaction tr(m_db, Transaction::ReadWrite); tr.addDocument(job.document()); tr.commit(); return job.document().id(); } void MetadataMoverTest::testRemoveFile() { - MetadataMoverTestDBusSpy dbusSignalSpy; - QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); - QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); - QTemporaryFile file; file.open(); QString url = file.fileName(); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 0); - QCOMPARE(fileChangedSpy.count(), 0); - MetadataMover mover(m_db, this); - - mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); - file.remove(); mover.removeFileMetadata(url); - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 0); - QCOMPARE(fileChangedSpy.count(), 0); - { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(!tr.hasDocument(fid)); } } static void touchFile(const QString& path) { QFile file(path); file.open(QIODevice::WriteOnly); QTextStream s(&file); s << "random"; } static void mkdir(const QString& path) { QDir().mkpath(path); QVERIFY(QDir(path).exists()); } void MetadataMoverTest::testRenameFile() { - MetadataMoverTestDBusSpy dbusSignalSpy; - QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); - QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); - QTemporaryDir dir; QString url = dir.path() + "/file"; touchFile(url); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 0); - QCOMPARE(fileChangedSpy.count(), 0); - MetadataMover mover(m_db, this); - - mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); - QString url2 = dir.path() + "/file2"; QFile::rename(url, url2); mover.moveFileMetadata(QFile::encodeName(url), QFile::encodeName(url2)); - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 1); - QCOMPARE(fileChangedSpy.count(), 0); - { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(url2)); } } void MetadataMoverTest::testMoveFile() { - MetadataMoverTestDBusSpy dbusSignalSpy; - QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); - QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); - QTemporaryDir dir; QDir().mkpath(dir.path() + "/a/b/c"); QString url = dir.path() + "/a/b/c/file"; touchFile(url); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 0); - QCOMPARE(fileChangedSpy.count(), 0); - MetadataMover mover(m_db, this); - - mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); - QString url2 = dir.path() + "/file2"; QFile::rename(url, url2); mover.moveFileMetadata(QFile::encodeName(url), QFile::encodeName(url2)); - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 1); - QCOMPARE(fileChangedSpy.count(), 0); - { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(url2)); } } void MetadataMoverTest::testMoveFolder() { - MetadataMoverTestDBusSpy dbusSignalSpy; - QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); - QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); - QTemporaryDir dir; QString folder = dir.path() + "/folder"; mkdir(folder); quint64 did = insertUrl(folder); QString fileUrl = folder + "/file"; touchFile(fileUrl); quint64 fid = insertUrl(fileUrl); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(did)); QVERIFY(tr.hasDocument(fid)); } - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 0); - QCOMPARE(fileChangedSpy.count(), 0); - QString newFolderUrl = dir.path() + "/dir"; QFile::rename(folder, newFolderUrl); MetadataMover mover(m_db, this); - - mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); - mover.moveFileMetadata(QFile::encodeName(folder), QFile::encodeName(newFolderUrl)); - renamedFilesSignalSpy.wait(100); - fileChangedSpy.wait(100); - - QCOMPARE(renamedFilesSignalSpy.count(), 1); - QCOMPARE(fileChangedSpy.count(), 0); - { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(did)); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(did), QFile::encodeName(newFolderUrl)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(newFolderUrl + "/file")); } } - - -MetadataMoverTestDBusSpy::MetadataMoverTestDBusSpy(QObject *parent) : QObject(parent) -{ - QDBusConnection con = QDBusConnection::sessionBus(); - - con.connect(QString(), QStringLiteral("/files"), QStringLiteral("org.kde"), - QStringLiteral("changed"), this, SLOT(slotFileMetaDataChanged(QStringList))); - - - auto mDbusAdaptor = new BalooWatcherApplicationAdaptor(this); - - QCOMPARE(con.registerService(QStringLiteral("org.kde.baloo.metadatamovertest")), true); - QCOMPARE(con.registerObject(QStringLiteral("/org/kde/BalooWatcherApplication"), mDbusAdaptor, QDBusConnection::ExportAllContents), true); -} - -void MetadataMoverTestDBusSpy::slotFileMetaDataChanged(QStringList fileList) -{ - Q_EMIT fileMetaDataChanged(fileList); -} - -void MetadataMoverTestDBusSpy::renamedFiles(const QString &from, const QString &to, const QStringList &listFiles) -{ - Q_EMIT renamedFilesSignal(from, to, listFiles); -} - QTEST_GUILESS_MAIN(MetadataMoverTest) #include "metadatamovertest.moc" diff --git a/autotests/unit/file/metadatamovertest.h b/autotests/unit/file/metadatamovertest.h deleted file mode 100644 index 9785c815..00000000 --- a/autotests/unit/file/metadatamovertest.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * This file is part of the KDE Baloo Project - * Copyright (C) 2017 Matthieu Gallien - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - * - */ - -#if !defined METADATAMOVERTEST_H_ -#define METADATAMOVERTEST_H_ - -#include -#include -#include - -class MetadataMoverTestDBusSpy : public QObject -{ - Q_OBJECT -public: - - MetadataMoverTestDBusSpy(QObject* parent = nullptr); - -Q_SIGNALS: - - void fileMetaDataChanged(QStringList fileList); - - void renamedFilesSignal(const QString &from, const QString &to, const QStringList &listFiles); - -public Q_SLOTS: - - void slotFileMetaDataChanged(QStringList fileList); - - void renamedFiles(const QString &from, const QString &to, const QStringList &listFiles); - -}; - -#endif diff --git a/src/engine/transaction.cpp b/src/engine/transaction.cpp index eb50d7d7..fa47e5d8 100644 --- a/src/engine/transaction.cpp +++ b/src/engine/transaction.cpp @@ -1,655 +1,648 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "transaction.h" #include "postingdb.h" #include "documentdb.h" #include "documenturldb.h" #include "documentiddb.h" #include "positiondb.h" #include "documentdatadb.h" #include "mtimedb.h" #include "document.h" #include "enginequery.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "phraseanditerator.h" #include "writetransaction.h" #include "idutils.h" #include "database.h" #include "databasesize.h" #include "enginedebug.h" #include #include #include using namespace Baloo; Transaction::Transaction(const Database& db, Transaction::TransactionType type) : m_dbis(db.m_dbis) , m_env(db.m_env) , m_writeTrans(nullptr) { uint flags = type == ReadOnly ? MDB_RDONLY : 0; int rc = mdb_txn_begin(db.m_env, nullptr, flags, &m_txn); if (rc) { qCDebug(ENGINE) << "Transaction" << mdb_strerror(rc); return; } if (type == ReadWrite) { m_writeTrans = new WriteTransaction(m_dbis, m_txn); } } Transaction::Transaction(Database* db, Transaction::TransactionType type) : Transaction(*db, type) { } Transaction::~Transaction() { if (m_writeTrans) { qWarning(ENGINE) << "Closing an active WriteTransaction without calling abort/commit"; } if (m_txn) { abort(); } } bool Transaction::hasDocument(quint64 id) const { Q_ASSERT(id > 0); IdFilenameDB idFilenameDb(m_dbis.idFilenameDbi, m_txn); return idFilenameDb.contains(id); } bool Transaction::inPhaseOne(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.contains(id); } bool Transaction::hasFailed(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); return failedIdDb.contains(id); } QVector Transaction::failedIds(quint64 limit) const { DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); return failedIdDb.fetchItems(limit); } QByteArray Transaction::documentUrl(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.get(id); } quint64 Transaction::documentId(const QByteArray& path) const { Q_ASSERT(m_txn); Q_ASSERT(!path.isEmpty()); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); QList li = path.split('/'); quint64 parentId = 0; for (const QByteArray& fileName : li) { if (fileName.isEmpty()) { continue; } parentId = docUrlDb.getId(parentId, fileName); if (!parentId) { return 0; } } return parentId; } -QVector Transaction::childrenDocumentId(quint64 parentId) const -{ - DocumentUrlDB docUrlDB(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); - - return docUrlDB.getChildren(parentId); -} - DocumentTimeDB::TimeInfo Transaction::documentTimeInfo(quint64 id) const { Q_ASSERT(m_txn); DocumentTimeDB docTimeDb(m_dbis.docTimeDbi, m_txn); return docTimeDb.get(id); } QByteArray Transaction::documentData(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentDataDB docDataDb(m_dbis.docDataDbi, m_txn); return docDataDb.get(id); } bool Transaction::hasChanges() const { Q_ASSERT(m_txn); if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return false; } return m_writeTrans->hasChanges(); } QVector Transaction::fetchPhaseOneIds(int size) const { Q_ASSERT(m_txn); Q_ASSERT(size > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.fetchItems(size); } QVector Transaction::fetchTermsStartingWith(const QByteArray& term) const { Q_ASSERT(term.size() > 0); PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.fetchTermsStartingWith(term); } uint Transaction::phaseOneSize() const { Q_ASSERT(m_txn); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.size(); } uint Transaction::size() const { Q_ASSERT(m_txn); DocumentDB docTermsDb(m_dbis.docTermsDbi, m_txn); return docTermsDb.size(); } // // Write Operations // void Transaction::setPhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.put(id); } void Transaction::removePhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.del(id); } void Transaction::addFailed(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); failedIdDb.put(id); } void Transaction::addDocument(const Document& doc) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return; } m_writeTrans->addDocument(doc); } void Transaction::removeDocument(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return; } m_writeTrans->removeDocument(id); } void Transaction::removeRecursively(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return; } m_writeTrans->removeRecursively(id); } void Transaction::replaceDocument(const Document& doc, DocumentOperations operations) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); if (!hasDocument(doc.id())) { qCDebug(ENGINE) << "Transaction::replaceDocument" << "Document does not exist"; } if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return; } m_writeTrans->replaceDocument(doc, operations); } void Transaction::commit() { Q_ASSERT(m_txn); if (!m_writeTrans) { qCWarning(ENGINE) << "m_writeTrans is null"; return; } m_writeTrans->commit(); delete m_writeTrans; m_writeTrans = nullptr; int rc = mdb_txn_commit(m_txn); if (rc) { qCWarning(ENGINE) << "Transaction::commit" << mdb_strerror(rc); } m_txn = nullptr; } void Transaction::abort() { Q_ASSERT(m_txn); mdb_txn_abort(m_txn); m_txn = nullptr; delete m_writeTrans; m_writeTrans = nullptr; } // // Queries // PostingIterator* Transaction::postingIterator(const EngineQuery& query) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); PositionDB positionDb(m_dbis.positionDBi, m_txn); if (query.leaf()) { if (query.op() == EngineQuery::Equal) { return postingDb.iter(query.term()); } else if (query.op() == EngineQuery::StartsWith) { return postingDb.prefixIter(query.term()); } else { Q_ASSERT(0); } } const auto subQueries = query.subQueries(); if (subQueries.isEmpty()) { return nullptr; } if (query.op() == EngineQuery::Phrase) { if (subQueries.size() == 1) { qCDebug(ENGINE) << "Degenerated Phrase with 1 Term:" << query; return postingIterator(subQueries[0]); } QVector vec; vec.reserve(subQueries.size()); for (const EngineQuery& q : subQueries) { if (!q.leaf()) { qCDebug(ENGINE) << "Transaction::toPostingIterator" << "Phrase subqueries must be leafs"; continue; } vec << positionDb.iter(q.term()); } return new PhraseAndIterator(vec); } QVector vec; vec.reserve(subQueries.size()); for (const EngineQuery& q : subQueries) { auto iterator = postingIterator(q); if (iterator) { vec << iterator; } else if (query.op() == EngineQuery::And) { return nullptr; } } if (vec.empty()) { return nullptr; } else if (vec.size() == 1) { return vec.takeFirst(); } if (query.op() == EngineQuery::And) { return new AndPostingIterator(vec); } else if (query.op() == EngineQuery::Or) { return new OrPostingIterator(vec); } Q_ASSERT(0); return nullptr; } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, qlonglong value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, double value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, const QByteArray& value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::mTimeRangeIter(quint32 beginTime, quint32 endTime) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iterRange(beginTime, endTime); } PostingIterator* Transaction::docUrlIter(quint64 id) const { DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.iter(id); } QVector Transaction::exec(const EngineQuery& query, int limit) const { Q_ASSERT(m_txn); QVector results; PostingIterator* it = postingIterator(query); if (!it) { return results; } while (it->next() && limit) { results << it->docId(); limit--; } return results; } // // Introspection // QVector Transaction::documentTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); return documentTermsDB.get(docId); } QVector Transaction::documentFileNameTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); return documentFileNameTermsDB.get(docId); } QVector Transaction::documentXattrTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); return documentXattrTermsDB.get(docId); } // // File Size // static size_t dbiSize(MDB_txn* txn, MDB_dbi dbi) { MDB_stat stat; mdb_stat(txn, dbi, &stat); return (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages) * stat.ms_psize; } DatabaseSize Transaction::dbSize() { DatabaseSize dbSize; dbSize.postingDb = dbiSize(m_txn, m_dbis.postingDbi); dbSize.positionDb = dbiSize(m_txn, m_dbis.positionDBi); dbSize.docTerms = dbiSize(m_txn, m_dbis.docTermsDbi); dbSize.docFilenameTerms = dbiSize(m_txn, m_dbis.docFilenameTermsDbi); dbSize.docXattrTerms = dbiSize(m_txn, m_dbis.docXattrTermsDbi); dbSize.idTree = dbiSize(m_txn, m_dbis.idTreeDbi); dbSize.idFilename = dbiSize(m_txn, m_dbis.idFilenameDbi); dbSize.docTime = dbiSize(m_txn, m_dbis.docTimeDbi); dbSize.docData = dbiSize(m_txn, m_dbis.docDataDbi); dbSize.contentIndexingIds = dbiSize(m_txn, m_dbis.contentIndexingDbi); dbSize.failedIds = dbiSize(m_txn, m_dbis.failedIdDbi); dbSize.mtimeDb = dbiSize(m_txn, m_dbis.mtimeDbi); dbSize.expectedSize = dbSize.postingDb + dbSize.positionDb + dbSize.docTerms + dbSize.docFilenameTerms + dbSize.docXattrTerms + dbSize.idTree + dbSize.idFilename + dbSize.docTime + dbSize.docData + dbSize.contentIndexingIds + dbSize.failedIds + dbSize.mtimeDb; MDB_envinfo info; mdb_env_info(m_env, &info); dbSize.actualSize = info.me_last_pgno * 4096; // TODO: separate page size return dbSize; } // // Debugging // void Transaction::checkFsTree() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); const auto map = postingDb.toTestMap(); QSet allIds; for (const auto& list : map) { for (quint64 id : list) { allIds << id; } } std::cout << "Total Document IDs: " << allIds.size() << std::endl; int count = 0; for (quint64 id: qAsConst(allIds)) { QByteArray url = docUrlDb.get(id); if (url.isEmpty()) { auto terms = documentTermsDB.get(id); auto fileNameTerms = documentFileNameTermsDB.get(id); auto xAttrTerms = documentXattrTermsDB.get(id); // Lets reverse engineer the terms QList newTerms; QMapIterator it(map); while (it.hasNext()) { it.next(); if (it.value().contains(id)) { newTerms << it.key(); } } std::cout << "Missing filePath for " << id << std::endl; std::cout << "\tPostingDB Terms: "; for (const QByteArray& term : qAsConst(newTerms)) { std::cout << qPrintable(term) << " "; } std::cout << std::endl; std::cout << "\tDocumentTermsDB: "; for (const QByteArray& term : terms) { std::cout << qPrintable(term) << " "; } std::cout << std::endl; std::cout << "\tFileNameTermsDB: "; for (const QByteArray& term : fileNameTerms) { std::cout << qPrintable(term) << " "; } std::cout << std::endl; std::cout << "\tXAttrTermsDB: "; for (const QByteArray& term : xAttrTerms) { std::cout << qPrintable(term) << " "; } std::cout << std::endl; count++; } else if (!QFileInfo::exists(QString::fromUtf8(url))) { std::cout << "FilePath " << qPrintable(url) << " for " << id << " does not exist"<< std::endl; count++; } } std::cout << "Invalid Entries: " << count << " (" << count * 100.0 / allIds.size() << "%)" << std::endl; } void Transaction::checkTermsDbinPostingDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); // Iterate over each document, and fetch all terms // check if each term maps to its own id in the posting db const auto map = postingDb.toTestMap(); QSet allIds; for (const auto& list : map) { for (quint64 id : list) { allIds << id; } } std::cout << "PostingDB check .." << std::endl; for (quint64 id : qAsConst(allIds)) { QVector terms = documentTermsDB.get(id); terms += documentXattrTermsDB.get(id); terms += documentFileNameTermsDB.get(id); for (const QByteArray& term : qAsConst(terms)) { PostingList plist = postingDb.get(term); if (!plist.contains(id)) { std::cout << id << " is missing term " << qPrintable(term) << std::endl; } } } } void Transaction::checkPostingDbinTermsDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); QMap map = postingDb.toTestMap(); QMapIterator it(map); std::cout << "DocumentTermsDB check .." << std::endl; while (it.hasNext()) { it.next(); const QByteArray& term = it.key(); const PostingList& list = it.value(); for (quint64 id : list) { if (documentTermsDB.get(id).contains(term)) { continue; } if (documentFileNameTermsDB.get(id).contains(term)) { continue; } if (documentXattrTermsDB.get(id).contains(term)) { continue; } std::cout << id << " is missing " << qPrintable(QString::fromUtf8(term)) << " from document terms db" << std::endl; } } } diff --git a/src/engine/transaction.h b/src/engine/transaction.h index 6c5a8e38..ab5388ef 100644 --- a/src/engine/transaction.h +++ b/src/engine/transaction.h @@ -1,141 +1,140 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_TRANSACTION_H #define BALOO_TRANSACTION_H #include "databasedbis.h" #include "mtimedb.h" #include "postingdb.h" #include "writetransaction.h" #include "documenttimedb.h" #include #include namespace Baloo { class Database; class Document; class PostingIterator; class EngineQuery; class DatabaseSize; class DBState; class BALOO_ENGINE_EXPORT Transaction { public: enum TransactionType { ReadOnly, ReadWrite }; Transaction(const Database& db, TransactionType type); Transaction(Database* db, TransactionType type); ~Transaction(); // // Getters // bool hasDocument(quint64 id) const; bool inPhaseOne(quint64 id) const; bool hasFailed(quint64 id) const; QVector failedIds(quint64 limit) const; QByteArray documentUrl(quint64 id) const; /** * This method is not cheap, and does not stat the filesystem in order to convert the path * \p path into an id. */ quint64 documentId(const QByteArray& path) const; - QVector childrenDocumentId(quint64 parentId) const; QByteArray documentData(quint64 id) const; DocumentTimeDB::TimeInfo documentTimeInfo(quint64 id) const; QVector exec(const EngineQuery& query, int limit = -1) const; PostingIterator* postingIterator(const EngineQuery& query) const; PostingIterator* postingCompIterator(const QByteArray& prefix, qlonglong value, PostingDB::Comparator com) const; PostingIterator* postingCompIterator(const QByteArray& prefix, double value, PostingDB::Comparator com) const; PostingIterator* postingCompIterator(const QByteArray& prefix, const QByteArray& value, PostingDB::Comparator com) const; PostingIterator* mTimeRangeIter(quint32 beginTime, quint32 endTime) const; PostingIterator* docUrlIter(quint64 id) const; QVector fetchPhaseOneIds(int size) const; uint phaseOneSize() const; uint size() const; QVector fetchTermsStartingWith(const QByteArray& term) const; // // Introspecing document data // QVector documentTerms(quint64 docId) const; QVector documentFileNameTerms(quint64 docId) const; QVector documentXattrTerms(quint64 docId) const; DatabaseSize dbSize(); // // Transaction handling // void commit(); void abort(); bool hasChanges() const; // // Write Methods // void addDocument(const Document& doc); void removeDocument(quint64 id); void removeRecursively(quint64 parentId); void addFailed(quint64 id); bool removeRecursively(quint64 parentId, std::function shouldDelete) { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); return m_writeTrans->removeRecursively(parentId, shouldDelete); } void replaceDocument(const Document& doc, DocumentOperations operations); void setPhaseOne(quint64 id); void removePhaseOne(quint64 id); // Debugging void checkFsTree(); void checkTermsDbinPostingDb(); void checkPostingDbinTermsDb(); private: Transaction(const Transaction& rhs) = delete; const DatabaseDbis& m_dbis; MDB_txn *m_txn = nullptr; MDB_env *m_env = nullptr; WriteTransaction *m_writeTrans = nullptr; friend class DatabaseSanitizerImpl; friend class DBState; // for testing }; } #endif // BALOO_TRANSACTION_H diff --git a/src/file/CMakeLists.txt b/src/file/CMakeLists.txt index 19608420..cdcee03c 100644 --- a/src/file/CMakeLists.txt +++ b/src/file/CMakeLists.txt @@ -1,84 +1,85 @@ include_directories(${Inotify_INCLUDE_DIRS}) add_definitions(-DTRANSLATION_DOMAIN=\"baloo_file5\") set(file_static_lib_SRCS # File Indexer mainhub.cpp mainadaptor.cpp fileindexerconfig.cpp basicindexingjob.cpp powerstatemonitor.cpp fileindexscheduler.cpp firstrunindexer.cpp newfileindexer.cpp xattrindexer.cpp modifiedfileindexer.cpp unindexedfileindexer.cpp filecontentindexer.cpp filecontentindexerprovider.cpp extractorprocess.cpp timeestimator.cpp indexcleaner.cpp # Common priority.cpp regexpcache.cpp fileexcludefilters.cpp storagedevices.cpp filtereddiriterator.cpp unindexedfileiterator.cpp migrator.cpp fileinfo.cpp # File Watcher filewatch.cpp pendingfilequeue.cpp metadatamover.cpp org.kde.BalooWatcherApplication.xml pendingfile.cpp kinotify.cpp propertydata.cpp ) ecm_qt_declare_logging_category(file_static_lib_SRCS HEADER baloodebug.h IDENTIFIER BALOO CATEGORY_NAME org.kde.baloo) +# KF6 TODO - remove BalooWatcherApplication qt5_add_dbus_interface(file_static_lib_SRCS org.kde.BalooWatcherApplication.xml baloowatcherapplication_interface) kconfig_add_kcfg_files(file_static_lib_SRCS ../lib/baloosettings.kcfgc GENERATE_MOC) add_library(baloofilecommon STATIC ${file_static_lib_SRCS}) target_link_libraries(baloofilecommon Qt5::DBus KF5::I18n KF5::Solid KF5::FileMetaData KF5::Crash KF5::ConfigCore KF5::BalooEngine ${Inotify_LIBRARIES} ) set(file_SRCS main.cpp ) add_executable(baloo_file ${file_SRCS}) ecm_mark_nongui_executable(baloo_file) target_compile_definitions(baloo_file PRIVATE -DPROJECT_VERSION="${PROJECT_VERSION}") target_link_libraries(baloo_file baloofilecommon ) install(TARGETS baloo_file ${INSTALL_TARGETS_DEFAULT_ARGS}) install(FILES baloo_file.desktop DESTINATION ${AUTOSTART_INSTALL_DIR}) add_subdirectory(extractor) install(FILES org.kde.BalooWatcherApplication.xml DESTINATION ${KDE_INSTALL_DBUSINTERFACEDIR}) diff --git a/src/file/filewatch.cpp b/src/file/filewatch.cpp index 1d1f5b1e..f5ea30bd 100644 --- a/src/file/filewatch.cpp +++ b/src/file/filewatch.cpp @@ -1,216 +1,207 @@ /* This file is part of the KDE Project Copyright (c) 2007-2011 Sebastian Trueg Copyright (c) 2012-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "filewatch.h" #include "metadatamover.h" #include "fileindexerconfig.h" #include "pendingfilequeue.h" #include "regexpcache.h" #include "database.h" #include "pendingfile.h" #include "baloodebug.h" #include "kinotify.h" #include #include #include #include using namespace Baloo; -FileWatch::FileWatch(Database* db, FileIndexerConfig* config, Baloo::MainHub *dbusInterface, QObject* parent) +FileWatch::FileWatch(Database* db, FileIndexerConfig* config, QObject* parent) : QObject(parent) , m_db(db) , m_config(config) , m_dirWatch(nullptr) { Q_ASSERT(db); Q_ASSERT(config); m_metadataMover = new MetadataMover(m_db, this); connect(m_metadataMover, &MetadataMover::movedWithoutData, this, &FileWatch::indexNewFile); connect(m_metadataMover, &MetadataMover::fileRemoved, this, &FileWatch::fileRemoved); m_pendingFileQueue = new PendingFileQueue(this); connect(m_pendingFileQueue, &PendingFileQueue::indexNewFile, this, &FileWatch::indexNewFile); connect(m_pendingFileQueue, &PendingFileQueue::indexModifiedFile, this, &FileWatch::indexModifiedFile); connect(m_pendingFileQueue, &PendingFileQueue::indexXAttr, this, &FileWatch::indexXAttr); connect(m_pendingFileQueue, &PendingFileQueue::removeFileIndex, m_metadataMover, &MetadataMover::removeFileMetadata); // monitor the file system for changes (restricted by the inotify limit) m_dirWatch = new KInotify(m_config, this); connect(m_dirWatch, &KInotify::moved, this, &FileWatch::slotFileMoved); connect(m_dirWatch, &KInotify::deleted, this, &FileWatch::slotFileDeleted); connect(m_dirWatch, &KInotify::created, this, &FileWatch::slotFileCreated); connect(m_dirWatch, &KInotify::modified, this, &FileWatch::slotFileModified); connect(m_dirWatch, &KInotify::closedWrite, this, &FileWatch::slotFileClosedAfterWrite); connect(m_dirWatch, &KInotify::attributeChanged, this, &FileWatch::slotAttributeChanged); connect(m_dirWatch, &KInotify::watchUserLimitReached, this, &FileWatch::slotInotifyWatchUserLimitReached); connect(m_dirWatch, &KInotify::installedWatches, this, &FileWatch::installedWatches); } FileWatch::~FileWatch() { } void FileWatch::watchIndexedFolders() { // Watch all indexed folders const QStringList folders = m_config->includeFolders(); for (const QString& folder : folders) { watchFolder(folder); } } -void FileWatch::registerBalooWatcher(const QString &service) -{ - if (!m_metadataMover) { - return; - } - - m_metadataMover->registerBalooWatcher(service); -} - // FIXME: listen to Create for folders! void FileWatch::watchFolder(const QString& path) { qCDebug(BALOO) << path; if (m_dirWatch && !m_dirWatch->watchingPath(path)) { KInotify::WatchEvents flags(KInotify::EventMove | KInotify::EventDelete | KInotify::EventDeleteSelf | KInotify::EventCloseWrite | KInotify::EventCreate | KInotify::EventAttributeChange | KInotify::EventModify); m_dirWatch->addWatch(path, flags, KInotify::FlagOnlyDir); } } void FileWatch::slotFileMoved(const QString& urlFrom, const QString& urlTo) { if (m_config->shouldBeIndexed(urlTo)) { m_metadataMover->moveFileMetadata(urlFrom, urlTo); } else { QFileInfo src(urlFrom); QString url = urlFrom; if (src.isDir()) { Q_ASSERT(!url.endsWith('/')); url.append(QLatin1Char('/')); } PendingFile file(url); file.setDeleted(); m_pendingFileQueue->enqueue(file); } } void FileWatch::slotFileDeleted(const QString& urlString, bool isDir) { // Directories must always end with a trailing slash '/' QString url = urlString; if (isDir) { Q_ASSERT(!url.endsWith('/')); url.append(QLatin1Char('/')); } PendingFile file(url); file.setDeleted(); m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileCreated(const QString& path, bool isDir) { Q_UNUSED(isDir); PendingFile file(path); file.setCreated(); m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileModified(const QString& path) { PendingFile file(path); file.setModified(); //qCDebug(BALOO) << "MOD" << path; m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileClosedAfterWrite(const QString& path) { QDateTime current = QDateTime::currentDateTime(); QDateTime fileModification = QFileInfo(path).lastModified(); QDateTime dirModification = QFileInfo(QFileInfo(path).absoluteDir().absolutePath()).lastModified(); // If we have received a FileClosedAfterWrite event, then the file must have been // closed within the last minute. // This is being done cause many applications open the file under write mode, do not // make any modifications and then close the file. This results in us getting // the FileClosedAfterWrite event if (fileModification.secsTo(current) <= 1000 * 60 || dirModification.secsTo(current) <= 1000 * 60) { PendingFile file(path); file.setClosedOnWrite(); //qCDebug(BALOO) << "CLOSE" << path; m_pendingFileQueue->enqueue(file); } } void FileWatch::slotAttributeChanged(const QString& path) { PendingFile file(path); file.setAttributeChanged(); m_pendingFileQueue->enqueue(file); } // This slot is connected to a signal emitted in KInotify when // inotify_add_watch fails with ENOSPC. void FileWatch::slotInotifyWatchUserLimitReached(const QString&) { //If we got here, we hit the limit and couldn't authenticate to raise it, // so put something in the syslog so someone notices. syslog(LOG_USER | LOG_WARNING, "KDE Baloo File Indexer has reached the inotify folder watch limit. File changes will be ignored."); // we do it the brutal way for now hoping with new kernels and defaults this will never happen // Delete the KInotify // FIXME: Maybe we should be aborting? if (m_dirWatch) { m_dirWatch->deleteLater(); m_dirWatch = nullptr; } Q_EMIT installedWatches(); } void FileWatch::updateIndexedFoldersWatches() { if (m_dirWatch) { const QStringList folders = m_config->includeFolders(); for (const QString& folder : folders) { m_dirWatch->removeWatch(folder); watchFolder(folder); } } } diff --git a/src/file/filewatch.h b/src/file/filewatch.h index e1287223..568e892c 100644 --- a/src/file/filewatch.h +++ b/src/file/filewatch.h @@ -1,96 +1,93 @@ /* This file is part of the KDE Project Copyright (c) 2007-2011 Sebastian Trueg This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef BALOO_FILE_WATCH_H_ #define BALOO_FILE_WATCH_H_ #include #include "pendingfile.h" class KInotify; namespace Baloo { class Database; class MetadataMover; class FileIndexerConfig; class PendingFileQueue; class FileWatchTest; -class MainHub; class FileWatch : public QObject { Q_OBJECT public: - FileWatch(Database* db, FileIndexerConfig* config, Baloo::MainHub *dbusInterface, QObject* parent = nullptr); + FileWatch(Database* db, FileIndexerConfig* config, QObject* parent = nullptr); ~FileWatch(); public Q_SLOTS: /** * To be called whenever the list of indexed folders changes. This is done because * the indexed folders are watched with the 'KInotify::EventCreate' event, and the * non-indexed folders are not. */ void updateIndexedFoldersWatches(); void watchIndexedFolders(); - void registerBalooWatcher(const QString &service); - Q_SIGNALS: void indexNewFile(const QString& string); void indexModifiedFile(const QString& string); void indexXAttr(const QString& path); /** * This signal is emitted when a file has been removed, and everyone else * should update their caches */ void fileRemoved(const QString& path); void installedWatches(); private Q_SLOTS: void slotFileMoved(const QString& from, const QString& to); void slotFileDeleted(const QString& urlString, bool isDir); void slotFileCreated(const QString& path, bool isDir); void slotFileClosedAfterWrite(const QString&); void slotAttributeChanged(const QString& path); void slotFileModified(const QString& path); void slotInotifyWatchUserLimitReached(const QString&); private: /** Watch a folder, provided it is not already watched*/ void watchFolder(const QString& path); Database* m_db; MetadataMover* m_metadataMover; FileIndexerConfig* m_config; KInotify* m_dirWatch; /// queue used to "compress" multiple file events like downloads PendingFileQueue* m_pendingFileQueue; friend class FileWatchTest; }; } #endif diff --git a/src/file/mainhub.cpp b/src/file/mainhub.cpp index 30dba3a8..b3f68fa1 100644 --- a/src/file/mainhub.cpp +++ b/src/file/mainhub.cpp @@ -1,83 +1,82 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "mainhub.h" #include "fileindexerconfig.h" #include "mainadaptor.h" #include #include #include using namespace Baloo; MainHub::MainHub(Database* db, FileIndexerConfig* config, bool firstRun) : m_db(db) , m_config(config) , m_fileWatcher(db, config, this) , m_fileIndexScheduler(db, config, firstRun, this) { Q_ASSERT(db); Q_ASSERT(config); connect(&m_fileWatcher, &FileWatch::indexNewFile, &m_fileIndexScheduler, &FileIndexScheduler::indexNewFile); connect(&m_fileWatcher, &FileWatch::indexModifiedFile, &m_fileIndexScheduler, &FileIndexScheduler::indexModifiedFile); connect(&m_fileWatcher, &FileWatch::indexXAttr, &m_fileIndexScheduler, &FileIndexScheduler::indexXAttrFile); connect(&m_fileWatcher, &FileWatch::fileRemoved, &m_fileIndexScheduler, &FileIndexScheduler::handleFileRemoved); connect(&m_fileWatcher, &FileWatch::installedWatches, &m_fileIndexScheduler, &FileIndexScheduler::scheduleIndexing); MainAdaptor* main = new MainAdaptor(this); Q_UNUSED(main) QDBusConnection bus = QDBusConnection::sessionBus(); bus.registerObject(QStringLiteral("/"), this, QDBusConnection::ExportAllSlots | QDBusConnection::ExportScriptableSignals | QDBusConnection::ExportAdaptors); if (firstRun) { QTimer::singleShot(5000, this, [this] { m_fileIndexScheduler.startupFinished(); }); } else { // Delay these checks so we don't end up consuming excessive resources on login QTimer::singleShot(5000, this, [this] { m_fileIndexScheduler.checkUnindexedFiles(); m_fileIndexScheduler.checkStaleIndexEntries(); m_fileIndexScheduler.startupFinished(); }); } QTimer::singleShot(0, &m_fileWatcher, &FileWatch::watchIndexedFolders); } void MainHub::quit() const { QCoreApplication::instance()->quit(); } void MainHub::updateConfig() { m_config->forceConfigUpdate(); m_fileWatcher.updateIndexedFoldersWatches(); m_fileIndexScheduler.updateConfig(); } -void MainHub::registerBalooWatcher(const QString &service) +void MainHub::registerBalooWatcher(const QString &) { - m_fileWatcher.registerBalooWatcher(service); } diff --git a/src/file/mainhub.h b/src/file/mainhub.h index 083ace83..65d03a4d 100644 --- a/src/file/mainhub.h +++ b/src/file/mainhub.h @@ -1,56 +1,57 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_MAINHUB_H #define BALOO_MAINHUB_H #include #include "filewatch.h" #include "fileindexscheduler.h" namespace Baloo { class Database; class FileIndexerConfig; class MainHub : public QObject { Q_OBJECT Q_CLASSINFO("D-Bus Interface", "org.kde.baloo.main") public: MainHub(Database* db, FileIndexerConfig* config, bool firstRun); public Q_SLOTS: Q_SCRIPTABLE void quit() const; Q_SCRIPTABLE void updateConfig(); + // TODO KF6 - remove Q_SCRIPTABLE void registerBalooWatcher(const QString &service); private: Database* m_db; FileIndexerConfig* m_config; FileWatch m_fileWatcher; FileIndexScheduler m_fileIndexScheduler; }; } #endif // BALOO_MAINHUB_H diff --git a/src/file/metadatamover.cpp b/src/file/metadatamover.cpp index 28330a08..1986f78c 100644 --- a/src/file/metadatamover.cpp +++ b/src/file/metadatamover.cpp @@ -1,197 +1,121 @@ /* This file is part of the KDE Project Copyright (c) 2009-2011 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "metadatamover.h" #include "database.h" #include "transaction.h" #include "basicindexingjob.h" #include "idutils.h" -#include "mainhub.h" #include "baloodebug.h" -#include "baloowatcherapplication_interface.h" - #include -#include using namespace Baloo; MetadataMover::MetadataMover(Database* db, QObject* parent) : QObject(parent) , m_db(db) { - m_serviceWatcher.setConnection(QDBusConnection::sessionBus()); - m_serviceWatcher.setWatchMode(QDBusServiceWatcher::WatchForUnregistration); - - connect(&m_serviceWatcher, &QDBusServiceWatcher::serviceUnregistered, - this, &MetadataMover::watcherServiceUnregistered); } MetadataMover::~MetadataMover() { } -bool MetadataMover::hasWatcher() const -{ - return !m_watcherApplications.isEmpty(); -} - - -static void buildRecursiveList(quint64 parentId, QList &fileList, Transaction &tr) -{ - fileList.push_back(QFile::decodeName(tr.documentUrl(parentId))); - - const auto childrenIds = tr.childrenDocumentId(parentId); - - for (const auto oneChildren : childrenIds) { - buildRecursiveList(oneChildren, fileList, tr); - } -} void MetadataMover::moveFileMetadata(const QString& from, const QString& to) { // qCDebug(BALOO) << from << to; Q_ASSERT(!from.isEmpty() && from != QLatin1String("/")); Q_ASSERT(!to.isEmpty() && to != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); - quint64 id = tr.documentId(QFile::encodeName(from)); - QList filesList; - qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << (hasWatcher() ? "has watcher" : "has no watcher"); - qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "id" << id; - if (id && hasWatcher()) { - buildRecursiveList(id, filesList, tr); - } - // We do NOT get deleted messages for overwritten files! Thus, we // have to remove all metadata for overwritten files first. removeMetadata(&tr, to); // and finally update the old statements updateMetadata(&tr, from, to); tr.commit(); - - if (hasWatcher()) { - qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "notifyWatchers" << filesList; - notifyWatchers(from, to, filesList); - } } void MetadataMover::removeFileMetadata(const QString& file) { Q_ASSERT(!file.isEmpty() && file != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); removeMetadata(&tr, file); tr.commit(); } -void MetadataMover::registerBalooWatcher(const QString &service) -{ - int firstSlash = service.indexOf('/'); - if (firstSlash == -1) { - return; - } - - QString dbusServiceName = service.left(firstSlash); - QString dbusPath = service.mid(firstSlash); - - m_serviceWatcher.addWatchedService(dbusServiceName); - - m_watcherApplications.insert(dbusServiceName, new org::kde::BalooWatcherApplication(dbusServiceName, dbusPath, QDBusConnection::sessionBus(), this)); - - qCDebug(BALOO) << "MetadataMover::registerBalooWatcher" << service << dbusServiceName << dbusPath; -} - void MetadataMover::removeMetadata(Transaction* tr, const QString& url) { Q_ASSERT(!url.isEmpty()); quint64 id = tr->documentId(QFile::encodeName(url)); if (!id) { Q_EMIT fileRemoved(url); return; } bool isDir = url.endsWith('/'); if (!isDir) { tr->removeDocument(id); } else { tr->removeRecursively(id); } Q_EMIT fileRemoved(url); } void MetadataMover::updateMetadata(Transaction* tr, const QString& from, const QString& to) { qCDebug(BALOO) << from << "->" << to; Q_ASSERT(!from.isEmpty() && !to.isEmpty()); Q_ASSERT(from[from.size()-1] != QLatin1Char('/')); Q_ASSERT(to[to.size()-1] != QLatin1Char('/')); QByteArray toPath = QFile::encodeName(to); quint64 id = filePathToId(toPath); if (!id) { qWarning() << "File moved to path which now no longer exists -" << to; return; } if (!tr->hasDocument(id)) { // // If we have no metadata yet we need to tell the file indexer so it can // create the metadata in case the target folder is configured to be indexed. // qCDebug(BALOO) << "Moved without data"; Q_EMIT movedWithoutData(to); return; } BasicIndexingJob job(toPath, QString(), BasicIndexingJob::NoLevel); job.index(); tr->replaceDocument(job.document(), DocumentUrl | FileNameTerms); // Possible scenarios // 1. file moves to the same device - id is preserved // 2. file moves to a different device - id is not preserved } - -void MetadataMover::notifyWatchers(const QString &from, const QString &to, const QList &filesList) -{ - for (org::kde::BalooWatcherApplication *watcherApplication : qAsConst(m_watcherApplications)) { - qCDebug(BALOO) << "MetadataMover::notifyWatchers" << watcherApplication->service() << watcherApplication->objectName() << watcherApplication->path(); - watcherApplication->renamedFiles(from, to, filesList); - } -} - -void MetadataMover::watcherServiceUnregistered(const QString &serviceName) -{ - auto itService = m_watcherApplications.find(serviceName); - if (itService == m_watcherApplications.end()) { - return; - } - - qCDebug(BALOO) << "MetadataMover::watcherServiceUnregistered" << itService.key(); - - delete itService.value(); - m_watcherApplications.erase(itService); -} diff --git a/src/file/metadatamover.h b/src/file/metadatamover.h index 7e0ec5a6..28ab852a 100644 --- a/src/file/metadatamover.h +++ b/src/file/metadatamover.h @@ -1,101 +1,71 @@ /* This file is part of the KDE Project Copyright (c) 2009-2011 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef BALOO_METADATA_MOVER_H_ #define BALOO_METADATA_MOVER_H_ #include -#include -#include - -class OrgKdeBalooWatcherApplicationInterface; - -namespace org -{ - -namespace kde -{ - -typedef ::OrgKdeBalooWatcherApplicationInterface BalooWatcherApplication; - -} - -} namespace Baloo { class Database; class Transaction; class MetadataMover : public QObject { Q_OBJECT public: MetadataMover(Database* db, QObject* parent = nullptr); ~MetadataMover(); - bool hasWatcher() const; - public Q_SLOTS: void moveFileMetadata(const QString& from, const QString& to); void removeFileMetadata(const QString& file); - void registerBalooWatcher(const QString &service); - Q_SIGNALS: /** * Emitted for files (and folders) that have been moved but * do not have metadata to be moved. This allows the file indexer * service to pick them up in case they are of interest. The * typical example would be moving a file from a non-indexed into * an indexed folder. */ void movedWithoutData(const QString& path); void fileRemoved(const QString& path); -private Q_SLOTS: - - void watcherServiceUnregistered(const QString &serviceName); - private: /** * Remove the metadata for file \p url */ void removeMetadata(Transaction* tr, const QString& url); /** * Recursively update the nie:url and nie:isPartOf properties * of the resource describing \p from. */ void updateMetadata(Transaction* tr, const QString& from, const QString& to); - void notifyWatchers(const QString &from, const QString &to, const QList &filesList); - - QMap m_watcherApplications; - - QDBusServiceWatcher m_serviceWatcher; - Database* m_db; }; } #endif