diff --git a/autotests/unit/file/CMakeLists.txt b/autotests/unit/file/CMakeLists.txt index 087f18fc..b5caedc0 100644 --- a/autotests/unit/file/CMakeLists.txt +++ b/autotests/unit/file/CMakeLists.txt @@ -1,34 +1,43 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") ecm_add_test(kinotifytest.cpp TEST_NAME "kinotifytest" LINK_LIBRARIES Qt5::Test baloofilecommon ) endif() +set (metadatamovertest_SOURCES + metadatamovertest.cpp +) + +qt5_add_dbus_adaptor(metadatamovertest_SOURCES + ${CMAKE_SOURCE_DIR}/src/file/org.kde.BalooWatcherApplication.xml + metadatamovertest.h MetadataMoverTestDBusSpy) + +ecm_add_test(${metadatamovertest_SOURCES} TEST_NAME metadatamovertest LINK_LIBRARIES Qt5::Test baloofilecommon KF5::Baloo) + MACRO(BALOO_FILE_AUTO_TESTS) FOREACH(_testname ${ARGN}) ecm_add_test(${_testname}.cpp TEST_NAME ${_testname} LINK_LIBRARIES Qt5::Test baloofilecommon KF5::Baloo) ENDFOREACH() ENDMACRO() baloo_file_auto_tests( pendingfilequeuetest fileindexerconfigtest basicindexingjobtest regularexpcachebenchmark filtereddiriteratortest unindexedfileiteratortest - metadatamovertest fileinfotest ) # # File Watch # set(fileWatch_SRC filewatchtest.cpp ../lib/xattrdetector.cpp) ecm_add_test(${fileWatch_SRC} TEST_NAME "filewatchtest" LINK_LIBRARIES Qt5::Test Qt5::DBus KF5::Baloo baloofilecommon ) diff --git a/autotests/unit/file/filewatchtest.cpp b/autotests/unit/file/filewatchtest.cpp index cf61842d..2ae37807 100644 --- a/autotests/unit/file/filewatchtest.cpp +++ b/autotests/unit/file/filewatchtest.cpp @@ -1,149 +1,181 @@ /* * * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filewatch.h" #include "fileindexerconfigutils.h" #include "database.h" #include "fileindexerconfig.h" #include "pendingfilequeue.h" +#include "mainhub.h" #include "../lib/xattrdetector.h" #include "../lib/baloo_xattr_p.h" #include #include #include namespace Baloo { class FileWatchTest : public QObject { Q_OBJECT private Q_SLOTS: + + void init() + { + m_tempDir = new QTemporaryDir(); + m_db = new Database(m_tempDir->path()); + m_db->open(Database::CreateDatabase); + m_dbusInterface = new MainHub(m_db, &m_config); + } + void testFileCreation(); + + void cleanupTestCase() + { + delete m_dbusInterface; + m_dbusInterface = nullptr; + + delete m_db; + m_db = nullptr; + + delete m_tempDir; + m_tempDir = nullptr; + } + +private: + + FileIndexerConfig m_config; + + Database* m_db; + + QTemporaryDir* m_tempDir; + + Baloo::MainHub *m_dbusInterface; }; } using namespace Baloo; namespace { bool createFile(const QString& fileUrl) { QFile f1(fileUrl); f1.open(QIODevice::WriteOnly); f1.close(); return QFile::exists(fileUrl); } void modifyFile(const QString& fileUrl) { QFile f1(fileUrl); f1.open(QIODevice::Append | QIODevice::Text); QTextStream stream(&f1); stream << "1"; } } void FileWatchTest::testFileCreation() { QTemporaryDir includeDir; QStringList includeFolders; includeFolders << includeDir.path(); QStringList excludeFolders; Test::writeIndexerConfig(includeFolders, excludeFolders); QTemporaryDir dbDir; Database db(dbDir.path()); db.open(Baloo::Database::CreateDatabase); FileIndexerConfig config; - FileWatch fileWatch(&db, &config); + FileWatch fileWatch(&db, &config, m_dbusInterface); fileWatch.m_pendingFileQueue->setMaximumTimeout(0); fileWatch.m_pendingFileQueue->setMinimumTimeout(0); fileWatch.m_pendingFileQueue->setTrackingTime(0); QSignalSpy spy(&fileWatch, SIGNAL(installedWatches())); QVERIFY(spy.isValid()); fileWatch.watchIndexedFolders(); QVERIFY(spy.count() || spy.wait()); QSignalSpy spyIndexNew(&fileWatch, SIGNAL(indexNewFile(QString))); QSignalSpy spyIndexModified(&fileWatch, SIGNAL(indexModifiedFile(QString))); QSignalSpy spyIndexXattr(&fileWatch, SIGNAL(indexXAttr(QString))); QVERIFY(spyIndexNew.isValid()); QVERIFY(spyIndexModified.isValid()); QVERIFY(spyIndexXattr.isValid()); // Create a file and see if it is indexed QString fileUrl(includeDir.path() + "/t1"); QVERIFY(createFile(fileUrl)); QVERIFY(spyIndexNew.wait()); QCOMPARE(spyIndexNew.count(), 1); QCOMPARE(spyIndexModified.count(), 0); QCOMPARE(spyIndexXattr.count(), 0); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); // // Modify the file // modifyFile(fileUrl); QVERIFY(spyIndexModified.wait()); QCOMPARE(spyIndexNew.count(), 0); QCOMPARE(spyIndexModified.count(), 1); QCOMPARE(spyIndexXattr.count(), 0); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); // // Set an Xattr // XattrDetector detector; if (!detector.isSupported(dbDir.path())) { qWarning() << "Xattr not supported on this filesystem"; return; } const QString userComment(QStringLiteral("UserComment")); QVERIFY(baloo_setxattr(fileUrl, QLatin1String("user.xdg.comment"), userComment) != -1); QVERIFY(spyIndexXattr.wait()); QCOMPARE(spyIndexNew.count(), 0); QCOMPARE(spyIndexModified.count(), 0); QCOMPARE(spyIndexXattr.count(), 1); spyIndexNew.clear(); spyIndexModified.clear(); spyIndexXattr.clear(); } QTEST_MAIN(FileWatchTest) #include "filewatchtest.moc" diff --git a/autotests/unit/file/metadatamovertest.cpp b/autotests/unit/file/metadatamovertest.cpp index 9a6ffc1b..dc4e92e9 100644 --- a/autotests/unit/file/metadatamovertest.cpp +++ b/autotests/unit/file/metadatamovertest.cpp @@ -1,214 +1,323 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013-2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ +#include "metadatamovertest.h" #include "metadatamover.h" +#include "baloowatcherapplicationadaptor.h" #include "database.h" #include "transaction.h" #include "document.h" #include "basicindexingjob.h" +#include "mainhub.h" #include #include #include #include #include #include +#include using namespace Baloo; class MetadataMoverTest : public QObject { Q_OBJECT public: MetadataMoverTest(QObject* parent = nullptr); private Q_SLOTS: void init(); void cleanupTestCase(); void testRemoveFile(); void testRenameFile(); void testMoveFile(); void testMoveFolder(); private: quint64 insertUrl(const QString& url); + FileIndexerConfig m_config; + Database* m_db; + QTemporaryDir* m_tempDir; }; MetadataMoverTest::MetadataMoverTest(QObject* parent) : QObject(parent) , m_db(nullptr) , m_tempDir(nullptr) { } void MetadataMoverTest::init() { m_tempDir = new QTemporaryDir(); m_db = new Database(m_tempDir->path()); m_db->open(Database::CreateDatabase); } void MetadataMoverTest::cleanupTestCase() { delete m_db; m_db = nullptr; delete m_tempDir; m_tempDir = nullptr; } quint64 MetadataMoverTest::insertUrl(const QString& url) { BasicIndexingJob job(url, QStringLiteral("text/plain")); job.index(); Transaction tr(m_db, Transaction::ReadWrite); tr.addDocument(job.document()); tr.commit(); return job.document().id(); } void MetadataMoverTest::testRemoveFile() { + MetadataMoverTestDBusSpy dbusSignalSpy; + QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); + QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); + QTemporaryFile file; file.open(); QString url = file.fileName(); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 0); + QCOMPARE(fileChangedSpy.count(), 0); + MetadataMover mover(m_db, this); + + mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); + file.remove(); mover.removeFileMetadata(url); + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 0); + QCOMPARE(fileChangedSpy.count(), 0); + { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(!tr.hasDocument(fid)); } } static void touchFile(const QString& path) { QFile file(path); file.open(QIODevice::WriteOnly); QTextStream s(&file); s << "random"; } static void mkdir(const QString& path) { QDir().mkpath(path); QVERIFY(QDir(path).exists()); } void MetadataMoverTest::testRenameFile() { + MetadataMoverTestDBusSpy dbusSignalSpy; + QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); + QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); + QTemporaryDir dir; QString url = dir.path() + "/file"; touchFile(url); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 0); + QCOMPARE(fileChangedSpy.count(), 0); + MetadataMover mover(m_db, this); + + mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); + QString url2 = dir.path() + "/file2"; QFile::rename(url, url2); mover.moveFileMetadata(QFile::encodeName(url), QFile::encodeName(url2)); + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 1); + QCOMPARE(fileChangedSpy.count(), 0); + { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(url2)); } } void MetadataMoverTest::testMoveFile() { + MetadataMoverTestDBusSpy dbusSignalSpy; + QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); + QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); + QTemporaryDir dir; QDir().mkpath(dir.path() + "/a/b/c"); QString url = dir.path() + "/a/b/c/file"; touchFile(url); quint64 fid = insertUrl(url); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); } + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 0); + QCOMPARE(fileChangedSpy.count(), 0); + MetadataMover mover(m_db, this); + + mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); + QString url2 = dir.path() + "/file2"; QFile::rename(url, url2); mover.moveFileMetadata(QFile::encodeName(url), QFile::encodeName(url2)); + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 1); + QCOMPARE(fileChangedSpy.count(), 0); + { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(url2)); } } void MetadataMoverTest::testMoveFolder() { + MetadataMoverTestDBusSpy dbusSignalSpy; + QSignalSpy renamedFilesSignalSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::renamedFilesSignal); + QSignalSpy fileChangedSpy(&dbusSignalSpy, &MetadataMoverTestDBusSpy::fileMetaDataChanged); + QTemporaryDir dir; QString folder = dir.path() + "/folder"; mkdir(folder); quint64 did = insertUrl(folder); QString fileUrl = folder + "/file"; touchFile(fileUrl); quint64 fid = insertUrl(fileUrl); { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(did)); QVERIFY(tr.hasDocument(fid)); } + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 0); + QCOMPARE(fileChangedSpy.count(), 0); + QString newFolderUrl = dir.path() + "/dir"; QFile::rename(folder, newFolderUrl); MetadataMover mover(m_db, this); + + mover.registerBalooWatcher(QStringLiteral("org.kde.baloo.metadatamovertest/org/kde/BalooWatcherApplication")); + mover.moveFileMetadata(QFile::encodeName(folder), QFile::encodeName(newFolderUrl)); + renamedFilesSignalSpy.wait(100); + fileChangedSpy.wait(100); + + QCOMPARE(renamedFilesSignalSpy.count(), 1); + QCOMPARE(fileChangedSpy.count(), 0); + { Transaction tr(m_db, Transaction::ReadOnly); QVERIFY(tr.hasDocument(did)); QVERIFY(tr.hasDocument(fid)); QCOMPARE(tr.documentUrl(did), QFile::encodeName(newFolderUrl)); QCOMPARE(tr.documentUrl(fid), QFile::encodeName(newFolderUrl + "/file")); } } + + +MetadataMoverTestDBusSpy::MetadataMoverTestDBusSpy(QObject *parent) : QObject(parent) +{ + QDBusConnection con = QDBusConnection::sessionBus(); + + con.connect(QString(), QStringLiteral("/files"), QStringLiteral("org.kde"), + QStringLiteral("changed"), this, SLOT(slotFileMetaDataChanged(QStringList))); + + + auto mDbusAdaptor = new BalooWatcherApplicationAdaptor(this); + + QCOMPARE(con.registerService(QStringLiteral("org.kde.baloo.metadatamovertest")), true); + QCOMPARE(con.registerObject(QStringLiteral("/org/kde/BalooWatcherApplication"), mDbusAdaptor, QDBusConnection::ExportAllContents), true); +} + +void MetadataMoverTestDBusSpy::slotFileMetaDataChanged(QStringList fileList) +{ + Q_EMIT fileMetaDataChanged(fileList); +} + +void MetadataMoverTestDBusSpy::renamedFiles(const QString &from, const QString &to, const QStringList &listFiles) +{ + Q_EMIT renamedFilesSignal(from, to, listFiles); +} + QTEST_GUILESS_MAIN(MetadataMoverTest) #include "metadatamovertest.moc" diff --git a/autotests/unit/file/metadatamovertest.h b/autotests/unit/file/metadatamovertest.h new file mode 100644 index 00000000..a669e531 --- /dev/null +++ b/autotests/unit/file/metadatamovertest.h @@ -0,0 +1,29 @@ +#if !defined METADATAMOVERTEST_H_ +#define METADATAMOVERTEST_H_ + +#include +#include +#include + +class MetadataMoverTestDBusSpy : public QObject +{ + Q_OBJECT +public: + + MetadataMoverTestDBusSpy(QObject* parent = nullptr); + +Q_SIGNALS: + + void fileMetaDataChanged(QStringList fileList); + + void renamedFilesSignal(const QString &from, const QString &to, const QStringList &listFiles); + +public Q_SLOTS: + + void slotFileMetaDataChanged(QStringList fileList); + + void renamedFiles(const QString &from, const QString &to, const QStringList &listFiles); + +}; + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fcb1daa8..0efb6698 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,12 +1,13 @@ add_subdirectory(lib) add_subdirectory(engine) add_subdirectory(codecs) add_subdirectory(qml) if (BUILD_KINOTIFY) add_subdirectory(file) add_subdirectory(kioslaves) add_subdirectory(tools) endif() add_subdirectory(dbus) + diff --git a/src/engine/transaction.cpp b/src/engine/transaction.cpp index df1e0ac9..aa6773a0 100644 --- a/src/engine/transaction.cpp +++ b/src/engine/transaction.cpp @@ -1,586 +1,593 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "transaction.h" #include "postingdb.h" #include "documentdb.h" #include "documenturldb.h" #include "documentiddb.h" #include "positiondb.h" #include "documentdatadb.h" #include "mtimedb.h" #include "document.h" #include "enginequery.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "phraseanditerator.h" #include "writetransaction.h" #include "idutils.h" #include "database.h" #include "databasesize.h" #include #include using namespace Baloo; Transaction::Transaction(const Database& db, Transaction::TransactionType type) : m_dbis(db.m_dbis) , m_env(db.m_env) , m_writeTrans(nullptr) { uint flags = type == ReadOnly ? MDB_RDONLY : 0; int rc = mdb_txn_begin(db.m_env, nullptr, flags, &m_txn); Q_ASSERT_X(rc == 0, "Transaction", mdb_strerror(rc)); if (type == ReadWrite) { m_writeTrans = new WriteTransaction(m_dbis, m_txn); } } Transaction::Transaction(Database* db, Transaction::TransactionType type) : Transaction(*db, type) { } Transaction::~Transaction() { if (m_writeTrans) qWarning() << "Closing an active WriteTransaction without calling abort/commit"; if (m_txn) { abort(); } } bool Transaction::hasDocument(quint64 id) const { Q_ASSERT(id > 0); IdFilenameDB idFilenameDb(m_dbis.idFilenameDbi, m_txn); return idFilenameDb.contains(id); } bool Transaction::inPhaseOne(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.contains(id); } bool Transaction::hasFailed(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); return failedIdDb.contains(id); } QByteArray Transaction::documentUrl(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.get(id); } quint64 Transaction::documentId(const QByteArray& path) const { Q_ASSERT(m_txn); Q_ASSERT(!path.isEmpty()); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); QList li = path.split('/'); quint64 parentId = 0; for (const QByteArray& fileName : li) { if (fileName.isEmpty()) { continue; } parentId = docUrlDb.getId(parentId, fileName); if (!parentId) { return 0; } } return parentId; } +QVector Transaction::childrenDocumentId(quint64 parentId) const +{ + DocumentUrlDB docUrlDB(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); + + return docUrlDB.getChildren(parentId); +} + DocumentTimeDB::TimeInfo Transaction::documentTimeInfo(quint64 id) const { Q_ASSERT(m_txn); DocumentTimeDB docTimeDb(m_dbis.docTimeDbi, m_txn); return docTimeDb.get(id); } QByteArray Transaction::documentData(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentDataDB docDataDb(m_dbis.docDataDbi, m_txn); return docDataDb.get(id); } bool Transaction::hasChanges() const { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); return m_writeTrans->hasChanges(); } QVector Transaction::fetchPhaseOneIds(int size) const { Q_ASSERT(m_txn); Q_ASSERT(size > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.fetchItems(size); } QVector Transaction::fetchTermsStartingWith(const QByteArray& term) const { Q_ASSERT(term.size() > 0); PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.fetchTermsStartingWith(term); } uint Transaction::phaseOneSize() const { Q_ASSERT(m_txn); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.size(); } uint Transaction::size() const { Q_ASSERT(m_txn); DocumentDB docTermsDb(m_dbis.docTermsDbi, m_txn); return docTermsDb.size(); } // // Write Operations // void Transaction::setPhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.put(id); } void Transaction::removePhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.del(id); } void Transaction::addFailed(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); failedIdDb.put(id); } void Transaction::addDocument(const Document& doc) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); m_writeTrans->addDocument(doc); } void Transaction::removeDocument(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); m_writeTrans->removeDocument(id); } void Transaction::removeRecursively(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); m_writeTrans->removeRecursively(id); } void Transaction::replaceDocument(const Document& doc, DocumentOperations operations) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); Q_ASSERT_X(hasDocument(doc.id()), "Transaction::replaceDocument", "Document does not exist"); m_writeTrans->replaceDocument(doc, operations); } void Transaction::commit() { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); m_writeTrans->commit(); delete m_writeTrans; m_writeTrans = nullptr; int rc = mdb_txn_commit(m_txn); Q_ASSERT_X(rc == 0, "Transaction::commit", mdb_strerror(rc)); m_txn = nullptr; } void Transaction::abort() { Q_ASSERT(m_txn); mdb_txn_abort(m_txn); m_txn = nullptr; delete m_writeTrans; m_writeTrans = nullptr; } // // Queries // PostingIterator* Transaction::postingIterator(const EngineQuery& query) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); PositionDB positionDb(m_dbis.positionDBi, m_txn); if (query.leaf()) { if (query.op() == EngineQuery::Equal) { return postingDb.iter(query.term()); } else if (query.op() == EngineQuery::StartsWith) { return postingDb.prefixIter(query.term()); } else { Q_ASSERT(0); } } if (query.subQueries().isEmpty()) { return nullptr; } QVector vec; vec.reserve(query.subQueries().size()); if (query.op() == EngineQuery::Phrase) { for (const EngineQuery& q : query.subQueries()) { Q_ASSERT_X(q.leaf(), "Transaction::toPostingIterator", "Phrase queries must contain leaf queries"); vec << positionDb.iter(q.term()); } return new PhraseAndIterator(vec); } for (const EngineQuery& q : query.subQueries()) { vec << postingIterator(q); } if (query.op() == EngineQuery::And) { return new AndPostingIterator(vec); } else if (query.op() == EngineQuery::Or) { return new OrPostingIterator(vec); } Q_ASSERT(0); return nullptr; } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, const QByteArray& value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::mTimeIter(quint32 mtime, MTimeDB::Comparator com) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iter(mtime, com); } PostingIterator* Transaction::mTimeRangeIter(quint32 beginTime, quint32 endTime) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iterRange(beginTime, endTime); } PostingIterator* Transaction::docUrlIter(quint64 id) const { DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.iter(id); } QVector Transaction::exec(const EngineQuery& query, int limit) const { Q_ASSERT(m_txn); QVector results; PostingIterator* it = postingIterator(query); if (!it) { return results; } while (it->next() && limit) { results << it->docId(); limit--; } return results; } // // Introspection // QVector Transaction::documentTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); return documentTermsDB.get(docId); } QVector Transaction::documentFileNameTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); return documentFileNameTermsDB.get(docId); } QVector Transaction::documentXattrTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); return documentXattrTermsDB.get(docId); } // // File Size // static size_t dbiSize(MDB_txn* txn, MDB_dbi dbi) { MDB_stat stat; mdb_stat(txn, dbi, &stat); return (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages) * stat.ms_psize; } DatabaseSize Transaction::dbSize() { DatabaseSize dbSize; dbSize.postingDb = dbiSize(m_txn, m_dbis.postingDbi); dbSize.positionDb = dbiSize(m_txn, m_dbis.positionDBi); dbSize.docTerms = dbiSize(m_txn, m_dbis.docTermsDbi); dbSize.docFilenameTerms = dbiSize(m_txn, m_dbis.docFilenameTermsDbi); dbSize.docXattrTerms = dbiSize(m_txn, m_dbis.docXattrTermsDbi); dbSize.idTree = dbiSize(m_txn, m_dbis.idTreeDbi); dbSize.idFilename = dbiSize(m_txn, m_dbis.idFilenameDbi); dbSize.docTime = dbiSize(m_txn, m_dbis.docTimeDbi); dbSize.docData = dbiSize(m_txn, m_dbis.docDataDbi); dbSize.contentIndexingIds = dbiSize(m_txn, m_dbis.contentIndexingDbi); dbSize.failedIds = dbiSize(m_txn, m_dbis.failedIdDbi); dbSize.mtimeDb = dbiSize(m_txn, m_dbis.mtimeDbi); dbSize.expectedSize = dbSize.positionDb + dbSize.positionDb + dbSize.docTerms + dbSize.docFilenameTerms + dbSize.docXattrTerms + dbSize.idTree + dbSize.idFilename + dbSize.docTime + dbSize.docData + dbSize.contentIndexingIds + dbSize.failedIds + dbSize.mtimeDb; MDB_envinfo info; mdb_env_info(m_env, &info); dbSize.actualSize = info.me_last_pgno * 4096; // TODO: separate page size return dbSize; } // // Debugging // void Transaction::checkFsTree() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "Total Document IDs: " << allIds.size() << endl; int count = 0; for (quint64 id: allIds) { QByteArray url = docUrlDb.get(id); if (url.isEmpty()) { auto terms = documentTermsDB.get(id); auto fileNameTerms = documentFileNameTermsDB.get(id); auto xAttrTerms = documentXattrTermsDB.get(id); // Lets reverse enginer the terms QList newTerms; QMapIterator it(map); while (it.hasNext()) { it.next(); if (it.value().contains(id)) { newTerms << it.key(); } } out << "Missing filePath for " << id << endl; out << "\tPostingDB Terms: "; for (const QByteArray& term : newTerms) { out << term << " "; } out << endl; out << "\tDocumentTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; out << "\tFileNameTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; out << "\tXAttrTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; count++; } else if (!QFileInfo::exists(url)) { out << "FilePath " << url << " for " << id << " does not exist"<< endl; count++; } } out << "Invalid Entries: " << count << " (" << count * 100.0 / allIds.size() << "%)" << endl; } void Transaction::checkTermsDbinPostingDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); // Iterate over each document, and fetch all terms // check if each term maps to its own id in the posting db auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "PostingDB check .." << endl; for (quint64 id : allIds) { QVector terms = documentTermsDB.get(id); terms += documentXattrTermsDB.get(id); terms += documentFileNameTermsDB.get(id); for (const QByteArray& term : terms) { PostingList plist = postingDb.get(term); if (!plist.contains(id)) { out << id << " is missing term " << term << endl; } } } } void Transaction::checkPostingDbinTermsDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); QMap map = postingDb.toTestMap(); QMapIterator it(map); QTextStream out(stdout); out << "DocumentTermsDB check .." << endl; while (it.hasNext()) { it.next(); const QByteArray& term = it.key(); const PostingList& list = it.value(); for (quint64 id : list) { if (documentTermsDB.get(id).contains(term)) { continue; } if (documentFileNameTermsDB.get(id).contains(term)) { continue; } if (documentXattrTermsDB.get(id).contains(term)) { continue; } out << id << " is missing " << QString::fromUtf8(term) << " from document terms db" << endl; } } } diff --git a/src/engine/transaction.h b/src/engine/transaction.h index efd9f3d6..8d9b9434 100644 --- a/src/engine/transaction.h +++ b/src/engine/transaction.h @@ -1,138 +1,139 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_TRANSACTION_H #define BALOO_TRANSACTION_H #include "databasedbis.h" #include "mtimedb.h" #include "postingdb.h" #include "writetransaction.h" #include "documenttimedb.h" #include #include namespace Baloo { class Database; class Document; class PostingIterator; class EngineQuery; class DatabaseSize; class DBState; class BALOO_ENGINE_EXPORT Transaction { public: enum TransactionType { ReadOnly, ReadWrite }; Transaction(const Database& db, TransactionType type); Transaction(Database* db, TransactionType type); ~Transaction(); // // Getters // bool hasDocument(quint64 id) const; bool inPhaseOne(quint64 id) const; bool hasFailed(quint64 id) const; QByteArray documentUrl(quint64 id) const; /** * This method is not cheap, and does not stat the filesystem in order to convert the path * \p path into an id. */ quint64 documentId(const QByteArray& path) const; + QVector childrenDocumentId(quint64 parentId) const; QByteArray documentData(quint64 id) const; DocumentTimeDB::TimeInfo documentTimeInfo(quint64 id) const; QVector exec(const EngineQuery& query, int limit = -1) const; PostingIterator* postingIterator(const EngineQuery& query) const; PostingIterator* postingCompIterator(const QByteArray& prefix, const QByteArray& value, PostingDB::Comparator com) const; PostingIterator* mTimeIter(quint32 mtime, MTimeDB::Comparator com) const; PostingIterator* mTimeRangeIter(quint32 beginTime, quint32 endTime) const; PostingIterator* docUrlIter(quint64 id) const; QVector fetchPhaseOneIds(int size) const; uint phaseOneSize() const; uint size() const; QVector fetchTermsStartingWith(const QByteArray& term) const; // // Introspecing document data // QVector documentTerms(quint64 docId) const; QVector documentFileNameTerms(quint64 docId) const; QVector documentXattrTerms(quint64 docId) const; DatabaseSize dbSize(); // // Transaction handling // void commit(); void abort(); bool hasChanges() const; // // Write Methods // void addDocument(const Document& doc); void removeDocument(quint64 id); void removeRecursively(quint64 parentId); void addFailed(quint64 id); template void removeRecursively(quint64 id, Functor shouldDelete) { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); m_writeTrans->removeRecursively(id, shouldDelete); } void replaceDocument(const Document& doc, DocumentOperations operations); void setPhaseOne(quint64 id); void removePhaseOne(quint64 id); // Debugging void checkFsTree(); void checkTermsDbinPostingDb(); void checkPostingDbinTermsDb(); private: Transaction(const Transaction& rhs) = delete; const DatabaseDbis& m_dbis; MDB_txn* m_txn; MDB_env* m_env; WriteTransaction* m_writeTrans; friend class DatabaseSanitizerImpl; friend class DBState; // for testing }; } #endif // BALOO_TRANSACTION_H diff --git a/src/file/CMakeLists.txt b/src/file/CMakeLists.txt index ef80b784..6cdfa98a 100644 --- a/src/file/CMakeLists.txt +++ b/src/file/CMakeLists.txt @@ -1,74 +1,81 @@ include_directories(${Inotify_INCLUDE_DIRS}) add_definitions(-DTRANSLATION_DOMAIN=\"baloo_file5\") set(file_static_lib_SRCS # File Indexer mainhub.cpp mainadaptor.cpp fileindexerconfig.cpp basicindexingjob.cpp powerstatemonitor.cpp fileindexscheduler.cpp firstrunindexer.cpp newfileindexer.cpp xattrindexer.cpp modifiedfileindexer.cpp unindexedfileindexer.cpp filecontentindexer.cpp filecontentindexerprovider.cpp extractorprocess.cpp timeestimator.cpp indexcleaner.cpp # Common priority.cpp regexpcache.cpp fileexcludefilters.cpp storagedevices.cpp filtereddiriterator.cpp unindexedfileiterator.cpp migrator.cpp fileinfo.cpp # File Watcher filewatch.cpp pendingfilequeue.cpp metadatamover.cpp + org.kde.BalooWatcherApplication.xml pendingfile.cpp kinotify.cpp ) ecm_qt_declare_logging_category(file_static_lib_SRCS HEADER baloodebug.h IDENTIFIER BALOO CATEGORY_NAME org.kde.baloo) +qt5_add_dbus_interface(file_static_lib_SRCS org.kde.BalooWatcherApplication.xml baloowatcherapplication_interface) + add_library(baloofilecommon STATIC ${file_static_lib_SRCS}) target_link_libraries(baloofilecommon Qt5::DBus KF5::I18n KF5::Solid KF5::FileMetaData KF5::Crash KF5::ConfigCore KF5::BalooEngine ${Inotify_LIBRARIES} ) set(file_SRCS main.cpp ) add_executable(baloo_file ${file_SRCS}) ecm_mark_nongui_executable(baloo_file) target_compile_definitions(baloo_file PRIVATE -DPROJECT_VERSION="${PROJECT_VERSION}") target_link_libraries(baloo_file baloofilecommon ) install(TARGETS baloo_file ${INSTALL_TARGETS_DEFAULT_ARGS}) install(FILES baloo_file.desktop DESTINATION ${AUTOSTART_INSTALL_DIR}) add_subdirectory(extractor) + +install(FILES + org.kde.BalooWatcherApplication.xml + DESTINATION ${KDE_INSTALL_DBUSINTERFACEDIR}) diff --git a/src/file/filewatch.cpp b/src/file/filewatch.cpp index a857240a..99374657 100644 --- a/src/file/filewatch.cpp +++ b/src/file/filewatch.cpp @@ -1,196 +1,205 @@ /* This file is part of the KDE Project Copyright (c) 2007-2011 Sebastian Trueg Copyright (c) 2012-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "filewatch.h" #include "metadatamover.h" #include "fileindexerconfig.h" #include "pendingfilequeue.h" #include "regexpcache.h" #include "database.h" #include "pendingfile.h" #include "baloodebug.h" #include "kinotify.h" #include #include #include #include using namespace Baloo; -FileWatch::FileWatch(Database* db, FileIndexerConfig* config, QObject* parent) +FileWatch::FileWatch(Database* db, FileIndexerConfig* config, Baloo::MainHub *dbusInterface, QObject* parent) : QObject(parent) , m_db(db) , m_config(config) , m_dirWatch(nullptr) { Q_ASSERT(db); Q_ASSERT(config); m_metadataMover = new MetadataMover(m_db, this); connect(m_metadataMover, &MetadataMover::movedWithoutData, this, &FileWatch::indexNewFile); connect(m_metadataMover, &MetadataMover::fileRemoved, this, &FileWatch::fileRemoved); m_pendingFileQueue = new PendingFileQueue(this); connect(m_pendingFileQueue, &PendingFileQueue::indexNewFile, this, &FileWatch::indexNewFile); connect(m_pendingFileQueue, &PendingFileQueue::indexModifiedFile, this, &FileWatch::indexModifiedFile); connect(m_pendingFileQueue, &PendingFileQueue::indexXAttr, this, &FileWatch::indexXAttr); connect(m_pendingFileQueue, &PendingFileQueue::removeFileIndex, m_metadataMover, &MetadataMover::removeFileMetadata); // monitor the file system for changes (restricted by the inotify limit) m_dirWatch = new KInotify(m_config, this); connect(m_dirWatch, &KInotify::moved, this, &FileWatch::slotFileMoved); connect(m_dirWatch, &KInotify::deleted, this, &FileWatch::slotFileDeleted); connect(m_dirWatch, &KInotify::created, this, &FileWatch::slotFileCreated); connect(m_dirWatch, &KInotify::modified, this, &FileWatch::slotFileModified); connect(m_dirWatch, &KInotify::closedWrite, this, &FileWatch::slotFileClosedAfterWrite); connect(m_dirWatch, &KInotify::attributeChanged, this, &FileWatch::slotAttributeChanged); connect(m_dirWatch, &KInotify::watchUserLimitReached, this, &FileWatch::slotInotifyWatchUserLimitReached); connect(m_dirWatch, &KInotify::installedWatches, this, &FileWatch::installedWatches); } FileWatch::~FileWatch() { } void FileWatch::watchIndexedFolders() { // Watch all indexed folders QStringList folders = m_config->includeFolders(); Q_FOREACH (const QString& folder, folders) { watchFolder(folder); } } +void FileWatch::registerBalooWatcher(const QString &service) +{ + if (!m_metadataMover) { + return; + } + + m_metadataMover->registerBalooWatcher(service); +} + // FIXME: listen to Create for folders! void FileWatch::watchFolder(const QString& path) { qCDebug(BALOO) << path; if (m_dirWatch && !m_dirWatch->watchingPath(path)) { KInotify::WatchEvents flags(KInotify::EventMove | KInotify::EventDelete | KInotify::EventDeleteSelf | KInotify::EventCloseWrite | KInotify::EventCreate | KInotify::EventAttributeChange | KInotify::EventModify); m_dirWatch->addWatch(path, flags, KInotify::WatchFlags()); } } void FileWatch::slotFileMoved(const QString& urlFrom, const QString& urlTo) { if (m_config->shouldBeIndexed(urlTo)) { m_metadataMover->moveFileMetadata(urlFrom, urlTo); } else { m_metadataMover->removeFileMetadata(urlFrom); } } void FileWatch::slotFileDeleted(const QString& urlString, bool isDir) { // Directories must always end with a trailing slash '/' QString url = urlString; if (isDir) { Q_ASSERT(!url.endsWith('/')); url.append(QLatin1Char('/')); } PendingFile file(url); file.setDeleted(); m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileCreated(const QString& path, bool isDir) { Q_UNUSED(isDir); PendingFile file(path); file.setCreated(); m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileModified(const QString& path) { PendingFile file(path); file.setModified(); //qCDebug(BALOO) << "MOD" << path; m_pendingFileQueue->enqueue(file); } void FileWatch::slotFileClosedAfterWrite(const QString& path) { QDateTime current = QDateTime::currentDateTime(); QDateTime fileModification = QFileInfo(path).lastModified(); QDateTime dirModification = QFileInfo(QFileInfo(path).absoluteDir().absolutePath()).lastModified(); // If we have received a FileClosedAfterWrite event, then the file must have been // closed within the last minute. // This is being done cause many applications open the file under write mode, do not // make any modifications and then close the file. This results in us getting // the FileClosedAfterWrite event if (fileModification.secsTo(current) <= 1000 * 60 || dirModification.secsTo(current) <= 1000 * 60) { PendingFile file(path); file.setClosedOnWrite(); //qCDebug(BALOO) << "CLOSE" << path; m_pendingFileQueue->enqueue(file); } } void FileWatch::slotAttributeChanged(const QString& path) { PendingFile file(path); file.setAttributeChanged(); m_pendingFileQueue->enqueue(file); } // This slot is connected to a signal emitted in KInotify when // inotify_add_watch fails with ENOSPC. void FileWatch::slotInotifyWatchUserLimitReached(const QString&) { //If we got here, we hit the limit and couldn't authenticate to raise it, // so put something in the syslog so someone notices. syslog(LOG_USER | LOG_WARNING, "KDE Baloo File Indexer has reached the inotify folder watch limit. File changes will be ignored."); // we do it the brutal way for now hoping with new kernels and defaults this will never happen // Delete the KInotify // FIXME: Maybe we should be aborting? if (m_dirWatch) { m_dirWatch->deleteLater(); m_dirWatch = nullptr; } Q_EMIT installedWatches(); } void FileWatch::updateIndexedFoldersWatches() { if (m_dirWatch) { QStringList folders = m_config->includeFolders(); Q_FOREACH (const QString& folder, folders) { m_dirWatch->removeWatch(folder); watchFolder(folder); } } } diff --git a/src/file/filewatch.h b/src/file/filewatch.h index 568e892c..e1287223 100644 --- a/src/file/filewatch.h +++ b/src/file/filewatch.h @@ -1,93 +1,96 @@ /* This file is part of the KDE Project Copyright (c) 2007-2011 Sebastian Trueg This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef BALOO_FILE_WATCH_H_ #define BALOO_FILE_WATCH_H_ #include #include "pendingfile.h" class KInotify; namespace Baloo { class Database; class MetadataMover; class FileIndexerConfig; class PendingFileQueue; class FileWatchTest; +class MainHub; class FileWatch : public QObject { Q_OBJECT public: - FileWatch(Database* db, FileIndexerConfig* config, QObject* parent = nullptr); + FileWatch(Database* db, FileIndexerConfig* config, Baloo::MainHub *dbusInterface, QObject* parent = nullptr); ~FileWatch(); public Q_SLOTS: /** * To be called whenever the list of indexed folders changes. This is done because * the indexed folders are watched with the 'KInotify::EventCreate' event, and the * non-indexed folders are not. */ void updateIndexedFoldersWatches(); void watchIndexedFolders(); + void registerBalooWatcher(const QString &service); + Q_SIGNALS: void indexNewFile(const QString& string); void indexModifiedFile(const QString& string); void indexXAttr(const QString& path); /** * This signal is emitted when a file has been removed, and everyone else * should update their caches */ void fileRemoved(const QString& path); void installedWatches(); private Q_SLOTS: void slotFileMoved(const QString& from, const QString& to); void slotFileDeleted(const QString& urlString, bool isDir); void slotFileCreated(const QString& path, bool isDir); void slotFileClosedAfterWrite(const QString&); void slotAttributeChanged(const QString& path); void slotFileModified(const QString& path); void slotInotifyWatchUserLimitReached(const QString&); private: /** Watch a folder, provided it is not already watched*/ void watchFolder(const QString& path); Database* m_db; MetadataMover* m_metadataMover; FileIndexerConfig* m_config; KInotify* m_dirWatch; /// queue used to "compress" multiple file events like downloads PendingFileQueue* m_pendingFileQueue; friend class FileWatchTest; }; } #endif diff --git a/src/file/mainhub.cpp b/src/file/mainhub.cpp index 8e5a7c02..3c9f3886 100644 --- a/src/file/mainhub.cpp +++ b/src/file/mainhub.cpp @@ -1,67 +1,73 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "mainhub.h" #include "fileindexerconfig.h" #include "mainadaptor.h" #include #include #include +#include using namespace Baloo; MainHub::MainHub(Database* db, FileIndexerConfig* config) : m_db(db) , m_config(config) , m_fileWatcher(db, config, this) , m_fileIndexScheduler(db, config, this) { Q_ASSERT(db); Q_ASSERT(config); connect(&m_fileWatcher, &FileWatch::indexNewFile, &m_fileIndexScheduler, &FileIndexScheduler::indexNewFile); connect(&m_fileWatcher, &FileWatch::indexModifiedFile, &m_fileIndexScheduler, &FileIndexScheduler::indexModifiedFile); connect(&m_fileWatcher, &FileWatch::indexXAttr, &m_fileIndexScheduler, &FileIndexScheduler::indexXAttrFile); connect(&m_fileWatcher, &FileWatch::fileRemoved, &m_fileIndexScheduler, &FileIndexScheduler::handleFileRemoved); connect(&m_fileWatcher, &FileWatch::installedWatches, &m_fileIndexScheduler, &FileIndexScheduler::scheduleIndexing); MainAdaptor* main = new MainAdaptor(this); Q_UNUSED(main) QDBusConnection bus = QDBusConnection::sessionBus(); bus.registerObject(QStringLiteral("/"), this, QDBusConnection::ExportAllSlots | QDBusConnection::ExportScriptableSignals | QDBusConnection::ExportAdaptors); QTimer::singleShot(0, &m_fileWatcher, &FileWatch::watchIndexedFolders); } void MainHub::quit() const { QCoreApplication::instance()->quit(); } void MainHub::updateConfig() { m_config->forceConfigUpdate(); // FIXME!! //m_fileIndexer.updateConfig(); m_fileWatcher.updateIndexedFoldersWatches(); } + +void MainHub::registerBalooWatcher(const QString &service) +{ + m_fileWatcher.registerBalooWatcher(service); +} diff --git a/src/file/mainhub.h b/src/file/mainhub.h index 88022658..abeacfc0 100644 --- a/src/file/mainhub.h +++ b/src/file/mainhub.h @@ -1,53 +1,57 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_MAINHUB_H #define BALOO_MAINHUB_H #include #include "filewatch.h" #include "fileindexscheduler.h" +#include + namespace Baloo { class Database; class FileIndexerConfig; class MainHub : public QObject { Q_OBJECT Q_CLASSINFO("D-Bus Interface", "org.kde.baloo.main") public: MainHub(Database* db, FileIndexerConfig* config); public Q_SLOTS: Q_SCRIPTABLE void quit() const; Q_SCRIPTABLE void updateConfig(); + Q_SCRIPTABLE void registerBalooWatcher(const QString &service); private: Database* m_db; FileIndexerConfig* m_config; FileWatch m_fileWatcher; FileIndexScheduler m_fileIndexScheduler; + }; } #endif // BALOO_MAINHUB_H diff --git a/src/file/metadatamover.cpp b/src/file/metadatamover.cpp index 1986f78c..c5d2a891 100644 --- a/src/file/metadatamover.cpp +++ b/src/file/metadatamover.cpp @@ -1,121 +1,198 @@ /* This file is part of the KDE Project Copyright (c) 2009-2011 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "metadatamover.h" #include "database.h" #include "transaction.h" #include "basicindexingjob.h" #include "idutils.h" +#include "mainhub.h" #include "baloodebug.h" +#include "baloowatcherapplication_interface.h" + #include +#include +#include using namespace Baloo; MetadataMover::MetadataMover(Database* db, QObject* parent) : QObject(parent) , m_db(db) { + m_serviceWatcher.setConnection(QDBusConnection::sessionBus()); + m_serviceWatcher.setWatchMode(QDBusServiceWatcher::WatchForUnregistration); + + connect(&m_serviceWatcher, &QDBusServiceWatcher::serviceUnregistered, + this, &MetadataMover::watcherServiceUnregistered); } MetadataMover::~MetadataMover() { } +bool MetadataMover::hasWatcher() const +{ + return !m_watcherApplications.isEmpty(); +} + + +static void buildRecursiveList(quint64 parentId, QList &fileList, Transaction &tr) +{ + fileList.push_back(QFile::decodeName(tr.documentUrl(parentId))); + + const auto childrenIds = tr.childrenDocumentId(parentId); + + for (const auto oneChildren : childrenIds) { + buildRecursiveList(oneChildren, fileList, tr); + } +} void MetadataMover::moveFileMetadata(const QString& from, const QString& to) { // qCDebug(BALOO) << from << to; Q_ASSERT(!from.isEmpty() && from != QLatin1String("/")); Q_ASSERT(!to.isEmpty() && to != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); + quint64 id = tr.documentId(QFile::encodeName(from)); + QList filesList; + qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << (hasWatcher() ? "has watcher" : "has no watcher"); + qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "id" << id; + if (id && hasWatcher()) { + buildRecursiveList(id, filesList, tr); + } + // We do NOT get deleted messages for overwritten files! Thus, we // have to remove all metadata for overwritten files first. removeMetadata(&tr, to); // and finally update the old statements updateMetadata(&tr, from, to); tr.commit(); + + if (hasWatcher()) { + qCDebug(BALOO) << "MetadataMover::moveFileMetadata" << "notifyWatchers" << filesList; + notifyWatchers(from, to, filesList); + } } void MetadataMover::removeFileMetadata(const QString& file) { Q_ASSERT(!file.isEmpty() && file != QLatin1String("/")); Transaction tr(m_db, Transaction::ReadWrite); removeMetadata(&tr, file); tr.commit(); } +void MetadataMover::registerBalooWatcher(const QString &service) +{ + int firstSlash = service.indexOf('/'); + if (firstSlash == -1) { + return; + } + + QString dbusServiceName = service.left(firstSlash); + QString dbusPath = service.mid(firstSlash); + + m_serviceWatcher.addWatchedService(dbusServiceName); + + m_watcherApplications.insert(dbusServiceName, new org::kde::BalooWatcherApplication(dbusServiceName, dbusPath, QDBusConnection::sessionBus(), this)); + + qCDebug(BALOO) << "MetadataMover::registerBalooWatcher" << service << dbusServiceName << dbusPath; +} + void MetadataMover::removeMetadata(Transaction* tr, const QString& url) { Q_ASSERT(!url.isEmpty()); quint64 id = tr->documentId(QFile::encodeName(url)); if (!id) { Q_EMIT fileRemoved(url); return; } bool isDir = url.endsWith('/'); if (!isDir) { tr->removeDocument(id); } else { tr->removeRecursively(id); } Q_EMIT fileRemoved(url); } void MetadataMover::updateMetadata(Transaction* tr, const QString& from, const QString& to) { qCDebug(BALOO) << from << "->" << to; Q_ASSERT(!from.isEmpty() && !to.isEmpty()); Q_ASSERT(from[from.size()-1] != QLatin1Char('/')); Q_ASSERT(to[to.size()-1] != QLatin1Char('/')); QByteArray toPath = QFile::encodeName(to); quint64 id = filePathToId(toPath); if (!id) { qWarning() << "File moved to path which now no longer exists -" << to; return; } if (!tr->hasDocument(id)) { // // If we have no metadata yet we need to tell the file indexer so it can // create the metadata in case the target folder is configured to be indexed. // qCDebug(BALOO) << "Moved without data"; Q_EMIT movedWithoutData(to); return; } BasicIndexingJob job(toPath, QString(), BasicIndexingJob::NoLevel); job.index(); tr->replaceDocument(job.document(), DocumentUrl | FileNameTerms); // Possible scenarios // 1. file moves to the same device - id is preserved // 2. file moves to a different device - id is not preserved } + +void MetadataMover::notifyWatchers(const QString &from, const QString &to, const QList &filesList) +{ + Q_FOREACH(org::kde::BalooWatcherApplication *watcherApplication, m_watcherApplications) { + qCDebug(BALOO) << "MetadataMover::notifyWatchers" << watcherApplication->service() << watcherApplication->objectName() << watcherApplication->path(); + watcherApplication->renamedFiles(from, to, filesList); + } +} + +void MetadataMover::watcherServiceUnregistered(const QString &serviceName) +{ + auto itService = m_watcherApplications.find(serviceName); + if (itService == m_watcherApplications.end()) { + return; + } + + qDebug() << "MetadataMover::watcherServiceUnregistered" << itService.key(); + + delete itService.value(); + m_watcherApplications.erase(itService); +} diff --git a/src/file/metadatamover.h b/src/file/metadatamover.h index 28ab852a..7e0ec5a6 100644 --- a/src/file/metadatamover.h +++ b/src/file/metadatamover.h @@ -1,71 +1,101 @@ /* This file is part of the KDE Project Copyright (c) 2009-2011 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef BALOO_METADATA_MOVER_H_ #define BALOO_METADATA_MOVER_H_ #include +#include +#include + +class OrgKdeBalooWatcherApplicationInterface; + +namespace org +{ + +namespace kde +{ + +typedef ::OrgKdeBalooWatcherApplicationInterface BalooWatcherApplication; + +} + +} namespace Baloo { class Database; class Transaction; class MetadataMover : public QObject { Q_OBJECT public: MetadataMover(Database* db, QObject* parent = nullptr); ~MetadataMover(); + bool hasWatcher() const; + public Q_SLOTS: void moveFileMetadata(const QString& from, const QString& to); void removeFileMetadata(const QString& file); + void registerBalooWatcher(const QString &service); + Q_SIGNALS: /** * Emitted for files (and folders) that have been moved but * do not have metadata to be moved. This allows the file indexer * service to pick them up in case they are of interest. The * typical example would be moving a file from a non-indexed into * an indexed folder. */ void movedWithoutData(const QString& path); void fileRemoved(const QString& path); +private Q_SLOTS: + + void watcherServiceUnregistered(const QString &serviceName); + private: /** * Remove the metadata for file \p url */ void removeMetadata(Transaction* tr, const QString& url); /** * Recursively update the nie:url and nie:isPartOf properties * of the resource describing \p from. */ void updateMetadata(Transaction* tr, const QString& from, const QString& to); + void notifyWatchers(const QString &from, const QString &to, const QList &filesList); + + QMap m_watcherApplications; + + QDBusServiceWatcher m_serviceWatcher; + Database* m_db; }; } #endif diff --git a/src/file/org.kde.BalooWatcherApplication.xml b/src/file/org.kde.BalooWatcherApplication.xml new file mode 100644 index 00000000..0caba0f9 --- /dev/null +++ b/src/file/org.kde.BalooWatcherApplication.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + +