diff --git a/src/file/filecontentindexer.cpp b/src/file/filecontentindexer.cpp index ae3e5be3..17601b70 100644 --- a/src/file/filecontentindexer.cpp +++ b/src/file/filecontentindexer.cpp @@ -1,152 +1,152 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filecontentindexer.h" #include "filecontentindexerprovider.h" #include "extractorprocess.h" #include #include #include using namespace Baloo; FileContentIndexer::FileContentIndexer(FileIndexerConfig* config, FileContentIndexerProvider* provider, uint& finishedCount, QObject* parent) : QObject(parent) , m_config(config) , m_batchSize(config->maxUncomittedFiles()) , m_provider(provider) , m_finishedCount(finishedCount) , m_stop(0) { Q_ASSERT(provider); QDBusConnection bus = QDBusConnection::sessionBus(); m_monitorWatcher.setConnection(bus); m_monitorWatcher.setWatchMode(QDBusServiceWatcher::WatchForUnregistration); connect(&m_monitorWatcher, &QDBusServiceWatcher::serviceUnregistered, this, &FileContentIndexer::monitorClosed); bus.registerObject(QStringLiteral("/fileindexer"), this, QDBusConnection::ExportScriptableContents); } void FileContentIndexer::run() { ExtractorProcess process; connect(&process, &ExtractorProcess::startedIndexingFile, this, &FileContentIndexer::slotStartedIndexingFile); connect(&process, &ExtractorProcess::finishedIndexingFile, this, &FileContentIndexer::slotFinishedIndexingFile); #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0) m_stop.store(false); #else m_stop.storeRelaxed(false); #endif auto batchSize = m_batchSize; while (true) { // // WARNING: This will go mad, if the Extractor does not commit after N=m_batchSize files // cause then we will keep fetching the same N files again and again. // QVector idList = m_provider->fetch(batchSize); #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0) if (idList.isEmpty() || m_stop.load()) { #else if (idList.isEmpty() || m_stop.loadRelaxed()) { #endif break; } QEventLoop loop; connect(&process, &ExtractorProcess::done, &loop, &QEventLoop::quit); bool hadErrors = false; connect(&process, &ExtractorProcess::failed, &loop, [&hadErrors, &loop]() { hadErrors = true; loop.quit(); }); uint batchStartCount = m_finishedCount; connect(&process, &ExtractorProcess::finishedIndexingFile, &loop, [this]() { m_finishedCount++; }); QElapsedTimer timer; timer.start(); process.index(idList); loop.exec(); batchSize = idList.size(); #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0) if (hadErrors && !m_stop.load()) { #else if (hadErrors && !m_stop.loadRelaxed()) { #endif if (batchSize == 1) { auto failedId = idList.first(); m_provider->markFailed(failedId); batchSize = m_batchSize; } else { batchSize /= 2; } // reset to old value - nothing comitted m_finishedCount = batchStartCount; process.start(); } else { auto elapsed = timer.elapsed(); QMetaObject::invokeMethod(this, - [this, elapsed, batchSize] { newBatchTime(elapsed, batchSize); }, + [this, elapsed, batchSize] { committedBatch(elapsed, batchSize); }, Qt::QueuedConnection); } } QMetaObject::invokeMethod(this, &FileContentIndexer::done, Qt::QueuedConnection); } void FileContentIndexer::slotStartedIndexingFile(const QString& filePath) { m_currentFile = filePath; if (!m_registeredMonitors.isEmpty()) { Q_EMIT startedIndexingFile(filePath); } } void FileContentIndexer::slotFinishedIndexingFile(const QString& filePath) { Q_UNUSED(filePath); if (!m_registeredMonitors.isEmpty()) { Q_EMIT finishedIndexingFile(filePath); } m_currentFile = QString(); } void FileContentIndexer::registerMonitor(const QDBusMessage& message) { if (!m_registeredMonitors.contains(message.service())) { m_registeredMonitors << message.service(); m_monitorWatcher.addWatchedService(message.service()); } } void FileContentIndexer::unregisterMonitor(const QDBusMessage& message) { m_registeredMonitors.removeAll(message.service()); m_monitorWatcher.removeWatchedService(message.service()); } void FileContentIndexer::monitorClosed(const QString& service) { m_registeredMonitors.removeAll(service); m_monitorWatcher.removeWatchedService(service); } diff --git a/src/file/filecontentindexer.h b/src/file/filecontentindexer.h index 35b264d2..6a9be4e7 100644 --- a/src/file/filecontentindexer.h +++ b/src/file/filecontentindexer.h @@ -1,89 +1,89 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_FILECONTENTINDEXER_H #define BALOO_FILECONTENTINDEXER_H #include #include #include #include #include #include #include "fileindexerconfig.h" namespace Baloo { class FileContentIndexerProvider; class FileContentIndexer : public QObject, public QRunnable { Q_OBJECT Q_CLASSINFO("D-Bus Interface", "org.kde.baloo.fileindexer") Q_PROPERTY(QString currentFile READ currentFile NOTIFY startedIndexingFile) public: FileContentIndexer(FileIndexerConfig* config, FileContentIndexerProvider* provider, uint& finishedCount, QObject* parent = nullptr); QString currentFile() { return m_currentFile; } void run() override; void quit() { #if QT_VERSION < QT_VERSION_CHECK(5, 14, 0) m_stop.store(true); #else m_stop.storeRelaxed(true); #endif } public Q_SLOTS: Q_SCRIPTABLE void registerMonitor(const QDBusMessage& message); Q_SCRIPTABLE void unregisterMonitor(const QDBusMessage& message); Q_SIGNALS: Q_SCRIPTABLE void startedIndexingFile(const QString& filePath); Q_SCRIPTABLE void finishedIndexingFile(const QString& filePath); + Q_SCRIPTABLE void committedBatch(uint time, uint batchSize); void done(); - void newBatchTime(uint time, uint batchSize); private Q_SLOTS: void monitorClosed(const QString& service); void slotStartedIndexingFile(const QString& filePath); void slotFinishedIndexingFile(const QString& filePath); private: FileIndexerConfig *m_config; uint m_batchSize; FileContentIndexerProvider* m_provider; uint& m_finishedCount; QAtomicInt m_stop; QString m_currentFile; QStringList m_registeredMonitors; QDBusServiceWatcher m_monitorWatcher; }; } #endif // BALOO_FILECONTENTINDEXER_H diff --git a/src/file/fileindexscheduler.cpp b/src/file/fileindexscheduler.cpp index e95ae224..d2323b3d 100644 --- a/src/file/fileindexscheduler.cpp +++ b/src/file/fileindexscheduler.cpp @@ -1,318 +1,318 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "fileindexscheduler.h" #include "baloodebug.h" #include "firstrunindexer.h" #include "newfileindexer.h" #include "modifiedfileindexer.h" #include "xattrindexer.h" #include "filecontentindexer.h" #include "filecontentindexerprovider.h" #include "unindexedfileindexer.h" #include "indexcleaner.h" #include "fileindexerconfig.h" #include #include using namespace Baloo; FileIndexScheduler::FileIndexScheduler(Database* db, FileIndexerConfig* config, bool firstRun, QObject* parent) : QObject(parent) , m_db(db) , m_config(config) , m_provider(db) , m_contentIndexer(nullptr) , m_indexerState(Startup) , m_timeEstimator(this) , m_checkUnindexedFiles(false) , m_checkStaleIndexEntries(false) , m_isGoingIdle(false) , m_isSuspended(false) , m_isFirstRun(firstRun) , m_inStartup(true) { Q_ASSERT(db); Q_ASSERT(config); m_threadPool.setMaxThreadCount(1); connect(&m_powerMonitor, &PowerStateMonitor::powerManagementStatusChanged, this, &FileIndexScheduler::powerManagementStatusChanged); if (m_powerMonitor.isOnBattery()) { m_indexerState = LowPowerIdle; } m_contentIndexer = new FileContentIndexer(m_config, &m_provider, m_indexFinishedFiles, this); m_contentIndexer->setAutoDelete(false); connect(m_contentIndexer, &FileContentIndexer::done, this, &FileIndexScheduler::runnerFinished); - connect(m_contentIndexer, &FileContentIndexer::newBatchTime, &m_timeEstimator, + connect(m_contentIndexer, &FileContentIndexer::committedBatch, &m_timeEstimator, &TimeEstimator::handleNewBatchTime); QDBusConnection::sessionBus().registerObject(QStringLiteral("/scheduler"), this, QDBusConnection::ExportScriptableContents); } FileIndexScheduler::~FileIndexScheduler() { m_contentIndexer->quit(); m_threadPool.waitForDone(0); // wait 0 msecs } void FileIndexScheduler::startupFinished() { m_inStartup = false; QTimer::singleShot(0, this, &FileIndexScheduler::scheduleIndexing); } void FileIndexScheduler::scheduleIndexing() { if (!isIndexerIdle()) { return; } m_isGoingIdle = false; if (m_isSuspended) { if (m_indexerState != Suspended) { m_indexerState = Suspended; Q_EMIT stateChanged(m_indexerState); } return; } if (m_isFirstRun) { if (m_inStartup) { return; } m_isFirstRun = false; auto runnable = new FirstRunIndexer(m_db, m_config, m_config->includeFolders()); connect(runnable, &FirstRunIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_indexerState = FirstRun; Q_EMIT stateChanged(m_indexerState); return; } if (!m_newFiles.isEmpty()) { auto runnable = new NewFileIndexer(m_db, m_config, m_newFiles); connect(runnable, &NewFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_newFiles.clear(); m_indexerState = NewFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_modifiedFiles.isEmpty()) { auto runnable = new ModifiedFileIndexer(m_db, m_config, m_modifiedFiles); connect(runnable, &ModifiedFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_modifiedFiles.clear(); m_indexerState = ModifiedFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_xattrFiles.isEmpty()) { auto runnable = new XAttrIndexer(m_db, m_config, m_xattrFiles); connect(runnable, &XAttrIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_xattrFiles.clear(); m_indexerState = XAttrFiles; Q_EMIT stateChanged(m_indexerState); return; } // No housekeeping, no content indexing if (m_powerMonitor.isOnBattery()) { if (m_indexerState != LowPowerIdle) { m_indexerState = LowPowerIdle; Q_EMIT stateChanged(m_indexerState); } return; } if (m_inStartup) { if (m_indexerState != Startup) { m_indexerState = Startup; Q_EMIT stateChanged(m_indexerState); } return; } // This has to be above content indexing, because there can be files that // should not be indexed in the DB (i.e. if config was changed) if (m_checkStaleIndexEntries) { auto runnable = new IndexCleaner(m_db, m_config); connect(runnable, &IndexCleaner::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_checkStaleIndexEntries = false; m_indexerState = StaleIndexEntriesClean; Q_EMIT stateChanged(m_indexerState); return; } m_indexPendingFiles = m_provider.size(); m_indexFinishedFiles = 0; if (m_indexPendingFiles) { m_threadPool.start(m_contentIndexer); m_indexerState = ContentIndexing; Q_EMIT stateChanged(m_indexerState); return; } if (m_checkUnindexedFiles) { auto runnable = new UnindexedFileIndexer(m_db, m_config); connect(runnable, &UnindexedFileIndexer::done, this, &FileIndexScheduler::runnerFinished); m_threadPool.start(runnable); m_checkUnindexedFiles = false; m_indexerState = UnindexedFileCheck; Q_EMIT stateChanged(m_indexerState); return; } if (m_indexerState != Idle) { m_indexerState = Idle; Q_EMIT stateChanged(m_indexerState); } } static void removeStartsWith(QStringList& list, const QString& dir) { const auto tail = std::remove_if(list.begin(), list.end(), [&dir](const QString& file) { return file.startsWith(dir); }); list.erase(tail, list.end()); } static void removeShouldNotIndex(QStringList& list, FileIndexerConfig* config) { const auto tail = std::remove_if(list.begin(), list.end(), [config](const QString& file) { return !config->shouldBeIndexed(file); }); list.erase(tail, list.end()); } void FileIndexScheduler::updateConfig() { // Interrupt content indexer, to avoid indexing files that should // not be indexed (bug 373430) if (m_indexerState == ContentIndexing) { m_contentIndexer->quit(); } removeShouldNotIndex(m_newFiles, m_config); removeShouldNotIndex(m_modifiedFiles, m_config); removeShouldNotIndex(m_xattrFiles, m_config); m_checkStaleIndexEntries = true; m_checkUnindexedFiles = true; scheduleIndexing(); } void FileIndexScheduler::handleFileRemoved(const QString& file) { if (!file.endsWith(QLatin1Char('/'))) { m_newFiles.removeOne(file); m_modifiedFiles.removeOne(file); m_xattrFiles.removeOne(file); } else { removeStartsWith(m_newFiles, file); removeStartsWith(m_modifiedFiles, file); removeStartsWith(m_xattrFiles, file); } } void FileIndexScheduler::powerManagementStatusChanged(bool isOnBattery) { qCDebug(BALOO) << "Power state changed - onBattery:" << isOnBattery; if (isOnBattery && m_indexerState == ContentIndexing) { qCDebug(BALOO) << "On battery, stopping content indexer"; m_contentIndexer->quit(); } else { scheduleIndexing(); } } void FileIndexScheduler::setSuspend(bool suspend) { m_isSuspended = suspend; if (suspend) { qCDebug(BALOO) << "Suspending"; if (m_indexerState == ContentIndexing) { m_contentIndexer->quit(); } else { scheduleIndexing(); } } else { qCDebug(BALOO) << "Resuming"; // No need to emit here we'll be emitting in scheduling scheduleIndexing(); } } uint FileIndexScheduler::getRemainingTime() { if (m_indexerState != ContentIndexing) { return 0; } uint remainingFiles = m_indexPendingFiles - m_indexFinishedFiles; return m_timeEstimator.calculateTimeLeft(remainingFiles); } void FileIndexScheduler::scheduleCheckUnindexedFiles() { m_checkUnindexedFiles = true; } void FileIndexScheduler::checkUnindexedFiles() { m_checkUnindexedFiles = true; scheduleIndexing(); } void FileIndexScheduler::scheduleCheckStaleIndexEntries() { m_checkStaleIndexEntries = true; } void FileIndexScheduler::checkStaleIndexEntries() { m_checkStaleIndexEntries = true; scheduleIndexing(); } uint FileIndexScheduler::getBatchSize() { return m_config->maxUncomittedFiles(); }