diff --git a/src/file/extractorprocess.cpp b/src/file/extractorprocess.cpp index e5470087..14dda763 100644 --- a/src/file/extractorprocess.cpp +++ b/src/file/extractorprocess.cpp @@ -1,89 +1,103 @@ /* * Copyright (C) 2015 Vishesh Handa * Copyright (C) 2015 Pinak Ahuja * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "extractorprocess.h" #include #include using namespace Baloo; ExtractorProcess::ExtractorProcess(QObject* parent) : QObject(parent) , m_extractorPath(QStandardPaths::findExecutable(QStringLiteral("baloo_file_extractor"))) , m_extractorProcess(this) { connect(&m_extractorProcess, &QProcess::readyRead, this, &ExtractorProcess::slotIndexingFile); + connect(&m_extractorProcess, QOverload::of(&QProcess::finished), + [=](int exitCode, QProcess::ExitStatus exitStatus) + { + if (exitStatus == QProcess::CrashExit) { + Q_EMIT failed(); + } + }); + + m_extractorProcess.setProgram(m_extractorPath); m_extractorProcess.setProcessChannelMode(QProcess::ForwardedErrorChannel); - m_extractorProcess.start(m_extractorPath, QStringList(), QIODevice::Unbuffered | QIODevice::ReadWrite); - m_extractorProcess.waitForStarted(); - m_extractorProcess.setReadChannel(QProcess::StandardOutput); + m_extractorProcess.start(); } ExtractorProcess::~ExtractorProcess() { m_extractorProcess.close(); } +void ExtractorProcess::start() +{ + m_extractorProcess.start(QIODevice::Unbuffered | QIODevice::ReadWrite); + m_extractorProcess.waitForStarted(); + m_extractorProcess.setReadChannel(QProcess::StandardOutput); +} + void ExtractorProcess::index(const QVector& fileIds) { Q_ASSERT(m_extractorProcess.state() == QProcess::Running); Q_ASSERT(!fileIds.isEmpty()); QByteArray batchData; quint32 batchSize = fileIds.size(); batchData.append(reinterpret_cast(&batchSize), sizeof(quint32)); for (quint64 id : fileIds) { batchData.append(reinterpret_cast(&id), sizeof(quint64)); } m_extractorProcess.write(batchData.data(), batchData.size()); } void ExtractorProcess::slotIndexingFile() { while (m_extractorProcess.canReadLine()) { QString line = m_extractorProcess.readLine().trimmed(); if (line.isEmpty()) { continue; } char command = line[0].toLatin1(); QString arg = line.mid(2); switch (command) { case 'S': Q_EMIT startedIndexingFile(arg); break; case 'F': Q_EMIT finishedIndexingFile(arg); break; case 'B': Q_EMIT done(); break; default: qCritical() << "Got unknown result from extractor" << command << arg; } } } diff --git a/src/file/extractorprocess.h b/src/file/extractorprocess.h index 8cdfab82..4a750e6a 100644 --- a/src/file/extractorprocess.h +++ b/src/file/extractorprocess.h @@ -1,56 +1,58 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_EXTRACTORPROCESS_H #define BALOO_EXTRACTORPROCESS_H #include #include #include #include namespace Baloo { class ExtractorProcess : public QObject { Q_OBJECT public: explicit ExtractorProcess(QObject* parent = nullptr); ~ExtractorProcess(); void index(const QVector& fileIds); + void start(); Q_SIGNALS: void startedIndexingFile(QString filePath); void finishedIndexingFile(QString filePath); void done(); + void failed(); private Q_SLOTS: void slotIndexingFile(); private: const QString m_extractorPath; QProcess m_extractorProcess; QTimer m_timeCurrentFile; int m_processTimeout; }; } #endif // BALOO_EXTRACTORPROCESS_H diff --git a/src/file/filecontentindexer.cpp b/src/file/filecontentindexer.cpp index df28d1fb..287565c4 100644 --- a/src/file/filecontentindexer.cpp +++ b/src/file/filecontentindexer.cpp @@ -1,117 +1,133 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filecontentindexer.h" #include "filecontentindexerprovider.h" #include "extractorprocess.h" #include #include #include #include using namespace Baloo; FileContentIndexer::FileContentIndexer(FileIndexerConfig* config, FileContentIndexerProvider* provider, QObject* parent) : QObject(parent) , m_config(config) , m_batchSize(config->maxUncomittedFiles()) , m_provider(provider) , m_stop(0) { Q_ASSERT(provider); QDBusConnection bus = QDBusConnection::sessionBus(); m_monitorWatcher.setConnection(bus); m_monitorWatcher.setWatchMode(QDBusServiceWatcher::WatchForUnregistration); connect(&m_monitorWatcher, &QDBusServiceWatcher::serviceUnregistered, this, &FileContentIndexer::monitorClosed); bus.registerObject(QStringLiteral("/fileindexer"), this, QDBusConnection::ExportScriptableContents); } void FileContentIndexer::run() { ExtractorProcess process; connect(&process, &ExtractorProcess::startedIndexingFile, this, &FileContentIndexer::slotStartedIndexingFile); connect(&process, &ExtractorProcess::finishedIndexingFile, this, &FileContentIndexer::slotFinishedIndexingFile); m_stop.store(false); + auto batchSize = m_batchSize; + while (m_provider->size() && !m_stop.load()) { // // WARNING: This will go mad, if the Extractor does not commit after N=m_batchSize files // cause then we will keep fetching the same N files again and again. // QElapsedTimer timer; timer.start(); - QVector idList = m_provider->fetch(m_batchSize); + QVector idList = m_provider->fetch(batchSize); if (idList.isEmpty() || m_stop.load()) { break; } QEventLoop loop; connect(&process, &ExtractorProcess::done, &loop, &QEventLoop::quit); + bool hadErrors = false; + connect(&process, &ExtractorProcess::failed, &loop, [&hadErrors, &loop]() { hadErrors = true; loop.quit(); }); + process.index(idList); loop.exec(); // QDbus requires us to be in object creation thread (thread affinity) // This signal is not even exported, and yet QDbus complains. QDbus bug? QMetaObject::invokeMethod(this, "newBatchTime", Qt::QueuedConnection, Q_ARG(uint, timer.elapsed())); + + if (hadErrors && !m_stop.load()) { + if (idList.size() == 1) { + auto failedId = idList.first(); + m_provider->markFailed(failedId); + batchSize = m_batchSize; + } else { + batchSize = idList.size() / 2; + } + process.start(); + } } QMetaObject::invokeMethod(this, "done", Qt::QueuedConnection); } void FileContentIndexer::slotStartedIndexingFile(const QString& filePath) { m_currentFile = filePath; if (!m_registeredMonitors.isEmpty()) { Q_EMIT startedIndexingFile(filePath); } } void FileContentIndexer::slotFinishedIndexingFile(const QString& filePath) { Q_UNUSED(filePath); if (!m_registeredMonitors.isEmpty()) { Q_EMIT finishedIndexingFile(filePath); } m_currentFile = QString(); } void FileContentIndexer::registerMonitor(const QDBusMessage& message) { if (!m_registeredMonitors.contains(message.service())) { m_registeredMonitors << message.service(); m_monitorWatcher.addWatchedService(message.service()); } } void FileContentIndexer::unregisterMonitor(const QDBusMessage& message) { m_registeredMonitors.removeAll(message.service()); m_monitorWatcher.removeWatchedService(message.service()); } void FileContentIndexer::monitorClosed(const QString& service) { m_registeredMonitors.removeAll(service); m_monitorWatcher.removeWatchedService(service); } diff --git a/src/file/filecontentindexerprovider.cpp b/src/file/filecontentindexerprovider.cpp index 91f3b6f2..291b5609 100644 --- a/src/file/filecontentindexerprovider.cpp +++ b/src/file/filecontentindexerprovider.cpp @@ -1,42 +1,52 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filecontentindexerprovider.h" #include "transaction.h" #include "database.h" using namespace Baloo; FileContentIndexerProvider::FileContentIndexerProvider(Database* db) : m_db(db) { } QVector FileContentIndexerProvider::fetch(uint size) { Transaction tr(m_db, Transaction::ReadOnly); return tr.fetchPhaseOneIds(size); } uint FileContentIndexerProvider::size() { Transaction tr(m_db, Transaction::ReadOnly); return tr.phaseOneSize(); } + +void FileContentIndexerProvider::markFailed(quint64 id) +{ + Transaction tr(m_db, Transaction::ReadWrite); + if (!tr.hasFailed(id)) { + tr.addFailed(id); + } + tr.removePhaseOne(id); + tr.commit(); +} diff --git a/src/file/filecontentindexerprovider.h b/src/file/filecontentindexerprovider.h index 56da01a6..936d7538 100644 --- a/src/file/filecontentindexerprovider.h +++ b/src/file/filecontentindexerprovider.h @@ -1,42 +1,43 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_FILECONTENTINDEXERPROVIDER_H #define BALOO_FILECONTENTINDEXERPROVIDER_H #include namespace Baloo { class Database; class FileContentIndexerProvider { public: explicit FileContentIndexerProvider(Database* db); uint size(); QVector fetch(uint size); + void markFailed(quint64 id); private: Database* m_db; }; } #endif // BALOO_FILECONTENTINDEXERPROVIDER_H