diff --git a/src/file/extractorprocess.h b/src/file/extractorprocess.h --- a/src/file/extractorprocess.h +++ b/src/file/extractorprocess.h @@ -35,11 +35,13 @@ ~ExtractorProcess(); void index(const QVector& fileIds); + void start(); Q_SIGNALS: void startedIndexingFile(QString filePath); void finishedIndexingFile(QString filePath); void done(); + void failed(); private Q_SLOTS: void slotIndexingFile(); diff --git a/src/file/extractorprocess.cpp b/src/file/extractorprocess.cpp --- a/src/file/extractorprocess.cpp +++ b/src/file/extractorprocess.cpp @@ -31,17 +31,31 @@ , m_extractorProcess(this) { connect(&m_extractorProcess, &QProcess::readyRead, this, &ExtractorProcess::slotIndexingFile); + connect(&m_extractorProcess, QOverload::of(&QProcess::finished), + [=](int exitCode, QProcess::ExitStatus exitStatus) + { + if (exitStatus == QProcess::CrashExit) { + Q_EMIT failed(); + } + }); + + m_extractorProcess.setProgram(m_extractorPath); m_extractorProcess.setProcessChannelMode(QProcess::ForwardedErrorChannel); - m_extractorProcess.start(m_extractorPath, QStringList(), QIODevice::Unbuffered | QIODevice::ReadWrite); - m_extractorProcess.waitForStarted(); - m_extractorProcess.setReadChannel(QProcess::StandardOutput); + m_extractorProcess.start(); } ExtractorProcess::~ExtractorProcess() { m_extractorProcess.close(); } +void ExtractorProcess::start() +{ + m_extractorProcess.start(QIODevice::Unbuffered | QIODevice::ReadWrite); + m_extractorProcess.waitForStarted(); + m_extractorProcess.setReadChannel(QProcess::StandardOutput); +} + void ExtractorProcess::index(const QVector& fileIds) { Q_ASSERT(m_extractorProcess.state() == QProcess::Running); diff --git a/src/file/filecontentindexer.cpp b/src/file/filecontentindexer.cpp --- a/src/file/filecontentindexer.cpp +++ b/src/file/filecontentindexer.cpp @@ -54,27 +54,43 @@ connect(&process, &ExtractorProcess::finishedIndexingFile, this, &FileContentIndexer::slotFinishedIndexingFile); m_stop.store(false); + auto batchSize = m_batchSize; + while (m_provider->size() && !m_stop.load()) { // // WARNING: This will go mad, if the Extractor does not commit after N=m_batchSize files // cause then we will keep fetching the same N files again and again. // QElapsedTimer timer; timer.start(); - QVector idList = m_provider->fetch(m_batchSize); + QVector idList = m_provider->fetch(batchSize); if (idList.isEmpty() || m_stop.load()) { break; } QEventLoop loop; connect(&process, &ExtractorProcess::done, &loop, &QEventLoop::quit); + bool hadErrors = false; + connect(&process, &ExtractorProcess::failed, &loop, [&hadErrors, &loop]() { hadErrors = true; loop.quit(); }); + process.index(idList); loop.exec(); // QDbus requires us to be in object creation thread (thread affinity) // This signal is not even exported, and yet QDbus complains. QDbus bug? QMetaObject::invokeMethod(this, "newBatchTime", Qt::QueuedConnection, Q_ARG(uint, timer.elapsed())); + + if (hadErrors && !m_stop.load()) { + if (idList.size() == 1) { + auto failedId = idList.first(); + m_provider->markFailed(failedId); + batchSize = m_batchSize; + } else { + batchSize = idList.size() / 2; + } + process.start(); + } } QMetaObject::invokeMethod(this, "done", Qt::QueuedConnection); } diff --git a/src/file/filecontentindexerprovider.h b/src/file/filecontentindexerprovider.h --- a/src/file/filecontentindexerprovider.h +++ b/src/file/filecontentindexerprovider.h @@ -33,6 +33,7 @@ uint size(); QVector fetch(uint size); + void markFailed(quint64 id); private: Database* m_db; diff --git a/src/file/filecontentindexerprovider.cpp b/src/file/filecontentindexerprovider.cpp --- a/src/file/filecontentindexerprovider.cpp +++ b/src/file/filecontentindexerprovider.cpp @@ -40,3 +40,13 @@ Transaction tr(m_db, Transaction::ReadOnly); return tr.phaseOneSize(); } + +void FileContentIndexerProvider::markFailed(quint64 id) +{ + Transaction tr(m_db, Transaction::ReadWrite); + if (!tr.hasFailed(id)) { + tr.addFailed(id); + } + tr.removePhaseOne(id); + tr.commit(); +}