diff --git a/autotests/unit/file/pendingfilequeuetest.cpp b/autotests/unit/file/pendingfilequeuetest.cpp index 47d48792..16dcbf3c 100644 --- a/autotests/unit/file/pendingfilequeuetest.cpp +++ b/autotests/unit/file/pendingfilequeuetest.cpp @@ -1,116 +1,116 @@ /* This file is part of the KDE Baloo project. Copyright (C) 2011 Sebastian Trueg Copyright (C) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) version 3, or any later version accepted by the membership of KDE e.V. (or its successor approved by the membership of KDE e.V.), which shall act as a proxy defined in Section 6 of version 3 of the license. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library. If not, see . */ #include "pendingfilequeue.h" #include #include #include using namespace Baloo; class PendingFileQueueTest : public QObject { Q_OBJECT public: PendingFileQueueTest(); private Q_SLOTS: void testTimeout(); void testRequeue(); }; PendingFileQueueTest::PendingFileQueueTest() { } void PendingFileQueueTest::testTimeout() { QString myUrl(QStringLiteral("/tmp")); PendingFileQueue queue; queue.setMinimumTimeout(2); QSignalSpy spy(&queue, SIGNAL(indexModifiedFile(QString))); QVERIFY(spy.isValid()); PendingFile file(myUrl); file.setModified(); queue.enqueue(file); // The signal should be emitted immediately QVERIFY(spy.wait()); QCOMPARE(spy.count(), 1); - QCOMPARE(spy.takeFirst().first().toString(), myUrl); + QCOMPARE(spy.takeFirst().constFirst().toString(), myUrl); // Enqueue the url again. This time it should wait for should wait for the // minimumTimeout queue.enqueue(file); QTest::qWait(1000); QVERIFY(spy.isEmpty()); QVERIFY(spy.wait(2000)); QCOMPARE(spy.count(), 1); - QCOMPARE(spy.takeFirst().first().toString(), myUrl); + QCOMPARE(spy.takeFirst().constFirst().toString(), myUrl); } void PendingFileQueueTest::testRequeue() { QString myUrl(QStringLiteral("/tmp")); PendingFileQueue queue; queue.setMinimumTimeout(2); queue.setMaximumTimeout(5); QSignalSpy spy(&queue, SIGNAL(indexModifiedFile(QString))); QVERIFY(spy.isValid()); PendingFile file(myUrl); file.setModified(); queue.enqueue(file); // The signal should be emitted immediately QTest::qWait(20); QCOMPARE(spy.count(), 1); - QCOMPARE(spy.takeFirst().first().toString(), myUrl); + QCOMPARE(spy.takeFirst().constFirst().toString(), myUrl); // Send many events queue.enqueue(file); QTest::qWait(20); queue.enqueue(file); QTest::qWait(20); queue.enqueue(file); QTest::qWait(20); QTest::qWait(3500); QVERIFY(spy.isEmpty()); QVERIFY(spy.wait(2500)); QCOMPARE(spy.count(), 1); - QCOMPARE(spy.takeFirst().first().toString(), myUrl); + QCOMPARE(spy.takeFirst().constFirst().toString(), myUrl); } QTEST_GUILESS_MAIN(PendingFileQueueTest) #include "pendingfilequeuetest.moc" diff --git a/src/engine/transaction.cpp b/src/engine/transaction.cpp index aa6773a0..07e232eb 100644 --- a/src/engine/transaction.cpp +++ b/src/engine/transaction.cpp @@ -1,593 +1,595 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "transaction.h" #include "postingdb.h" #include "documentdb.h" #include "documenturldb.h" #include "documentiddb.h" #include "positiondb.h" #include "documentdatadb.h" #include "mtimedb.h" #include "document.h" #include "enginequery.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "phraseanditerator.h" #include "writetransaction.h" #include "idutils.h" #include "database.h" #include "databasesize.h" #include #include using namespace Baloo; Transaction::Transaction(const Database& db, Transaction::TransactionType type) : m_dbis(db.m_dbis) , m_env(db.m_env) , m_writeTrans(nullptr) { uint flags = type == ReadOnly ? MDB_RDONLY : 0; int rc = mdb_txn_begin(db.m_env, nullptr, flags, &m_txn); Q_ASSERT_X(rc == 0, "Transaction", mdb_strerror(rc)); if (type == ReadWrite) { m_writeTrans = new WriteTransaction(m_dbis, m_txn); } } Transaction::Transaction(Database* db, Transaction::TransactionType type) : Transaction(*db, type) { } Transaction::~Transaction() { if (m_writeTrans) qWarning() << "Closing an active WriteTransaction without calling abort/commit"; if (m_txn) { abort(); } } bool Transaction::hasDocument(quint64 id) const { Q_ASSERT(id > 0); IdFilenameDB idFilenameDb(m_dbis.idFilenameDbi, m_txn); return idFilenameDb.contains(id); } bool Transaction::inPhaseOne(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.contains(id); } bool Transaction::hasFailed(quint64 id) const { Q_ASSERT(id > 0); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); return failedIdDb.contains(id); } QByteArray Transaction::documentUrl(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.get(id); } quint64 Transaction::documentId(const QByteArray& path) const { Q_ASSERT(m_txn); Q_ASSERT(!path.isEmpty()); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); QList li = path.split('/'); quint64 parentId = 0; for (const QByteArray& fileName : li) { if (fileName.isEmpty()) { continue; } parentId = docUrlDb.getId(parentId, fileName); if (!parentId) { return 0; } } return parentId; } QVector Transaction::childrenDocumentId(quint64 parentId) const { DocumentUrlDB docUrlDB(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDB.getChildren(parentId); } DocumentTimeDB::TimeInfo Transaction::documentTimeInfo(quint64 id) const { Q_ASSERT(m_txn); DocumentTimeDB docTimeDb(m_dbis.docTimeDbi, m_txn); return docTimeDb.get(id); } QByteArray Transaction::documentData(quint64 id) const { Q_ASSERT(m_txn); Q_ASSERT(id > 0); DocumentDataDB docDataDb(m_dbis.docDataDbi, m_txn); return docDataDb.get(id); } bool Transaction::hasChanges() const { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); return m_writeTrans->hasChanges(); } QVector Transaction::fetchPhaseOneIds(int size) const { Q_ASSERT(m_txn); Q_ASSERT(size > 0); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.fetchItems(size); } QVector Transaction::fetchTermsStartingWith(const QByteArray& term) const { Q_ASSERT(term.size() > 0); PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.fetchTermsStartingWith(term); } uint Transaction::phaseOneSize() const { Q_ASSERT(m_txn); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); return contentIndexingDb.size(); } uint Transaction::size() const { Q_ASSERT(m_txn); DocumentDB docTermsDb(m_dbis.docTermsDbi, m_txn); return docTermsDb.size(); } // // Write Operations // void Transaction::setPhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.put(id); } void Transaction::removePhaseOne(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB contentIndexingDb(m_dbis.contentIndexingDbi, m_txn); contentIndexingDb.del(id); } void Transaction::addFailed(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); DocumentIdDB failedIdDb(m_dbis.failedIdDbi, m_txn); failedIdDb.put(id); } void Transaction::addDocument(const Document& doc) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); m_writeTrans->addDocument(doc); } void Transaction::removeDocument(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); m_writeTrans->removeDocument(id); } void Transaction::removeRecursively(quint64 id) { Q_ASSERT(m_txn); Q_ASSERT(id > 0); Q_ASSERT(m_writeTrans); m_writeTrans->removeRecursively(id); } void Transaction::replaceDocument(const Document& doc, DocumentOperations operations) { Q_ASSERT(m_txn); Q_ASSERT(doc.id() > 0); Q_ASSERT(m_writeTrans); Q_ASSERT_X(hasDocument(doc.id()), "Transaction::replaceDocument", "Document does not exist"); m_writeTrans->replaceDocument(doc, operations); } void Transaction::commit() { Q_ASSERT(m_txn); Q_ASSERT(m_writeTrans); m_writeTrans->commit(); delete m_writeTrans; m_writeTrans = nullptr; int rc = mdb_txn_commit(m_txn); Q_ASSERT_X(rc == 0, "Transaction::commit", mdb_strerror(rc)); m_txn = nullptr; } void Transaction::abort() { Q_ASSERT(m_txn); mdb_txn_abort(m_txn); m_txn = nullptr; delete m_writeTrans; m_writeTrans = nullptr; } // // Queries // PostingIterator* Transaction::postingIterator(const EngineQuery& query) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); PositionDB positionDb(m_dbis.positionDBi, m_txn); if (query.leaf()) { if (query.op() == EngineQuery::Equal) { return postingDb.iter(query.term()); } else if (query.op() == EngineQuery::StartsWith) { return postingDb.prefixIter(query.term()); } else { Q_ASSERT(0); } } if (query.subQueries().isEmpty()) { return nullptr; } QVector vec; vec.reserve(query.subQueries().size()); if (query.op() == EngineQuery::Phrase) { - for (const EngineQuery& q : query.subQueries()) { + const auto subQueries = query.subQueries(); + for (const EngineQuery& q : subQueries) { Q_ASSERT_X(q.leaf(), "Transaction::toPostingIterator", "Phrase queries must contain leaf queries"); vec << positionDb.iter(q.term()); } return new PhraseAndIterator(vec); } - for (const EngineQuery& q : query.subQueries()) { + const auto subQueries = query.subQueries(); + for (const EngineQuery& q : subQueries) { vec << postingIterator(q); } if (query.op() == EngineQuery::And) { return new AndPostingIterator(vec); } else if (query.op() == EngineQuery::Or) { return new OrPostingIterator(vec); } Q_ASSERT(0); return nullptr; } PostingIterator* Transaction::postingCompIterator(const QByteArray& prefix, const QByteArray& value, PostingDB::Comparator com) const { PostingDB postingDb(m_dbis.postingDbi, m_txn); return postingDb.compIter(prefix, value, com); } PostingIterator* Transaction::mTimeIter(quint32 mtime, MTimeDB::Comparator com) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iter(mtime, com); } PostingIterator* Transaction::mTimeRangeIter(quint32 beginTime, quint32 endTime) const { MTimeDB mTimeDb(m_dbis.mtimeDbi, m_txn); return mTimeDb.iterRange(beginTime, endTime); } PostingIterator* Transaction::docUrlIter(quint64 id) const { DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); return docUrlDb.iter(id); } QVector Transaction::exec(const EngineQuery& query, int limit) const { Q_ASSERT(m_txn); QVector results; PostingIterator* it = postingIterator(query); if (!it) { return results; } while (it->next() && limit) { results << it->docId(); limit--; } return results; } // // Introspection // QVector Transaction::documentTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); return documentTermsDB.get(docId); } QVector Transaction::documentFileNameTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); return documentFileNameTermsDB.get(docId); } QVector Transaction::documentXattrTerms(quint64 docId) const { Q_ASSERT(docId); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); return documentXattrTermsDB.get(docId); } // // File Size // static size_t dbiSize(MDB_txn* txn, MDB_dbi dbi) { MDB_stat stat; mdb_stat(txn, dbi, &stat); return (stat.ms_branch_pages + stat.ms_leaf_pages + stat.ms_overflow_pages) * stat.ms_psize; } DatabaseSize Transaction::dbSize() { DatabaseSize dbSize; dbSize.postingDb = dbiSize(m_txn, m_dbis.postingDbi); dbSize.positionDb = dbiSize(m_txn, m_dbis.positionDBi); dbSize.docTerms = dbiSize(m_txn, m_dbis.docTermsDbi); dbSize.docFilenameTerms = dbiSize(m_txn, m_dbis.docFilenameTermsDbi); dbSize.docXattrTerms = dbiSize(m_txn, m_dbis.docXattrTermsDbi); dbSize.idTree = dbiSize(m_txn, m_dbis.idTreeDbi); dbSize.idFilename = dbiSize(m_txn, m_dbis.idFilenameDbi); dbSize.docTime = dbiSize(m_txn, m_dbis.docTimeDbi); dbSize.docData = dbiSize(m_txn, m_dbis.docDataDbi); dbSize.contentIndexingIds = dbiSize(m_txn, m_dbis.contentIndexingDbi); dbSize.failedIds = dbiSize(m_txn, m_dbis.failedIdDbi); dbSize.mtimeDb = dbiSize(m_txn, m_dbis.mtimeDbi); dbSize.expectedSize = dbSize.positionDb + dbSize.positionDb + dbSize.docTerms + dbSize.docFilenameTerms + dbSize.docXattrTerms + dbSize.idTree + dbSize.idFilename + dbSize.docTime + dbSize.docData + dbSize.contentIndexingIds + dbSize.failedIds + dbSize.mtimeDb; MDB_envinfo info; mdb_env_info(m_env, &info); dbSize.actualSize = info.me_last_pgno * 4096; // TODO: separate page size return dbSize; } // // Debugging // void Transaction::checkFsTree() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); DocumentUrlDB docUrlDb(m_dbis.idTreeDbi, m_dbis.idFilenameDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "Total Document IDs: " << allIds.size() << endl; int count = 0; - for (quint64 id: allIds) { + for (quint64 id: qAsConst(allIds)) { QByteArray url = docUrlDb.get(id); if (url.isEmpty()) { auto terms = documentTermsDB.get(id); auto fileNameTerms = documentFileNameTermsDB.get(id); auto xAttrTerms = documentXattrTermsDB.get(id); // Lets reverse enginer the terms QList newTerms; QMapIterator it(map); while (it.hasNext()) { it.next(); if (it.value().contains(id)) { newTerms << it.key(); } } out << "Missing filePath for " << id << endl; out << "\tPostingDB Terms: "; - for (const QByteArray& term : newTerms) { + for (const QByteArray& term : qAsConst(newTerms)) { out << term << " "; } out << endl; out << "\tDocumentTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; out << "\tFileNameTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; out << "\tXAttrTermsDB: "; for (const QByteArray& term : terms) { out << term << " "; } out << endl; count++; } else if (!QFileInfo::exists(url)) { out << "FilePath " << url << " for " << id << " does not exist"<< endl; count++; } } out << "Invalid Entries: " << count << " (" << count * 100.0 / allIds.size() << "%)" << endl; } void Transaction::checkTermsDbinPostingDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); // Iterate over each document, and fetch all terms // check if each term maps to its own id in the posting db auto map = postingDb.toTestMap(); QSet allIds; Q_FOREACH (const auto& list, map) { for (quint64 id : list) { allIds << id; } } QTextStream out(stdout); out << "PostingDB check .." << endl; - for (quint64 id : allIds) { + for (quint64 id : qAsConst(allIds)) { QVector terms = documentTermsDB.get(id); terms += documentXattrTermsDB.get(id); terms += documentFileNameTermsDB.get(id); - for (const QByteArray& term : terms) { + for (const QByteArray& term : qAsConst(terms)) { PostingList plist = postingDb.get(term); if (!plist.contains(id)) { out << id << " is missing term " << term << endl; } } } } void Transaction::checkPostingDbinTermsDb() { DocumentDB documentTermsDB(m_dbis.docTermsDbi, m_txn); DocumentDB documentXattrTermsDB(m_dbis.docXattrTermsDbi, m_txn); DocumentDB documentFileNameTermsDB(m_dbis.docFilenameTermsDbi, m_txn); PostingDB postingDb(m_dbis.postingDbi, m_txn); QMap map = postingDb.toTestMap(); QMapIterator it(map); QTextStream out(stdout); out << "DocumentTermsDB check .." << endl; while (it.hasNext()) { it.next(); const QByteArray& term = it.key(); const PostingList& list = it.value(); for (quint64 id : list) { if (documentTermsDB.get(id).contains(term)) { continue; } if (documentFileNameTermsDB.get(id).contains(term)) { continue; } if (documentXattrTermsDB.get(id).contains(term)) { continue; } out << id << " is missing " << QString::fromUtf8(term) << " from document terms db" << endl; } } } diff --git a/src/file/fileindexerconfig.cpp b/src/file/fileindexerconfig.cpp index eae3a2c4..2e694e69 100644 --- a/src/file/fileindexerconfig.cpp +++ b/src/file/fileindexerconfig.cpp @@ -1,399 +1,400 @@ /* This file is part of the KDE Project Copyright (c) 2008-2010 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "fileindexerconfig.h" #include "fileexcludefilters.h" #include "storagedevices.h" #include #include #include #include #include namespace { /// recursively check if a folder is hidden bool isDirHidden(QDir& dir) { #ifdef __unix__ return dir.absolutePath().contains(QLatin1String("/.")); #else if (QFileInfo(dir.path()).isHidden()) return true; else if (dir.cdUp()) return isDirHidden(dir); else return false; #endif } QString stripTrailingSlash(const QString& path) { return path.endsWith('/') ? path.mid(0, path.length()-1) : path; } } using namespace Baloo; FileIndexerConfig::FileIndexerConfig(QObject* parent) : QObject(parent) , m_config(QStringLiteral("baloofilerc")) , m_folderCacheDirty(true) , m_indexHidden(false) , m_devices(nullptr) , m_maxUncomittedFiles(40) { forceConfigUpdate(); } FileIndexerConfig::~FileIndexerConfig() { } QStringList FileIndexerConfig::includeFolders() const { const_cast(this)->buildFolderCache(); QStringList fl; for (int i = 0; i < m_folderCache.count(); ++i) { if (m_folderCache[i].second) fl << m_folderCache[i].first; } return fl; } QStringList FileIndexerConfig::excludeFolders() const { const_cast(this)->buildFolderCache(); QStringList fl; for (int i = 0; i < m_folderCache.count(); ++i) { if (!m_folderCache[i].second) fl << m_folderCache[i].first; } return fl; } QStringList FileIndexerConfig::excludeFilters() const { KConfigGroup cfg = m_config.group("General"); // read configured exclude filters QSet filters = cfg.readEntry("exclude filters", defaultExcludeFilterList()).toSet(); // make sure we always keep the latest default exclude filters // TODO: there is one problem here. What if the user removed some of the default filters? if (cfg.readEntry("exclude filters version", 0) < defaultExcludeFilterListVersion()) { filters += defaultExcludeFilterList().toSet(); // write the config directly since the KCM does not have support for the version yet // TODO: make this class public and use it in the KCM KConfig config(m_config.name()); KConfigGroup cfg = config.group("General"); cfg.writeEntry("exclude filters", QStringList::fromSet(filters)); cfg.writeEntry("exclude filters version", defaultExcludeFilterListVersion()); } // remove duplicates return QStringList::fromSet(filters); } QStringList FileIndexerConfig::excludeMimetypes() const { return QStringList::fromSet(m_excludeMimetypes); } bool FileIndexerConfig::indexHiddenFilesAndFolders() const { return m_indexHidden; } bool FileIndexerConfig::onlyBasicIndexing() const { return m_onlyBasicIndexing; } bool FileIndexerConfig::isInitialRun() const { return m_config.group("General").readEntry("first run", true); } bool FileIndexerConfig::canBeSearched(const QString& folder) const { QFileInfo fi(folder); QString path = fi.absolutePath(); if (!fi.isDir()) { return false; } else if (shouldFolderBeIndexed(path)) { return true; } const_cast(this)->buildFolderCache(); // Look for included descendants for (const QPair& fld: m_folderCache) { if (fld.second && fld.first.startsWith(path)) { return true; } } return false; } bool FileIndexerConfig::shouldBeIndexed(const QString& path) const { QFileInfo fi(path); if (fi.isDir()) { return shouldFolderBeIndexed(path); } else { return (shouldFolderBeIndexed(fi.absolutePath()) && (!fi.isHidden() || indexHiddenFilesAndFolders()) && shouldFileBeIndexed(fi.fileName())); } } bool FileIndexerConfig::shouldFolderBeIndexed(const QString& path) const { QString folder; if (folderInFolderList(path, folder)) { // we always index the folders in the list // ignoring the name filters if (folder == path) return true; // check for hidden folders QDir dir(path); if (!indexHiddenFilesAndFolders() && isDirHidden(dir)) return false; // reset dir, cause isDirHidden modifies the QDir dir = path; // check the exclude filters for all components of the path // after folder const QStringList pathComponents = path.mid(folder.count()).split(QLatin1Char('/'), QString::SkipEmptyParts); Q_FOREACH (const QString& c, pathComponents) { if (!shouldFileBeIndexed(c)) { return false; } } return true; } else { return false; } } bool FileIndexerConfig::shouldFileBeIndexed(const QString& fileName) const { if (!indexHiddenFilesAndFolders() && fileName.startsWith('.')) { return false; } return !m_excludeFilterRegExpCache.exactMatch(fileName); } bool FileIndexerConfig::shouldMimeTypeBeIndexed(const QString& mimeType) const { return !m_excludeMimetypes.contains(mimeType); } bool FileIndexerConfig::folderInFolderList(const QString& path) { QString str; return folderInFolderList(path, str); } bool FileIndexerConfig::folderInFolderList(const QString& path, QString& folder) const { const_cast(this)->buildFolderCache(); const QString p = stripTrailingSlash(path); // we traverse the list backwards to catch all exclude folders int i = m_folderCache.count(); while (--i >= 0) { const QString& f = m_folderCache[i].first; const bool include = m_folderCache[i].second; if (p.startsWith(f)) { folder = f; return include; } } // path is not in the list, thus it should not be included folder.clear(); return false; } namespace { /** * Returns true if the specified folder f would already be excluded using the list * folders. */ bool alreadyExcluded(const QList >& folders, const QString& f) { bool included = false; for (int i = 0; i < folders.count(); ++i) { QString path = folders[i].first; if (!path.endsWith(QLatin1Char('/'))) path.append(QLatin1Char('/')); if (f != folders[i].first && f.startsWith(path)) { included = folders[i].second; } } return !included; } /** * Simple insertion sort */ void insertSortFolders(const QStringList& folders, bool include, QList >& result) { Q_FOREACH (const QString& f, folders) { int pos = 0; QString path = stripTrailingSlash(f); while (result.count() > pos && result[pos].first < path) ++pos; result.insert(pos, qMakePair(path, include)); } } /** * Remove useless exclude entries which would confuse the folderInFolderList algo. * This makes sure all top-level items are include folders. * This runs in O(n^2) and could be optimized but what for. */ void cleanupList(QList >& result) { int i = 0; while (i < result.count()) { if (result[i].first.isEmpty() || (!result[i].second && alreadyExcluded(result, result[i].first))) result.removeAt(i); else ++i; } } } void FileIndexerConfig::buildFolderCache() { if (!m_folderCacheDirty) { return; } if (!m_devices) { m_devices = new StorageDevices(this); } KConfigGroup group = m_config.group("General"); QStringList includeFoldersPlain = group.readPathEntry("folders", QStringList() << QDir::homePath()); QStringList excludeFoldersPlain = group.readPathEntry("exclude folders", QStringList()); // Add all removable media and network shares as ignored unless they have // been explicitly added in the include list - for (const auto& device: m_devices->allMedia()) { + const auto allMedia = m_devices->allMedia(); + for (const auto& device: allMedia) { const QString mountPath = device.mountPath(); if (!device.isUsable() && !mountPath.isEmpty()) { if (!includeFoldersPlain.contains(mountPath)) { excludeFoldersPlain << mountPath; } } } m_folderCache.clear(); insertSortFolders(includeFoldersPlain, true, m_folderCache); insertSortFolders(excludeFoldersPlain, false, m_folderCache); cleanupList(m_folderCache); m_folderCacheDirty = false; } void FileIndexerConfig::buildExcludeFilterRegExpCache() { QStringList newFilters = excludeFilters(); m_excludeFilterRegExpCache.rebuildCacheFromFilterList(newFilters); } void FileIndexerConfig::buildMimeTypeCache() { m_excludeMimetypes = m_config.group("General").readPathEntry("exclude mimetypes", defaultExcludeMimetypes()).toSet(); } void FileIndexerConfig::forceConfigUpdate() { m_config.reparseConfiguration(); m_folderCacheDirty = true; buildExcludeFilterRegExpCache(); buildMimeTypeCache(); m_indexHidden = m_config.group("General").readEntry("index hidden folders", false); m_onlyBasicIndexing = m_config.group("General").readEntry("only basic indexing", false); } void FileIndexerConfig::setInitialRun(bool isInitialRun) { m_config.group("General").writeEntry("first run", isInitialRun); m_config.sync(); } bool FileIndexerConfig::initialUpdateDisabled() const { return m_config.group("General").readEntry("disable initial update", true); } int FileIndexerConfig::databaseVersion() const { return m_config.group("General").readEntry("dbVersion", 0); } void FileIndexerConfig::setDatabaseVersion(int version) { m_config.group("General").writeEntry("dbVersion", version); m_config.sync(); } bool FileIndexerConfig::indexingEnabled() const { return m_config.group("Basic Settings").readEntry("Indexing-Enabled", true); } uint FileIndexerConfig::maxUncomittedFiles() { return m_maxUncomittedFiles; } diff --git a/src/file/fileindexscheduler.cpp b/src/file/fileindexscheduler.cpp index f5ab2a77..d4188f01 100644 --- a/src/file/fileindexscheduler.cpp +++ b/src/file/fileindexscheduler.cpp @@ -1,215 +1,215 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "fileindexscheduler.h" #include "firstrunindexer.h" #include "newfileindexer.h" #include "modifiedfileindexer.h" #include "xattrindexer.h" #include "filecontentindexer.h" #include "filecontentindexerprovider.h" #include "unindexedfileindexer.h" #include "fileindexerconfig.h" #include #include #include using namespace Baloo; FileIndexScheduler::FileIndexScheduler(Database* db, FileIndexerConfig* config, QObject* parent) : QObject(parent) , m_db(db) , m_config(config) , m_provider(db) , m_contentIndexer(nullptr) , m_indexerState(Idle) , m_timeEstimator(config, this) , m_checkUnindexedFiles(false) { Q_ASSERT(db); Q_ASSERT(config); m_threadPool.setMaxThreadCount(1); connect(&m_powerMonitor, &PowerStateMonitor::powerManagementStatusChanged, this, &FileIndexScheduler::powerManagementStatusChanged); m_contentIndexer = new FileContentIndexer(m_config, &m_provider, this); m_contentIndexer->setAutoDelete(false); connect(m_contentIndexer, &FileContentIndexer::done, this, &FileIndexScheduler::scheduleIndexing); connect(m_contentIndexer, &FileContentIndexer::newBatchTime, &m_timeEstimator, &TimeEstimator::handleNewBatchTime); QDBusConnection::sessionBus().registerObject(QStringLiteral("/scheduler"), this, QDBusConnection::ExportScriptableContents); } FileIndexScheduler::~FileIndexScheduler() { m_threadPool.waitForDone(0); // wait 0 msecs } void FileIndexScheduler::scheduleIndexing() { if (m_threadPool.activeThreadCount() || m_indexerState == Suspended) { return; } if (m_config->isInitialRun()) { auto runnable = new FirstRunIndexer(m_db, m_config, m_config->includeFolders()); connect(runnable, &FirstRunIndexer::done, this, &FileIndexScheduler::scheduleIndexing); m_threadPool.start(runnable); m_indexerState = FirstRun; Q_EMIT stateChanged(m_indexerState); return; } if (!m_newFiles.isEmpty()) { auto runnable = new NewFileIndexer(m_db, m_config, m_newFiles); connect(runnable, &NewFileIndexer::done, this, &FileIndexScheduler::scheduleIndexing); m_threadPool.start(runnable); m_newFiles.clear(); m_indexerState = NewFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_modifiedFiles.isEmpty()) { auto runnable = new ModifiedFileIndexer(m_db, m_config, m_modifiedFiles); connect(runnable, &ModifiedFileIndexer::done, this, &FileIndexScheduler::scheduleIndexing); m_threadPool.start(runnable); m_modifiedFiles.clear(); m_indexerState = ModifiedFiles; Q_EMIT stateChanged(m_indexerState); return; } if (!m_xattrFiles.isEmpty()) { auto runnable = new XAttrIndexer(m_db, m_config, m_xattrFiles); connect(runnable, &XAttrIndexer::done, this, &FileIndexScheduler::scheduleIndexing); m_threadPool.start(runnable); m_xattrFiles.clear(); m_indexerState = XAttrFiles; Q_EMIT stateChanged(m_indexerState); return; } if (m_provider.size() && !m_powerMonitor.isOnBattery()) { m_threadPool.start(m_contentIndexer); m_indexerState = ContentIndexing; Q_EMIT stateChanged(m_indexerState); return; } if (m_checkUnindexedFiles) { auto runnable = new UnindexedFileIndexer(m_db, m_config); connect(runnable, &UnindexedFileIndexer::done, this, &FileIndexScheduler::scheduleIndexing); m_threadPool.start(runnable); m_checkUnindexedFiles = false; m_indexerState = UnindexedFileCheck; Q_EMIT stateChanged(m_indexerState); return; } m_indexerState = Idle; Q_EMIT stateChanged(m_indexerState); } static void removeStartsWith(QStringList& list, const QString& dir) { QMutableListIterator it(list); while (it.hasNext()) { const QString& file = it.next(); if (file.startsWith(dir)) { it.remove(); } } } void FileIndexScheduler::handleFileRemoved(const QString& file) { if (!file.endsWith('/')) { m_newFiles.removeOne(file); m_modifiedFiles.removeOne(file); m_xattrFiles.removeOne(file); } else { removeStartsWith(m_newFiles, file); removeStartsWith(m_modifiedFiles, file); removeStartsWith(m_xattrFiles, file); } } void FileIndexScheduler::powerManagementStatusChanged(bool isOnBattery) { qDebug() << "Power state changed"; if (isOnBattery && m_indexerState == ContentIndexing) { qDebug() << "On battery stopping content indexer"; m_contentIndexer->quit(); //TODO: Maybe we can add a special state for suspended due to being on battery. m_indexerState = Idle; - stateChanged(m_indexerState); + Q_EMIT stateChanged(m_indexerState); } else if (!isOnBattery) { scheduleIndexing(); } } void FileIndexScheduler::setSuspend(bool suspend) { if (suspend) { qDebug() << "Suspending"; if (m_indexerState == ContentIndexing) { m_contentIndexer->quit(); } m_indexerState = Suspended; Q_EMIT stateChanged(m_indexerState); } else { qDebug() << "Resuming"; m_indexerState = Idle; // No need to emit here we'll be emitting in scheduling scheduleIndexing(); } } uint FileIndexScheduler::getRemainingTime() { if (m_indexerState != ContentIndexing) { return 0; } return m_timeEstimator.calculateTimeLeft(m_provider.size()); } void FileIndexScheduler::checkUnindexedFiles() { m_checkUnindexedFiles = true; scheduleIndexing(); } uint FileIndexScheduler::getBatchSize() { return m_config->maxUncomittedFiles(); } diff --git a/src/file/firstrunindexer.cpp b/src/file/firstrunindexer.cpp index ffd7a359..eb9651a0 100644 --- a/src/file/firstrunindexer.cpp +++ b/src/file/firstrunindexer.cpp @@ -1,87 +1,87 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "firstrunindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "filtereddiriterator.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; FirstRunIndexer::FirstRunIndexer(Database* db, FileIndexerConfig* config, const QStringList& folders) : m_db(db) , m_config(config) , m_folders(folders) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_folders.isEmpty()); } void FirstRunIndexer::run() { Q_ASSERT(m_config->isInitialRun()); { Transaction tr(m_db, Transaction::ReadOnly); Q_ASSERT_X(tr.size() == 0, "FirstRunIndexer", "The database is not empty on first run"); } QMimeDatabase mimeDb; - for (const QString& folder : m_folders) { + for (const QString& folder : qAsConst(m_folders)) { Transaction tr(m_db, Transaction::ReadWrite); FilteredDirIterator it(m_config, folder); while (!it.next().isEmpty()) { QString mimetype = mimeDb.mimeTypeForFile(it.filePath(), QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(it.filePath(), mimetype, level); if (!job.index()) { continue; } // Even though this is the first run, because 2 hard links will resolve to the same id, // we land up crashing (due to the asserts in addDocument). // Hence we are checking before. // FIXME: Silently ignore hard links! // if (tr.hasDocument(job.document().id())) { continue; } tr.addDocument(job.document()); } // FIXME: This would consume too much memory. We should make some more commits // based on how much memory we consume tr.commit(); } m_config->setInitialRun(false); Q_EMIT done(); } diff --git a/src/file/indexcleaner.cpp b/src/file/indexcleaner.cpp index 5a30f512..20eabe15 100644 --- a/src/file/indexcleaner.cpp +++ b/src/file/indexcleaner.cpp @@ -1,81 +1,82 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "indexcleaner.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include "idutils.h" #include #include #include using namespace Baloo; IndexCleaner::IndexCleaner(Database* db, FileIndexerConfig* config) : m_db(db) , m_config(config) { Q_ASSERT(db); Q_ASSERT(config); } void IndexCleaner::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); auto shouldDelete = [&](quint64 id) { if (!id) { return false; } QString url = tr.documentUrl(id); if (!QFile::exists(url)) { qDebug() << "not exists: " << url; return true; } if (!m_config->shouldBeIndexed(url)) { qDebug() << "should not be indexed: " << url; return true; } // FIXME: This mimetype is not completely accurate! QString mimetype = mimeDb.mimeTypeForFile(url, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { qDebug() << "mimetype should not be indexed: " << url << mimetype; return true; } return false; }; - for (const QString& folder : m_config->includeFolders()) { + const auto includeFolders = m_config->includeFolders(); + for (const QString& folder : includeFolders) { quint64 id = filePathToId(QFile::encodeName(folder)); tr.removeRecursively(id, shouldDelete); } tr.commit(); Q_EMIT done(); } diff --git a/src/file/modifiedfileindexer.cpp b/src/file/modifiedfileindexer.cpp index d1f5011b..f7111177 100644 --- a/src/file/modifiedfileindexer.cpp +++ b/src/file/modifiedfileindexer.cpp @@ -1,105 +1,105 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "modifiedfileindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "idutils.h" #include "database.h" #include "transaction.h" #include #include #include #include using namespace Baloo; ModifiedFileIndexer::ModifiedFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& files) : m_db(db) , m_config(config) , m_files(files) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void ModifiedFileIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); - for (const QString& filePath : m_files) { + for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith('/')); QString fileName = filePath.mid(filePath.lastIndexOf('/') + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } quint64 fileId = filePathToId(QFile::encodeName(filePath)); if (!fileId) { continue; } quint32 mTime = tr.documentTimeInfo(fileId).mTime; // A folders mtime is updated when a new file is added / removed / renamed // we don't really need to reindex a folder when that happens // In fact, we never need to reindex a folder if (mTime && mimetype == QLatin1String("inode/directory")) { continue; } // FIXME: Using QFileInfo over here is quite expensive! QFileInfo fileInfo(filePath); if (mTime == fileInfo.lastModified().toTime_t()) { continue; } // FIXME: The BasicIndexingJob extracts too much info. We only need the time BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // we can get modified events for files which do not exist // cause Baloo was not running and missed those events if (tr.hasDocument(job.document().id())) { tr.replaceDocument(job.document(), DocumentTime); tr.setPhaseOne(job.document().id()); } else { tr.addDocument(job.document()); } } tr.commit(); Q_EMIT done(); } diff --git a/src/file/newfileindexer.cpp b/src/file/newfileindexer.cpp index 7e196229..8291d415 100644 --- a/src/file/newfileindexer.cpp +++ b/src/file/newfileindexer.cpp @@ -1,77 +1,77 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "newfileindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; NewFileIndexer::NewFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& newFiles) : m_db(db) , m_config(config) , m_files(newFiles) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void NewFileIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); - for (const QString& filePath : m_files) { + for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith('/')); QString fileName = filePath.mid(filePath.lastIndexOf('/') + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // The same file can be sent twice though it shouldn't be. // Lets just silently ignore it instead of crashing if (tr.hasDocument(job.document().id())) { continue; } tr.addDocument(job.document()); } tr.commit(); Q_EMIT done(); } diff --git a/src/file/xattrindexer.cpp b/src/file/xattrindexer.cpp index 3daecef6..1aaee422 100644 --- a/src/file/xattrindexer.cpp +++ b/src/file/xattrindexer.cpp @@ -1,82 +1,82 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "xattrindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; XAttrIndexer::XAttrIndexer(Database* db, FileIndexerConfig* config, const QStringList& files) : m_db(db) , m_config(config) , m_files(files) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void XAttrIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); - for (const QString& filePath : m_files) { + for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith('/')); QString fileName = filePath.mid(filePath.lastIndexOf('/') + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } // FIXME: The BasicIndexingJob extracts too much info. We only need the xattr BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // FIXME: This slightly defeats the point of having separate indexers // But we can get xattr changes of a file, even when it doesn't exist // cause we missed its creation somehow if (!tr.hasDocument(job.document().id())) { tr.addDocument(job.document()); continue; } // FIXME: Do we also need to update the ctime of the file? tr.replaceDocument(job.document(), XAttrTerms); } tr.commit(); Q_EMIT done(); } diff --git a/src/kioslaves/kded/baloosearchmodule.cpp b/src/kioslaves/kded/baloosearchmodule.cpp index dfeaddf0..9c723287 100644 --- a/src/kioslaves/kded/baloosearchmodule.cpp +++ b/src/kioslaves/kded/baloosearchmodule.cpp @@ -1,110 +1,110 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "baloosearchmodule.h" #include #include #include #include #include #include namespace { inline bool isSearchUrl(const QUrl& url) { return url.scheme() == QStringLiteral("baloosearch") || url.scheme() == QStringLiteral("timeline") || url.scheme() == QStringLiteral("tags"); } } using namespace Baloo; SearchModule::SearchModule(QObject* parent, const QList&) : KDEDModule(parent) , m_dirNotify(nullptr) { QTimer::singleShot(0, this, SLOT(init())); } void SearchModule::init() { m_dirNotify = new org::kde::KDirNotify(QString(), QString(), QDBusConnection::sessionBus(), this); connect(m_dirNotify, &OrgKdeKDirNotifyInterface::enteredDirectory, this, &SearchModule::registerSearchUrl); connect(m_dirNotify, &OrgKdeKDirNotifyInterface::leftDirectory, this, &SearchModule::unregisterSearchUrl); // FIXME: Listen to changes from Baloo!! // Listen to dbChanged QDBusConnection con = QDBusConnection::sessionBus(); con.connect(QString(), QStringLiteral("/files"), QStringLiteral("org.kde.baloo"), QStringLiteral("updated"), this, SLOT(slotBalooFileDbChanged())); con.connect(QString(), QStringLiteral("/files"), QStringLiteral("org.kde"), QStringLiteral("changed"), this, SLOT(slotFileMetaDataChanged(QStringList))); } void SearchModule::registerSearchUrl(const QString& urlString) { QUrl url(urlString); if (isSearchUrl(url)) { m_searchUrls << url; } } void SearchModule::unregisterSearchUrl(const QString& urlString) { QUrl url(urlString); m_searchUrls.removeAll(url); } void SearchModule::slotBalooFileDbChanged() { qDebug() << m_searchUrls; - for (const QUrl& dirUrl : m_searchUrls) { + for (const QUrl& dirUrl : qAsConst(m_searchUrls)) { org::kde::KDirNotify::emitFilesAdded(dirUrl); } } void SearchModule::slotFileMetaDataChanged(const QStringList& list) { qDebug() << m_searchUrls; qDebug() << list; QList localFileUrls; for (const QString& path : list) { localFileUrls << QUrl::fromLocalFile(path); } org::kde::KDirNotify::emitFilesChanged(localFileUrls); slotBalooFileDbChanged(); } K_PLUGIN_FACTORY_WITH_JSON(Factory, "baloosearchmodule.json", registerPlugin();) #include "baloosearchmodule.moc" diff --git a/src/lib/query.cpp b/src/lib/query.cpp index f373c59f..ee0ff190 100644 --- a/src/lib/query.cpp +++ b/src/lib/query.cpp @@ -1,352 +1,352 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "query.h" #include "term.h" #include "advancedqueryparser.h" #include "searchstore.h" #include #include #include #include #include #include #include using namespace Baloo; const int defaultLimit = -1; class Baloo::Query::Private { public: Private() { m_limit = defaultLimit; m_offset = 0; m_yearFilter = 0; m_monthFilter = 0; m_dayFilter = 0; m_sortingOption = SortAuto; } Term m_term; QStringList m_types; QString m_searchString; int m_limit; uint m_offset; int m_yearFilter; int m_monthFilter; int m_dayFilter; SortingOption m_sortingOption; QString m_includeFolder; }; Query::Query() : d(new Private) { } Query::Query(const Query& rhs) : d(new Private(*rhs.d)) { } Query::~Query() { delete d; } void Query::addType(const QString& type) { d->m_types << type.split(QLatin1Char('/'), QString::SkipEmptyParts); } void Query::addTypes(const QStringList& typeList) { Q_FOREACH (const QString& type, typeList) { addType(type); } } void Query::setType(const QString& type) { d->m_types.clear(); addType(type); } void Query::setTypes(const QStringList& types) { d->m_types = types; } QStringList Query::types() const { return d->m_types; } QString Query::searchString() const { return d->m_searchString; } void Query::setSearchString(const QString& str) { d->m_searchString = str; AdvancedQueryParser parser; d->m_term = parser.parse(str); } uint Query::limit() const { return d->m_limit; } void Query::setLimit(uint limit) { d->m_limit = limit; } uint Query::offset() const { return d->m_offset; } void Query::setOffset(uint offset) { d->m_offset = offset; } void Query::setDateFilter(int year, int month, int day) { d->m_yearFilter = year; d->m_monthFilter = month; d->m_dayFilter = day; } int Query::yearFilter() const { return d->m_yearFilter; } int Query::monthFilter() const { return d->m_monthFilter; } int Query::dayFilter() const { return d->m_dayFilter; } void Query::setSortingOption(Query::SortingOption option) { d->m_sortingOption = option; } Query::SortingOption Query::sortingOption() const { return d->m_sortingOption; } QString Query::includeFolder() const { return d->m_includeFolder; } void Query::setIncludeFolder(const QString& folder) { d->m_includeFolder = folder; } ResultIterator Query::exec() { Term term(d->m_term); if (!d->m_types.isEmpty()) { - for (const QString& type : d->m_types) { + for (const QString& type : qAsConst(d->m_types)) { term = term && Term(QStringLiteral("type"), type); } } if (!d->m_includeFolder.isEmpty()) { term = term && Term(QStringLiteral("includefolder"), d->m_includeFolder); } if (d->m_yearFilter || d->m_monthFilter || d->m_dayFilter) { QByteArray ba = QByteArray::number(d->m_yearFilter); if (d->m_monthFilter < 10) ba += '0'; ba += QByteArray::number(d->m_monthFilter); if (d->m_dayFilter < 10) ba += '0'; ba += QByteArray::number(d->m_dayFilter); term = term && Term(QStringLiteral("modified"), ba, Term::Equal); } SearchStore searchStore; QStringList result = searchStore.exec(term, d->m_offset, d->m_limit, d->m_sortingOption == SortAuto); return ResultIterator(result); } QByteArray Query::toJSON() { QVariantMap map; if (!d->m_types.isEmpty()) map[QStringLiteral("type")] = d->m_types; if (d->m_limit != defaultLimit) map[QStringLiteral("limit")] = d->m_limit; if (d->m_offset) map[QStringLiteral("offset")] = d->m_offset; if (!d->m_searchString.isEmpty()) map[QStringLiteral("searchString")] = d->m_searchString; if (d->m_term.isValid()) map[QStringLiteral("term")] = QVariant(d->m_term.toVariantMap()); if (d->m_yearFilter >= 0) map[QStringLiteral("yearFilter")] = d->m_yearFilter; if (d->m_monthFilter >= 0) map[QStringLiteral("monthFilter")] = d->m_monthFilter; if (d->m_dayFilter >= 0) map[QStringLiteral("dayFilter")] = d->m_dayFilter; if (d->m_sortingOption != SortAuto) map[QStringLiteral("sortingOption")] = static_cast(d->m_sortingOption); if (!d->m_includeFolder.isEmpty()) map[QStringLiteral("includeFolder")] = d->m_includeFolder; QJsonObject jo = QJsonObject::fromVariantMap(map); QJsonDocument jdoc; jdoc.setObject(jo); return jdoc.toJson(); } // static Query Query::fromJSON(const QByteArray& arr) { QJsonDocument jdoc = QJsonDocument::fromJson(arr); const QVariantMap map = jdoc.object().toVariantMap(); Query query; query.d->m_types = map[QStringLiteral("type")].toStringList(); if (map.contains(QStringLiteral("limit"))) query.d->m_limit = map[QStringLiteral("limit")].toUInt(); else query.d->m_limit = defaultLimit; query.d->m_offset = map[QStringLiteral("offset")].toUInt(); query.d->m_searchString = map[QStringLiteral("searchString")].toString(); query.d->m_term = Term::fromVariantMap(map[QStringLiteral("term")].toMap()); if (map.contains(QStringLiteral("yearFilter"))) query.d->m_yearFilter = map[QStringLiteral("yearFilter")].toInt(); if (map.contains(QStringLiteral("monthFilter"))) query.d->m_monthFilter = map[QStringLiteral("monthFilter")].toInt(); if (map.contains(QStringLiteral("dayFilter"))) query.d->m_dayFilter = map[QStringLiteral("dayFilter")].toInt(); if (map.contains(QStringLiteral("sortingOption"))) { int option = map.value(QStringLiteral("sortingOption")).toInt(); query.d->m_sortingOption = static_cast(option); } if (map.contains(QStringLiteral("includeFolder"))) { query.d->m_includeFolder = map.value(QStringLiteral("includeFolder")).toString(); } return query; } QUrl Query::toSearchUrl(const QString& title) { QUrl url; url.setScheme(QStringLiteral("baloosearch")); QUrlQuery urlQuery; urlQuery.addQueryItem(QStringLiteral("json"), QString::fromUtf8(toJSON())); if (title.size()) urlQuery.addQueryItem(QStringLiteral("title"), title); url.setQuery(urlQuery); return url; } Query Query::fromSearchUrl(const QUrl& url) { if (url.scheme() != QLatin1String("baloosearch")) return Query(); QUrlQuery urlQuery(url); QString jsonString = urlQuery.queryItemValue(QStringLiteral("json"), QUrl::FullyDecoded); return Query::fromJSON(jsonString.toUtf8()); } QString Query::titleFromQueryUrl(const QUrl& url) { QUrlQuery urlQuery(url); return urlQuery.queryItemValue(QStringLiteral("title"), QUrl::FullyDecoded); } bool Query::operator==(const Query& rhs) const { if (rhs.d->m_limit != d->m_limit || rhs.d->m_offset != d->m_offset || rhs.d->m_dayFilter != d->m_dayFilter || rhs.d->m_monthFilter != d->m_monthFilter || rhs.d->m_yearFilter != d->m_yearFilter || rhs.d->m_includeFolder != d->m_includeFolder || rhs.d->m_searchString != d->m_searchString || rhs.d->m_sortingOption != d->m_sortingOption) { return false; } if (rhs.d->m_types.size() != d->m_types.size()) return false; Q_FOREACH (const QString& type, rhs.d->m_types) { if (!d->m_types.contains(type)) return false; } return d->m_term == rhs.d->m_term; } bool Query::operator!=(const Query& rhs) const { return !(*this == rhs); } Query& Query::operator=(const Query& rhs) { *d = *rhs.d; return *this; } diff --git a/src/lib/searchstore.cpp b/src/lib/searchstore.cpp index e391c50f..82d514f7 100644 --- a/src/lib/searchstore.cpp +++ b/src/lib/searchstore.cpp @@ -1,406 +1,406 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013-2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "searchstore.h" #include "term.h" #include "global.h" #include "database.h" #include "transaction.h" #include "enginequery.h" #include "queryparser.h" #include "termgenerator.h" #include "andpostingiterator.h" #include "orpostingiterator.h" #include "idutils.h" #include #include #include #include #include #include #include #include using namespace Baloo; SearchStore::SearchStore() : m_db(nullptr) { m_db = globalDatabaseInstance(); if (!m_db->open(Database::ReadOnlyDatabase)) { m_db = nullptr; } m_prefixes.insert(QByteArray("filename"), QByteArray("F")); m_prefixes.insert(QByteArray("mimetype"), QByteArray("M")); m_prefixes.insert(QByteArray("rating"), QByteArray("R")); m_prefixes.insert(QByteArray("tag"), QByteArray("TAG-")); m_prefixes.insert(QByteArray("tags"), QByteArray("TA")); m_prefixes.insert(QByteArray("usercomment"), QByteArray("C")); } SearchStore::~SearchStore() { } // Return the result with-in [offset, offset + limit) QStringList SearchStore::exec(const Term& term, uint offset, int limit, bool sortResults) { if (!m_db || !m_db->isOpen()) { return QStringList(); } Transaction tr(m_db, Transaction::ReadOnly); QScopedPointer it(constructQuery(&tr, term)); if (!it) { return QStringList(); } if (sortResults) { QVector> resultIds; while (it->next()) { quint64 id = it->docId(); quint32 mtime = tr.documentTimeInfo(id).mTime; resultIds << std::pair{id, mtime}; Q_ASSERT(id > 0); } // Not enough results within range, no need to sort. if (offset >= static_cast(resultIds.size())) { return QStringList(); } auto compFunc = [](const std::pair& lhs, const std::pair& rhs) { return lhs.second > rhs.second; }; std::sort(resultIds.begin(), resultIds.end(), compFunc); if (limit < 0) { limit = resultIds.size(); } QStringList results; const uint end = qMin(static_cast(resultIds.size()), offset + static_cast(limit)); for (uint i = offset; i < end; i++) { const quint64 id = resultIds[i].first; const QString filePath = tr.documentUrl(id); results << filePath; } return results; } else { QStringList results; uint ulimit = limit < 0 ? UINT_MAX : limit; while (offset && it->next()) { offset--; } while (ulimit && it->next()) { quint64 id = it->docId(); Q_ASSERT(id > 0); results << tr.documentUrl(it->docId()); Q_ASSERT(!results.last().isEmpty()); ulimit--; } return results; } } QByteArray SearchStore::fetchPrefix(const QByteArray& property) const { auto it = m_prefixes.constFind(property.toLower()); if (it != m_prefixes.constEnd()) { return it.value(); } else { KFileMetaData::PropertyInfo pi = KFileMetaData::PropertyInfo::fromName(property); if (pi.property() == KFileMetaData::Property::Empty) { qDebug() << "Property" << property << "not found"; return QByteArray(); } int propPrefix = static_cast(pi.property()); return 'X' + QByteArray::number(propPrefix) + '-'; } } PostingIterator* SearchStore::constructQuery(Transaction* tr, const Term& term) { Q_ASSERT(tr); if (term.operation() == Term::And || term.operation() == Term::Or) { - QList subTerms = term.subTerms(); + const QList subTerms = term.subTerms(); QVector vec; vec.reserve(subTerms.size()); - for (const Term& t : term.subTerms()) { + for (const Term& t : subTerms) { vec << constructQuery(tr, t); } if (vec.isEmpty()) { return nullptr; } if (term.operation() == Term::And) { return new AndPostingIterator(vec); } else { return new OrPostingIterator(vec); } } if (term.value().isNull()) { return nullptr; } Q_ASSERT(term.value().isValid()); Q_ASSERT(term.comparator() != Term::Auto); Q_ASSERT(term.comparator() == Term::Contains ? term.value().type() == QVariant::String : true); const QVariant value = term.value(); const QByteArray property = term.property().toLower().toUtf8(); if (property == "type" || property == "kind") { EngineQuery q = constructTypeQuery(value.toString()); return tr->postingIterator(q); } else if (property == "includefolder") { const QByteArray folder = QFile::encodeName(QFileInfo(value.toString()).canonicalFilePath()); Q_ASSERT(!folder.isEmpty()); Q_ASSERT(folder.startsWith('/')); quint64 id = filePathToId(folder); if (!id) { qDebug() << "Folder" << value.toString() << "does not exist"; return nullptr; } return tr->docUrlIter(id); } else if (property == "modified" || property == "mtime") { if (value.type() == QVariant::ByteArray) { QByteArray ba = value.toByteArray(); Q_ASSERT(ba.size() >= 4); int year = ba.mid(0, 4).toInt(); int month = ba.mid(4, 2).toInt(); int day = ba.mid(6, 2).toInt(); Q_ASSERT(year); // uses 0 to represent whole month or whole year month = month >= 0 && month <= 12 ? month : 0; day = day >= 0 && day <= 31 ? day : 0; QDate startDate(year, month ? month : 1, day ? day : 1); QDate endDate(startDate); if (month == 0) { endDate.setDate(endDate.year(), 12, 31); } else if (day == 0) { endDate.setDate(endDate.year(), endDate.month(), endDate.daysInMonth()); } return tr->mTimeRangeIter(QDateTime(startDate).toTime_t(), QDateTime(endDate, QTime(23, 59, 59)).toTime_t()); } else if (value.type() == QVariant::Date || value.type() == QVariant::DateTime) { const QDateTime dt = value.toDateTime(); return constructMTimeQuery(tr, dt, term.comparator()); } else { Q_ASSERT_X(0, "SearchStore::constructQuery", "modified property must contain date/datetime values"); } } else if (property == "rating") { bool okay = false; int rating = value.toInt(&okay); if (!okay) { qDebug() << "Rating comparisons must be with an integer"; return nullptr; } PostingDB::Comparator pcom; if (term.comparator() == Term::Greater || term.comparator() == Term::GreaterEqual) { pcom = PostingDB::GreaterEqual; if (term.comparator() == Term::Greater && rating) rating++; } else if (term.comparator() == Term::Less || term.comparator() == Term::LessEqual) { pcom = PostingDB::LessEqual; if (term.comparator() == Term::Less) rating--; } else if (term.comparator() == Term::Equal) { EngineQuery q = constructEqualsQuery("R", value.toString()); return tr->postingIterator(q); } else { Q_ASSERT(0); return nullptr; } const QByteArray prefix = "R"; const QByteArray val = QByteArray::number(rating); return tr->postingCompIterator(prefix, val, pcom); } else if (property == "tag") { if (term.comparator() == Term::Equal) { const QByteArray prefix = "TAG-"; EngineQuery q = EngineQuery(prefix + value.toByteArray()); return tr->postingIterator(q); } else if (term.comparator() == Term::Contains) { const QByteArray prefix = "TA"; EngineQuery q = constructEqualsQuery(prefix, value.toString()); return tr->postingIterator(q); } else { Q_ASSERT(0); return nullptr; } } QByteArray prefix; if (!property.isEmpty()) { prefix = fetchPrefix(property); if (prefix.isEmpty()) { return nullptr; } } auto com = term.comparator(); if (com == Term::Contains) { EngineQuery q = constructContainsQuery(prefix, value.toString()); return tr->postingIterator(q); } if (com == Term::Equal) { EngineQuery q = constructEqualsQuery(prefix, value.toString()); return tr->postingIterator(q); } QVariant val = term.value(); if (val.type() == QVariant::Int) { int intVal = value.toInt(); PostingDB::Comparator pcom; if (term.comparator() == Term::Greater || term.comparator() == Term::GreaterEqual) { pcom = PostingDB::GreaterEqual; if (term.comparator() == Term::Greater && intVal) intVal++; } else if (term.comparator() == Term::Less || term.comparator() == Term::LessEqual) { pcom = PostingDB::LessEqual; if (term.comparator() == Term::Less) intVal--; } else { Q_ASSERT(0); return nullptr; } return tr->postingCompIterator(prefix, QByteArray::number(intVal), pcom); } return nullptr; } EngineQuery SearchStore::constructContainsQuery(const QByteArray& prefix, const QString& value) { QueryParser parser; return parser.parseQuery(value, prefix); } EngineQuery SearchStore::constructEqualsQuery(const QByteArray& prefix, const QString& value) { // We use the TermGenerator to normalize the words in the value and to // split it into other words. If we split the words, we then add them as a // phrase query. QStringList terms = TermGenerator::termList(value); QVector queries; int position = 1; for (const QString& term : terms) { QByteArray arr = prefix + term.toUtf8(); queries << EngineQuery(arr, position++); } if (queries.isEmpty()) { return EngineQuery(); } else if (queries.size() == 1) { return queries.first(); } else { return EngineQuery(queries, EngineQuery::Phrase); } } EngineQuery SearchStore::constructTypeQuery(const QString& value) { Q_ASSERT(!value.isEmpty()); KFileMetaData::TypeInfo ti = KFileMetaData::TypeInfo::fromName(value); if (ti == KFileMetaData::Type::Empty) { qDebug() << "Type" << value << "does not exist"; return EngineQuery(); } int num = static_cast(ti.type()); return EngineQuery('T' + QByteArray::number(num)); } PostingIterator* SearchStore::constructMTimeQuery(Transaction* tr, const QDateTime& dt, Term::Comparator com) { Q_ASSERT(dt.isValid()); quint32 timet = dt.toTime_t(); MTimeDB::Comparator mtimeCom; if (com == Term::Equal) { mtimeCom = MTimeDB::Equal; quint32 end = QDateTime(dt.date().addDays(1)).toTime_t() - 1; return tr->mTimeRangeIter(timet, end); } else if (com == Term::GreaterEqual) { mtimeCom = MTimeDB::GreaterEqual; } else if (com == Term::Greater) { timet++; mtimeCom = MTimeDB::GreaterEqual; } else if (com == Term::LessEqual) { mtimeCom = MTimeDB::LessEqual; } else if (com == Term::Less) { mtimeCom = MTimeDB::LessEqual; timet--; } else { Q_ASSERT_X(0, "SearchStore::constructQuery", "mtime query must contain a valid comparator"); return nullptr; } return tr->mTimeIter(timet, mtimeCom); } diff --git a/src/lib/taglistjob.cpp b/src/lib/taglistjob.cpp index 3d1b0c81..74938191 100644 --- a/src/lib/taglistjob.cpp +++ b/src/lib/taglistjob.cpp @@ -1,72 +1,72 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2014-2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "taglistjob.h" #include "global.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; class TagListJob::Private { public: QStringList tags; }; TagListJob::TagListJob(QObject* parent) : KJob(parent) , d(new Private) { } TagListJob::~TagListJob() { delete d; } void TagListJob::start() { Database *db = globalDatabaseInstance(); if (!db->open(Database::ReadOnlyDatabase)) { setError(UserDefinedError); setErrorText(QStringLiteral("Failed to open the database")); emitResult(); return; } QVector tagList; { Transaction tr(db, Transaction::ReadOnly); tagList = tr.fetchTermsStartingWith("TAG-"); } d->tags.reserve(tagList.size()); - for (const QByteArray& ba : tagList) { + for (const QByteArray& ba : qAsConst(tagList)) { d->tags << QString::fromUtf8(ba.mid(4)); } emitResult(); } QStringList TagListJob::tags() { return d->tags; } diff --git a/src/lib/term.cpp b/src/lib/term.cpp index 43822782..d34fe224 100644 --- a/src/lib/term.cpp +++ b/src/lib/term.cpp @@ -1,469 +1,470 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "term.h" #include #include using namespace Baloo; class Baloo::Term::Private { public: Operation m_op; Comparator m_comp; QString m_property; QVariant m_value; bool m_isNegated; QList m_subTerms; QVariantHash m_userData; Private() { m_op = None; m_comp = Auto; m_isNegated = false; } }; Term::Term() : d(new Private) { } Term::Term(const Term& t) : d(new Private(*t.d)) { } Term::Term(const QString& property) : d(new Private) { d->m_property = property; } Term::Term(const QString& property, const QVariant& value, Term::Comparator c) : d(new Private) { d->m_property = property; d->m_value = value; if (c == Auto) { if (value.type() == QVariant::String) d->m_comp = Contains; else if (value.type() == QVariant::DateTime) d->m_comp = Contains; else d->m_comp = Equal; } else { d->m_comp = c; } } /* Term::Term(const QString& property, const QVariant& start, const QVariant& end) : d(new Private) { d->m_property = property; d->m_op = Range; // FIXME: How to save range queries? } */ Term::Term(Term::Operation op) : d(new Private) { d->m_op = op; } Term::Term(Term::Operation op, const Term& t) : d(new Private) { d->m_op = op; d->m_subTerms << t; } Term::Term(Term::Operation op, const QList& t) : d(new Private) { d->m_op = op; d->m_subTerms = t; } Term::Term(const Term& lhs, Term::Operation op, const Term& rhs) : d(new Private) { d->m_op = op; if (lhs.operation() == op) { d->m_subTerms << lhs.subTerms(); } else { d->m_subTerms << lhs; } if (rhs.operation() == op) { d->m_subTerms << rhs.subTerms(); } else { d->m_subTerms << rhs; } } Term::~Term() { delete d; } bool Term::isValid() const { // Terms with an operator but no subterms are still valid if (d->m_op != Term::None) { return true; } if (d->m_comp == Term::Auto) { return false; } return true; } void Term::setNegation(bool isNegated) { d->m_isNegated = isNegated; } bool Term::isNegated() const { return d->m_isNegated; } bool Term::negated() const { return d->m_isNegated; } void Term::addSubTerm(const Term& term) { d->m_subTerms << term; } void Term::setSubTerms(const QList& terms) { d->m_subTerms = terms; } Term Term::subTerm() const { if (d->m_subTerms.size()) return d->m_subTerms.first(); return Term(); } QList Term::subTerms() const { return d->m_subTerms; } void Term::setOperation(Term::Operation op) { d->m_op = op; } Term::Operation Term::operation() const { return d->m_op; } bool Term::empty() const { return isEmpty(); } bool Term::isEmpty() const { return d->m_property.isEmpty() && d->m_value.isNull() && d->m_subTerms.isEmpty(); } QString Term::property() const { return d->m_property; } void Term::setProperty(const QString& property) { d->m_property = property; } void Term::setValue(const QVariant& value) { d->m_value = value; } QVariant Term::value() const { return d->m_value; } Term::Comparator Term::comparator() const { return d->m_comp; } void Term::setComparator(Term::Comparator c) { d->m_comp = c; } void Term::setUserData(const QString& name, const QVariant& value) { d->m_userData.insert(name, value); } QVariant Term::userData(const QString& name) const { return d->m_userData.value(name); } QVariantMap Term::toVariantMap() const { QVariantMap map; if (d->m_op != None) { QVariantList variantList; Q_FOREACH (const Term& term, d->m_subTerms) { variantList << QVariant(term.toVariantMap()); } if (d->m_op == And) map[QStringLiteral("$and")] = variantList; else map[QStringLiteral("$or")] = variantList; return map; } QString op; switch (d->m_comp) { case Equal: map[d->m_property] = d->m_value; return map; case Contains: op = QStringLiteral("$ct"); break; case Greater: op = QStringLiteral("$gt"); break; case GreaterEqual: op = QStringLiteral("$gte"); break; case Less: op = QStringLiteral("$lt"); break; case LessEqual: op = QStringLiteral("$lte"); break; case Auto: Q_ASSERT(0); } QVariantMap m; m[op] = d->m_value; map[d->m_property] = QVariant(m); return map; } namespace { // QJson does not recognize QDate/QDateTime parameters. We try to guess // and see if they can be converted into date/datetime. QVariant tryConvert(const QVariant& var) { if (var.canConvert(QVariant::DateTime)) { QDateTime dt = var.toDateTime(); if (!dt.isValid()) return var; if (!var.toString().contains(QLatin1String("T"))) { return QVariant(var.toDate()); } return dt; } return var; } } Term Term::fromVariantMap(const QVariantMap& map) { if (map.size() != 1) return Term(); Term term; QString andOrString; if (map.contains(QStringLiteral("$and"))) { andOrString = QStringLiteral("$and"); term.setOperation(And); } else if (map.contains(QStringLiteral("$or"))) { andOrString = QStringLiteral("$or"); term.setOperation(Or); } if (andOrString.size()) { QList subTerms; QVariantList list = map[andOrString].toList(); Q_FOREACH (const QVariant& var, list) subTerms << Term::fromVariantMap(var.toMap()); term.setSubTerms(subTerms); return term; } QString prop = map.cbegin().key(); term.setProperty(prop); QVariant value = map.value(prop); if (value.type() == QVariant::Map) { QVariantMap mapVal = value.toMap(); if (mapVal.size() != 1) return term; QString op = mapVal.cbegin().key(); Term::Comparator com; if (op == QLatin1String("$ct")) com = Contains; else if (op == QLatin1String("$gt")) com = Greater; else if (op == QLatin1String("$gte")) com = GreaterEqual; else if (op == QLatin1String("$lt")) com = Less; else if (op == QLatin1String("$lte")) com = LessEqual; else return term; term.setComparator(com); term.setValue(tryConvert(mapVal.value(op))); return term; } term.setComparator(Equal); term.setValue(tryConvert(value)); return term; } bool Term::operator==(const Term& rhs) const { if (d->m_op != rhs.d->m_op || d->m_comp != rhs.d->m_comp || d->m_isNegated != rhs.d->m_isNegated || d->m_property != rhs.d->m_property || d->m_value != rhs.d->m_value) { return false; } if (d->m_subTerms.size() != rhs.d->m_subTerms.size()) return false; if (d->m_subTerms.isEmpty()) return true; Q_FOREACH (const Term& t, d->m_subTerms) { if (!rhs.d->m_subTerms.contains(t)) return false; } return true; } Term& Term::operator=(const Term& rhs) { *d = *rhs.d; return *this; } namespace { QString comparatorToString(Baloo::Term::Comparator c) { switch (c) { case Baloo::Term::Auto: return QStringLiteral("Auto"); case Baloo::Term::Equal: return QStringLiteral("="); case Baloo::Term::Contains: return QStringLiteral(":"); case Baloo::Term::Less: return QStringLiteral("<"); case Baloo::Term::LessEqual: return QStringLiteral("<="); case Baloo::Term::Greater: return QStringLiteral(">"); case Baloo::Term::GreaterEqual: return QStringLiteral(">="); } return QString(); } QString operationToString(Baloo::Term::Operation op) { switch (op) { case Baloo::Term::None: return QStringLiteral("NONE"); case Baloo::Term::And: return QStringLiteral("AND"); case Baloo::Term::Or: return QStringLiteral("OR"); } return QString(); } } QDebug operator <<(QDebug d, const Baloo::Term& t) { if (t.subTerms().isEmpty()) { d << QStringLiteral("(%1 %2 %3 (%4))").arg(t.property(), comparatorToString(t.comparator()), t.value().toString(), t.value().typeName()).toUtf8().constData(); } else { d << "[" << operationToString(t.operation()).toUtf8().constData(); - for (const Term& term : t.subTerms()) { + const auto subTerms = t.subTerms(); + for (const Term& term : subTerms) { d << term; } d << "]"; } return d; }