diff --git a/src/file/fileindexerconfig.cpp b/src/file/fileindexerconfig.cpp index f1eeb1e8..414a4577 100644 --- a/src/file/fileindexerconfig.cpp +++ b/src/file/fileindexerconfig.cpp @@ -1,399 +1,399 @@ /* This file is part of the KDE Project Copyright (c) 2008-2010 Sebastian Trueg Copyright (c) 2013-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "fileindexerconfig.h" #include "fileexcludefilters.h" #include "storagedevices.h" #include #include #include #include namespace { /// recursively check if a folder is hidden bool isDirHidden(QDir& dir) { #ifdef __unix__ return dir.absolutePath().contains(QLatin1String("/.")); #else if (QFileInfo(dir.path()).isHidden()) return true; else if (dir.cdUp()) return isDirHidden(dir); else return false; #endif } QString stripTrailingSlash(const QString& path) { return path.endsWith(QLatin1Char('/')) ? path.mid(0, path.length()-1) : path; } } using namespace Baloo; FileIndexerConfig::FileIndexerConfig(QObject* parent) : QObject(parent) , m_config(QStringLiteral("baloofilerc")) , m_folderCacheDirty(true) , m_indexHidden(false) , m_devices(nullptr) , m_maxUncomittedFiles(40) { forceConfigUpdate(); } FileIndexerConfig::~FileIndexerConfig() { } QStringList FileIndexerConfig::includeFolders() const { const_cast(this)->buildFolderCache(); QStringList fl; for (int i = 0; i < m_folderCache.count(); ++i) { if (m_folderCache[i].second) fl << m_folderCache[i].first; } return fl; } QStringList FileIndexerConfig::excludeFolders() const { const_cast(this)->buildFolderCache(); QStringList fl; for (int i = 0; i < m_folderCache.count(); ++i) { if (!m_folderCache[i].second) fl << m_folderCache[i].first; } return fl; } QStringList FileIndexerConfig::excludeFilters() const { KConfigGroup cfg = m_config.group("General"); // read configured exclude filters QSet filters = cfg.readEntry("exclude filters", defaultExcludeFilterList()).toSet(); // make sure we always keep the latest default exclude filters // TODO: there is one problem here. What if the user removed some of the default filters? if (cfg.readEntry("exclude filters version", 0) < defaultExcludeFilterListVersion()) { filters += defaultExcludeFilterList().toSet(); // write the config directly since the KCM does not have support for the version yet // TODO: make this class public and use it in the KCM KConfig config(m_config.name()); KConfigGroup cfg = config.group("General"); cfg.writeEntry("exclude filters", QStringList::fromSet(filters)); cfg.writeEntry("exclude filters version", defaultExcludeFilterListVersion()); } // remove duplicates return QStringList::fromSet(filters); } QStringList FileIndexerConfig::excludeMimetypes() const { return QStringList::fromSet(m_excludeMimetypes); } bool FileIndexerConfig::indexHiddenFilesAndFolders() const { return m_indexHidden; } bool FileIndexerConfig::onlyBasicIndexing() const { return m_onlyBasicIndexing; } bool FileIndexerConfig::isInitialRun() const { return m_config.group("General").readEntry("first run", true); } bool FileIndexerConfig::canBeSearched(const QString& folder) const { QFileInfo fi(folder); QString path = fi.absolutePath(); if (!fi.isDir()) { return false; } else if (shouldFolderBeIndexed(path)) { return true; } const_cast(this)->buildFolderCache(); // Look for included descendants for (const QPair& fld: m_folderCache) { if (fld.second && fld.first.startsWith(path)) { return true; } } return false; } bool FileIndexerConfig::shouldBeIndexed(const QString& path) const { QFileInfo fi(path); if (fi.isDir()) { return shouldFolderBeIndexed(path); } else { return (shouldFolderBeIndexed(fi.absolutePath()) && (!fi.isHidden() || indexHiddenFilesAndFolders()) && shouldFileBeIndexed(fi.fileName())); } } bool FileIndexerConfig::shouldFolderBeIndexed(const QString& path) const { QString folder; if (folderInFolderList(path, folder)) { // we always index the folders in the list // ignoring the name filters if (folder == path) return true; // check for hidden folders QDir dir(path); if (!indexHiddenFilesAndFolders() && isDirHidden(dir)) return false; // reset dir, cause isDirHidden modifies the QDir dir = path; // check the exclude filters for all components of the path // after folder const QStringList pathComponents = path.mid(folder.count()).split(QLatin1Char('/'), QString::SkipEmptyParts); Q_FOREACH (const QString& c, pathComponents) { if (!shouldFileBeIndexed(c)) { return false; } } return true; } else { return false; } } bool FileIndexerConfig::shouldFileBeIndexed(const QString& fileName) const { if (!indexHiddenFilesAndFolders() && fileName.startsWith(QLatin1Char('.'))) { return false; } return !m_excludeFilterRegExpCache.exactMatch(fileName); } bool FileIndexerConfig::shouldMimeTypeBeIndexed(const QString& mimeType) const { return !m_excludeMimetypes.contains(mimeType); } bool FileIndexerConfig::folderInFolderList(const QString& path) { QString str; return folderInFolderList(path, str); } bool FileIndexerConfig::folderInFolderList(const QString& path, QString& folder) const { const_cast(this)->buildFolderCache(); const QString p = stripTrailingSlash(path); // we traverse the list backwards to catch all exclude folders int i = m_folderCache.count(); while (--i >= 0) { const QString& f = m_folderCache[i].first; const bool include = m_folderCache[i].second; if (p.startsWith(f)) { folder = f; return include; } } // path is not in the list, thus it should not be included folder.clear(); return false; } namespace { /** * Returns true if the specified folder f would already be excluded using the list * folders. */ bool alreadyExcluded(const QList >& folders, const QString& f) { bool included = false; for (int i = 0; i < folders.count(); ++i) { QString path = folders[i].first; if (!path.endsWith(QLatin1Char('/'))) path.append(QLatin1Char('/')); if (f != folders[i].first && f.startsWith(path)) { included = folders[i].second; } } return !included; } /** * Simple insertion sort */ void insertSortFolders(const QStringList& folders, bool include, QList >& result) { Q_FOREACH (const QString& f, folders) { int pos = 0; QString path = stripTrailingSlash(f); while (result.count() > pos && result[pos].first < path) ++pos; result.insert(pos, qMakePair(path, include)); } } /** * Remove useless exclude entries which would confuse the folderInFolderList algo. * This makes sure all top-level items are include folders. * This runs in O(n^2) and could be optimized but what for. */ void cleanupList(QList >& result) { int i = 0; while (i < result.count()) { if (result[i].first.isEmpty() || (!result[i].second && alreadyExcluded(result, result[i].first))) result.removeAt(i); else ++i; } } } void FileIndexerConfig::buildFolderCache() { if (!m_folderCacheDirty) { return; } if (!m_devices) { m_devices = new StorageDevices(this); } KConfigGroup group = m_config.group("General"); QStringList includeFoldersPlain = group.readPathEntry("folders", QStringList() << QDir::homePath()); QStringList excludeFoldersPlain = group.readPathEntry("exclude folders", QStringList()); // Add all removable media and network shares as ignored unless they have // been explicitly added in the include list const auto allMedia = m_devices->allMedia(); for (const auto& device: allMedia) { const QString mountPath = device.mountPath(); if (!device.isUsable() && !mountPath.isEmpty()) { if (!includeFoldersPlain.contains(mountPath)) { excludeFoldersPlain << mountPath; } } } m_folderCache.clear(); insertSortFolders(includeFoldersPlain, true, m_folderCache); insertSortFolders(excludeFoldersPlain, false, m_folderCache); cleanupList(m_folderCache); m_folderCacheDirty = false; } void FileIndexerConfig::buildExcludeFilterRegExpCache() { QStringList newFilters = excludeFilters(); m_excludeFilterRegExpCache.rebuildCacheFromFilterList(newFilters); } void FileIndexerConfig::buildMimeTypeCache() { m_excludeMimetypes = m_config.group("General").readPathEntry("exclude mimetypes", defaultExcludeMimetypes()).toSet(); } void FileIndexerConfig::forceConfigUpdate() { m_config.reparseConfiguration(); m_folderCacheDirty = true; buildExcludeFilterRegExpCache(); buildMimeTypeCache(); m_indexHidden = m_config.group("General").readEntry("index hidden folders", false); m_onlyBasicIndexing = m_config.group("General").readEntry("only basic indexing", false); } void FileIndexerConfig::setInitialRun(bool isInitialRun) { m_config.group("General").writeEntry("first run", isInitialRun); m_config.sync(); } bool FileIndexerConfig::initialUpdateDisabled() const { return m_config.group("General").readEntry("disable initial update", true); } int FileIndexerConfig::databaseVersion() const { return m_config.group("General").readEntry("dbVersion", 0); } void FileIndexerConfig::setDatabaseVersion(int version) { m_config.group("General").writeEntry("dbVersion", version); m_config.sync(); } bool FileIndexerConfig::indexingEnabled() const { return m_config.group("Basic Settings").readEntry("Indexing-Enabled", true); } -uint FileIndexerConfig::maxUncomittedFiles() +uint FileIndexerConfig::maxUncomittedFiles() const { return m_maxUncomittedFiles; } diff --git a/src/file/fileindexerconfig.h b/src/file/fileindexerconfig.h index 8c7fe36f..d7c5e5c5 100644 --- a/src/file/fileindexerconfig.h +++ b/src/file/fileindexerconfig.h @@ -1,213 +1,213 @@ /* Copyright (c) 2008-2009 Sebastian Trueg Copyright (c) 2012-2014 Vishesh Handa This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License version 2 as published by the Free Software Foundation. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef BALOO_FILEINDEXER_CONFIG_H_ #define BALOO_FILEINDEXER_CONFIG_H_ #include #include #include #include #include "regexpcache.h" namespace Baloo { class StorageDevices; /** * Active config class which emits signals if the config * was changed, for example if the KCM saved the config file. */ class FileIndexerConfig : public QObject { Q_OBJECT public: explicit FileIndexerConfig(QObject* parent = nullptr); ~FileIndexerConfig(); /** * Folders to search for files to index and analyze. * \return list of paths. */ QStringList includeFolders() const; /** * Folders that are excluded from indexing. * (Descendant folders of an excluded folder can be added * and they will be indexed.) * \return list of paths. */ QStringList excludeFolders() const; QStringList excludeFilters() const; QStringList excludeMimetypes() const; bool indexHiddenFilesAndFolders() const; bool onlyBasicIndexing() const; /** * \return \c true if the service is run for the first time * (or after manually setting "first run=true" in the config). */ bool isInitialRun() const; /** * A "hidden" config option which allows to disable the initial * update of all indexed folders. * * This should be used in combination with isInitialRun() to make * sure all folders are at least indexed once. */ bool initialUpdateDisabled() const; /** * Check if \p folder can be searched. * \p folder can be searched if itself or one of its descendants is indexed. * * Example: * if ~/foo is not indexed and ~/foo/bar is indexed * then ~/foo can be searched. * * \return \c true if the \p folder can be searched. */ bool canBeSearched(const QString& folder) const; /** * Check if file or folder \p path should be indexed taking into account * the includeFolders(), the excludeFolders(), and the excludeFilters(). * Inclusion takes precedence. * * Be aware that this method does not check if parent dirs * match any of the exclude filters. Only the path of the * dir itself it checked. * * \return \c true if the file or folder at \p path should * be indexed according to the configuration. */ bool shouldBeIndexed(const QString& path) const; /** * Check if the folder at \p path should be indexed. * * Be aware that this method does not check if parent dirs * match any of the exclude filters. Only the name of the * dir itself it checked. * * \return \c true if the folder at \p path should * be indexed according to the configuration. */ bool shouldFolderBeIndexed(const QString& path) const; /** * Check \p fileName for all exclude filters. This does * not take file paths into account. * * \return \c true if a file with name \p filename should * be indexed according to the configuration. */ bool shouldFileBeIndexed(const QString& fileName) const; /** * Checks if \p mimeType should be indexed * * \return \c true if the mimetype should be indexed according * to the configuration */ bool shouldMimeTypeBeIndexed(const QString& mimeType) const; /** * Returns true if the folder is in the list indexed directories * and not in the list of exclude directories */ bool folderInFolderList(const QString& path); /** * Check if \p path is in the list of folders to be indexed taking * include and exclude folders into account. * \p folder is set to the folder which was the reason for the descision. */ bool folderInFolderList(const QString& path, QString& folder) const; /** * Returns the internal version number of the Baloo database */ int databaseVersion() const; void setDatabaseVersion(int version); bool indexingEnabled() const; /** * Returns batch size */ - uint maxUncomittedFiles(); + uint maxUncomittedFiles() const; public Q_SLOTS: /** * Reread the config from disk and update the configuration cache. * This is only required for testing as normally the config updates * itself whenever the config file on disk changes. * * \return \c true if the config has actually changed */ void forceConfigUpdate(); /** * Should be called once the initial indexing is done, ie. all folders * have been indexed. */ void setInitialRun(bool isInitialRun); private: void buildFolderCache(); void buildExcludeFilterRegExpCache(); void buildMimeTypeCache(); KConfig m_config; /// Caching cleaned up list (no duplicates, no useless entries, etc.) QList > m_folderCache; /// Whether the folder cache needs to be rebuilt the next time it is used bool m_folderCacheDirty; /// cache of regexp objects for all exclude filters /// to prevent regexp parsing over and over RegExpCache m_excludeFilterRegExpCache; /// A set of mimetypes which should never be indexed QSet m_excludeMimetypes; bool m_indexHidden; bool m_onlyBasicIndexing; StorageDevices* m_devices; const uint m_maxUncomittedFiles; }; } #endif diff --git a/src/file/filtereddiriterator.cpp b/src/file/filtereddiriterator.cpp index 355422d6..fc79973b 100644 --- a/src/file/filtereddiriterator.cpp +++ b/src/file/filtereddiriterator.cpp @@ -1,137 +1,137 @@ /* * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "filtereddiriterator.h" #include "fileindexerconfig.h" #include using namespace Baloo; -FilteredDirIterator::FilteredDirIterator(FileIndexerConfig* config, const QString& folder, Filter filter) +FilteredDirIterator::FilteredDirIterator(const FileIndexerConfig* config, const QString& folder, Filter filter) : m_config(config) , m_currentIter(nullptr) , m_filters(QDir::NoDotAndDotDot | QDir::Readable | QDir::NoSymLinks) , m_firstItem(false) { if (filter == DirsOnly) { m_filters |= QDir::Dirs; } else if (filter == FilesAndDirs) { m_filters |= (QDir::Files | QDir::Dirs); } if (!m_config || m_config->shouldFolderBeIndexed(folder)) { m_currentIter = new QDirIterator(folder, m_filters); m_firstItem = true; } } FilteredDirIterator::~FilteredDirIterator() { delete m_currentIter; } QString FilteredDirIterator::next() { if (m_firstItem) { m_firstItem = false; m_filePath = m_currentIter->path(); return m_filePath; } m_filePath.clear(); if (!m_currentIter) { return QString(); } while (!m_currentIter->hasNext()) { delete m_currentIter; m_currentIter = nullptr; if (!m_paths.isEmpty()) { const QString path = m_paths.pop(); m_currentIter = new QDirIterator(path, m_filters); } else { return QString(); } } m_filePath = m_currentIter->next(); const QFileInfo info = m_currentIter->fileInfo(); if (info.isDir()) { if (shouldIndexFolder(m_filePath)) { m_paths.push(m_filePath); return m_filePath; } else { return next(); } } else if (info.isFile()) { bool shouldIndexHidden = false; if (m_config) shouldIndexHidden = m_config->indexHiddenFilesAndFolders(); bool shouldIndexFile = (!info.isHidden() || shouldIndexHidden) && (!m_config || m_config->shouldFileBeIndexed(info.fileName())); if (shouldIndexFile) { return m_filePath; } else { return next(); } } else { return next(); } } QString FilteredDirIterator::filePath() const { return m_filePath; } QFileInfo FilteredDirIterator::fileInfo() const { return m_currentIter ? m_currentIter->fileInfo() : QFileInfo(); } bool FilteredDirIterator::shouldIndexFolder(const QString& path) const { if (!m_config) { return true; } QString folder; if (m_config->folderInFolderList(path, folder)) { // we always index the folders in the list // ignoring the name filters if (folder == path) return true; // check for hidden folders QFileInfo fi(path); if (!m_config->indexHiddenFilesAndFolders() && fi.isHidden()) return false; return m_config->shouldFileBeIndexed(fi.fileName()); } return false; } diff --git a/src/file/filtereddiriterator.h b/src/file/filtereddiriterator.h index ae4ef7d5..7377928d 100644 --- a/src/file/filtereddiriterator.h +++ b/src/file/filtereddiriterator.h @@ -1,64 +1,64 @@ /* * Copyright (C) 2014 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef FILTEREDDIRITERATOR_H #define FILTEREDDIRITERATOR_H #include #include #include namespace Baloo { class FileIndexerConfig; class FilteredDirIterator { public: enum Filter { FilesAndDirs, DirsOnly }; - FilteredDirIterator(FileIndexerConfig* config, const QString& folder, Filter filter = FilesAndDirs); + FilteredDirIterator(const FileIndexerConfig* config, const QString& folder, Filter filter = FilesAndDirs); ~FilteredDirIterator(); QString next(); QString filePath() const; QFileInfo fileInfo() const; private: /** * Checks if the folder should be indexed. It only performs filename checks * on the filename, not on every part of the path. */ bool shouldIndexFolder(const QString& filePath) const; - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; QDirIterator* m_currentIter; QStack m_paths; QDir::Filters m_filters; QString m_filePath; bool m_firstItem; }; } #endif // FILTEREDDIRITERATOR_H diff --git a/src/file/modifiedfileindexer.cpp b/src/file/modifiedfileindexer.cpp index 112ebe8e..f52300ef 100644 --- a/src/file/modifiedfileindexer.cpp +++ b/src/file/modifiedfileindexer.cpp @@ -1,104 +1,104 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "modifiedfileindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "idutils.h" #include "database.h" #include "transaction.h" #include #include #include #include using namespace Baloo; -ModifiedFileIndexer::ModifiedFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& files) +ModifiedFileIndexer::ModifiedFileIndexer(Database* db, const FileIndexerConfig* config, const QStringList& files) : m_db(db) , m_config(config) , m_files(files) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void ModifiedFileIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith('/')); QString fileName = filePath.mid(filePath.lastIndexOf('/') + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } quint64 fileId = filePathToId(QFile::encodeName(filePath)); if (!fileId) { continue; } quint32 mTime = tr.documentTimeInfo(fileId).mTime; // A folders mtime is updated when a new file is added / removed / renamed // we don't really need to reindex a folder when that happens // In fact, we never need to reindex a folder if (mTime && mimetype == QLatin1String("inode/directory")) { continue; } // FIXME: Using QFileInfo over here is quite expensive! QFileInfo fileInfo(filePath); if (mTime == fileInfo.lastModified().toTime_t()) { continue; } // FIXME: The BasicIndexingJob extracts too much info. We only need the time BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // we can get modified events for files which do not exist // cause Baloo was not running and missed those events if (tr.hasDocument(job.document().id())) { tr.replaceDocument(job.document(), DocumentTime); } else { tr.addDocument(job.document()); } } tr.commit(); Q_EMIT done(); } diff --git a/src/file/modifiedfileindexer.h b/src/file/modifiedfileindexer.h index bd598fb1..a888df9c 100644 --- a/src/file/modifiedfileindexer.h +++ b/src/file/modifiedfileindexer.h @@ -1,50 +1,50 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_MODIFIEDFILEINDEXER_H #define BALOO_MODIFIEDFILEINDEXER_H #include #include #include namespace Baloo { class Database; class FileIndexerConfig; class ModifiedFileIndexer : public QObject, public QRunnable { Q_OBJECT public: - ModifiedFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& files); + ModifiedFileIndexer(Database* db, const FileIndexerConfig* config, const QStringList& files); void run() override; Q_SIGNALS: void done(); private: Database* m_db; - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; QStringList m_files; }; } #endif // BALOO_XATTRINDEXER_H diff --git a/src/file/newfileindexer.cpp b/src/file/newfileindexer.cpp index e73714bd..5452927d 100644 --- a/src/file/newfileindexer.cpp +++ b/src/file/newfileindexer.cpp @@ -1,77 +1,77 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "newfileindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; -NewFileIndexer::NewFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& newFiles) +NewFileIndexer::NewFileIndexer(Database* db, const FileIndexerConfig* config, const QStringList& newFiles) : m_db(db) , m_config(config) , m_files(newFiles) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void NewFileIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith(QLatin1Char('/'))); QString fileName = filePath.mid(filePath.lastIndexOf(QLatin1Char('/')) + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // The same file can be sent twice though it shouldn't be. // Lets just silently ignore it instead of crashing if (tr.hasDocument(job.document().id())) { continue; } tr.addDocument(job.document()); } tr.commit(); Q_EMIT done(); } diff --git a/src/file/newfileindexer.h b/src/file/newfileindexer.h index 2fb17118..04a5ea53 100644 --- a/src/file/newfileindexer.h +++ b/src/file/newfileindexer.h @@ -1,54 +1,54 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_NEWFILEINDEXER_H #define BALOO_NEWFILEINDEXER_H #include #include #include namespace Baloo { class Database; class FileIndexerConfig; /** * Does not check the folder path or the mtime of the file */ class NewFileIndexer : public QObject, public QRunnable { Q_OBJECT public: - NewFileIndexer(Database* db, FileIndexerConfig* config, const QStringList& newFiles); + NewFileIndexer(Database* db, const FileIndexerConfig* config, const QStringList& newFiles); void run() override; Q_SIGNALS: void done(); private: Database* m_db; - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; QStringList m_files; }; } #endif // BALOO_NEWFILEINDEXER_H diff --git a/src/file/unindexedfileindexer.cpp b/src/file/unindexedfileindexer.cpp index 7e6137b9..d10a298b 100644 --- a/src/file/unindexedfileindexer.cpp +++ b/src/file/unindexedfileindexer.cpp @@ -1,80 +1,80 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2015 Pinak Ahuja * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "unindexedfileindexer.h" #include "unindexedfileiterator.h" #include "transaction.h" #include "fileindexerconfig.h" #include "basicindexingjob.h" #include using namespace Baloo; -UnindexedFileIndexer::UnindexedFileIndexer(Database* db, FileIndexerConfig* config) +UnindexedFileIndexer::UnindexedFileIndexer(Database* db, const FileIndexerConfig* config) : m_db(db) , m_config(config) { } void UnindexedFileIndexer::run() { QMimeDatabase m_mimeDb; QStringList includeFolders = m_config->includeFolders(); for (const QString& includeFolder : includeFolders) { Transaction tr(m_db, Transaction::ReadWrite); UnIndexedFileIterator it(m_config, &tr, includeFolder); while (!it.next().isEmpty()) { BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(it.filePath(), it.mimetype(), level); job.index(); // We handle modified files by simply updating the mTime and filename in the Db and marking them for ContentIndexing const quint64 id = job.document().id(); if (tr.hasDocument(id)) { DocumentOperations ops = DocumentTime; if (it.cTimeChanged()) { ops |= XAttrTerms; if (tr.documentUrl(id) != it.filePath()) { ops |= (FileNameTerms | DocumentUrl); } } tr.replaceDocument(job.document(), ops); if (it.mTimeChanged()) { tr.setPhaseOne(id); } } else { // New file tr.addDocument(job.document()); } } tr.commit(); } Q_EMIT done(); } diff --git a/src/file/unindexedfileindexer.h b/src/file/unindexedfileindexer.h index f8c8b02a..02213d37 100644 --- a/src/file/unindexedfileindexer.h +++ b/src/file/unindexedfileindexer.h @@ -1,51 +1,51 @@ /* * This file is part of the KDE Baloo Project * Copyright (C) 2015 Pinak Ahuja * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #ifndef BALOO_UNINDEXEDFILEINDEXER #define BALOO_UNINDEXEDFILEINDEXER #include #include namespace Baloo { class Database; class FileIndexerConfig; class UnindexedFileIndexer : public QObject, public QRunnable { Q_OBJECT public: - UnindexedFileIndexer(Database* db, FileIndexerConfig* config); + UnindexedFileIndexer(Database* db, const FileIndexerConfig* config); void run() override; Q_SIGNALS: void done(); private: Database* m_db; - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; }; } #endif //BALOO_UNINDEXEDFILEINDEXER diff --git a/src/file/unindexedfileiterator.cpp b/src/file/unindexedfileiterator.cpp index b7c26a60..1d402204 100644 --- a/src/file/unindexedfileiterator.cpp +++ b/src/file/unindexedfileiterator.cpp @@ -1,125 +1,125 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "unindexedfileiterator.h" #include "fileindexerconfig.h" #include "idutils.h" #include "transaction.h" #include #include using namespace Baloo; -UnIndexedFileIterator::UnIndexedFileIterator(FileIndexerConfig* config, Transaction* transaction, const QString& folder) +UnIndexedFileIterator::UnIndexedFileIterator(const FileIndexerConfig* config, Transaction* transaction, const QString& folder) : m_config(config) , m_transaction(transaction) , m_iter(config, folder, FilteredDirIterator::FilesAndDirs) , m_mTimeChanged(false) , m_cTimeChanged(false) { } UnIndexedFileIterator::~UnIndexedFileIterator() { } QString UnIndexedFileIterator::filePath() const { return m_iter.filePath(); } QString UnIndexedFileIterator::mimetype() const { return m_mimetype; } bool UnIndexedFileIterator::mTimeChanged() const { return m_mTimeChanged; } bool UnIndexedFileIterator::cTimeChanged() const { return m_cTimeChanged; } QString UnIndexedFileIterator::next() { while (1) { const QString filePath = m_iter.next(); m_mTimeChanged = false; m_cTimeChanged = false; if (filePath.isEmpty()) { m_mimetype.clear(); return QString(); } // This mimetype may not be completely accurate, but that's okay. This is // just the initial phase of indexing. The second phase can try to find // a more accurate mimetype. m_mimetype = m_mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (shouldIndex(filePath, m_mimetype)) { return filePath; } } } bool UnIndexedFileIterator::shouldIndex(const QString& filePath, const QString& mimetype) { bool shouldIndexType = m_config->shouldMimeTypeBeIndexed(mimetype); if (!shouldIndexType) return false; const QFileInfo fileInfo = m_iter.fileInfo(); if (!fileInfo.exists()) return false; quint64 fileId = filePathToId(QFile::encodeName(filePath)); Q_ASSERT_X(fileId, "UnIndexedFileIterator::shouldIndex", "file id is 0"); if (!fileId) { return true; } DocumentTimeDB::TimeInfo timeInfo = m_transaction->documentTimeInfo(fileId); // A folders mtime is updated when a new file is added / removed / renamed // we don't really need to reindex a folder when that happens // In fact, we never need to reindex a folder if (timeInfo.mTime && mimetype == QLatin1String("inode/directory")) { return false; } if (timeInfo.mTime != fileInfo.lastModified().toTime_t()) { m_mTimeChanged = true; } if (timeInfo.cTime != fileInfo.created().toTime_t()) { m_cTimeChanged = true; } if (m_mTimeChanged || m_cTimeChanged) { return true; } return false; } diff --git a/src/file/unindexedfileiterator.h b/src/file/unindexedfileiterator.h index 52812ae8..13302792 100644 --- a/src/file/unindexedfileiterator.h +++ b/src/file/unindexedfileiterator.h @@ -1,68 +1,68 @@ /* * This file is part of the KDE Baloo project. * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_UNINDEXEDFILEITERATOR_H #define BALOO_UNINDEXEDFILEITERATOR_H #include "filtereddiriterator.h" #include namespace Baloo { class Transaction; /** * Iterate over all the files (and directories) under a specific directory which require * indexing. This checks the following - * - Config include / exclude path * - Config filters * - Config mimetype filters * - Datbase for mtime differences */ class UnIndexedFileIterator { public: - UnIndexedFileIterator(FileIndexerConfig* config, Transaction* transaction, const QString& folder); + UnIndexedFileIterator(const FileIndexerConfig* config, Transaction* transaction, const QString& folder); ~UnIndexedFileIterator(); QString next(); QString filePath() const; QString mimetype() const; bool mTimeChanged() const; bool cTimeChanged() const; private: bool shouldIndex(const QString& filePath, const QString& mimetype); - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; Transaction* m_transaction; FilteredDirIterator m_iter; QMimeDatabase m_mimeDb; QString m_mimetype; bool m_mTimeChanged; bool m_cTimeChanged; }; } #endif // BALOO_UNINDEXEDFILEITERATOR_H diff --git a/src/file/xattrindexer.cpp b/src/file/xattrindexer.cpp index 8d319a3f..009a7ebf 100644 --- a/src/file/xattrindexer.cpp +++ b/src/file/xattrindexer.cpp @@ -1,82 +1,82 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #include "xattrindexer.h" #include "basicindexingjob.h" #include "fileindexerconfig.h" #include "database.h" #include "transaction.h" #include using namespace Baloo; -XAttrIndexer::XAttrIndexer(Database* db, FileIndexerConfig* config, const QStringList& files) +XAttrIndexer::XAttrIndexer(Database* db, const FileIndexerConfig* config, const QStringList& files) : m_db(db) , m_config(config) , m_files(files) { Q_ASSERT(m_db); Q_ASSERT(m_config); Q_ASSERT(!m_files.isEmpty()); } void XAttrIndexer::run() { QMimeDatabase mimeDb; Transaction tr(m_db, Transaction::ReadWrite); for (const QString& filePath : qAsConst(m_files)) { Q_ASSERT(!filePath.endsWith(QLatin1Char('/'))); QString fileName = filePath.mid(filePath.lastIndexOf(QLatin1Char('/')) + 1); if (!m_config->shouldFileBeIndexed(fileName)) { continue; } QString mimetype = mimeDb.mimeTypeForFile(filePath, QMimeDatabase::MatchExtension).name(); if (!m_config->shouldMimeTypeBeIndexed(mimetype)) { continue; } // FIXME: The BasicIndexingJob extracts too much info. We only need the xattr BasicIndexingJob::IndexingLevel level = m_config->onlyBasicIndexing() ? BasicIndexingJob::NoLevel : BasicIndexingJob::MarkForContentIndexing; BasicIndexingJob job(filePath, mimetype, level); if (!job.index()) { continue; } // FIXME: This slightly defeats the point of having separate indexers // But we can get xattr changes of a file, even when it doesn't exist // cause we missed its creation somehow if (!tr.hasDocument(job.document().id())) { tr.addDocument(job.document()); continue; } // FIXME: Do we also need to update the ctime of the file? tr.replaceDocument(job.document(), XAttrTerms); } tr.commit(); Q_EMIT done(); } diff --git a/src/file/xattrindexer.h b/src/file/xattrindexer.h index 8686efe8..bb80a29c 100644 --- a/src/file/xattrindexer.h +++ b/src/file/xattrindexer.h @@ -1,50 +1,50 @@ /* * Copyright (C) 2015 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ #ifndef BALOO_XATTRINDEXER_H #define BALOO_XATTRINDEXER_H #include #include #include namespace Baloo { class Database; class FileIndexerConfig; class XAttrIndexer : public QObject, public QRunnable { Q_OBJECT public: - XAttrIndexer(Database* db, FileIndexerConfig* config, const QStringList& files); + XAttrIndexer(Database* db, const FileIndexerConfig* config, const QStringList& files); void run() override; Q_SIGNALS: void done(); private: Database* m_db; - FileIndexerConfig* m_config; + const FileIndexerConfig* m_config; QStringList m_files; }; } #endif // BALOO_XATTRINDEXER_H