diff --git a/CMakeLists.txt b/CMakeLists.txt index 8518752..d737241 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,130 +1,132 @@ cmake_minimum_required(VERSION 3.0) set(PIM_VERSION "5.9.40") project(AkonadiSearch VERSION ${PIM_VERSION}) # ECM setup set(KF5_VERSION "5.48.0") find_package(ECM ${KF5_VERSION} CONFIG REQUIRED) set(CMAKE_MODULE_PATH ${ECM_MODULE_PATH}) +set(CMAKE_CXX_STANDARD 14) + include(GenerateExportHeader) include(ECMGenerateHeaders) include(CMakePackageConfigHelpers) include(ECMSetupVersion) include(FeatureSummary) include(KDEInstallDirs) include(KDECMakeSettings) include(KDEFrameworkCompilerSettings NO_POLICY_SCOPE) include(ECMInstallIcons) include(ECMAddTests) include(ECMQtDeclareLoggingCategory) include(ECMCoverageOption) set(AKONADISEARCH_VERSION ${PIM_VERSION}) set(AKONADI_VERSION "5.8.80") set(AKONADI_MIMELIB_VERSION "5.8.80") set(KCONTACTS_LIB_VERSION "5.8.80") set(KCALENDARCORE_LIB_VERSION "5.8.80") set(KMIME_LIB_VERSION "5.8.80") set(QT_REQUIRED_VERSION "5.9.0") find_package(Qt5 ${QT_REQUIRED_VERSION} CONFIG REQUIRED Core Test) find_package(KF5 ${KF5_VERSION} REQUIRED COMPONENTS I18n Config Crash KCMUtils) if (NOT WIN32) find_package(KF5Runner ${KF5_VERSION} REQUIRED) endif() find_package(Xapian CONFIG) set_package_properties(Xapian PROPERTIES DESCRIPTION "The Xapian search engine library" URL "http://xapian.org" TYPE REQUIRED ) find_package(KF5Akonadi ${AKONADI_VERSION} CONFIG REQUIRED) find_package(KF5Contacts ${KCONTACTS_LIB_VERSION} CONFIG REQUIRED) find_package(KF5Mime ${KMIME_LIB_VERSION} CONFIG REQUIRED) find_package(KF5AkonadiMime ${AKONADI_MIMELIB_VERSION} CONFIG REQUIRED) find_package(KF5CalendarCore ${KCALENDARCORE_LIB_VERSION} CONFIG REQUIRED) find_package(KF5KIO ${KF5_VERSION} CONFIG REQUIRED) find_package(KF5DBusAddons ${KF5_VERSION} CONFIG REQUIRED) find_package(KF5Codecs ${KF5_VERSION} CONFIG REQUIRED) ecm_setup_version(PROJECT VARIABLE_PREFIX AKONADISEARCH VERSION_HEADER "${CMAKE_CURRENT_BINARY_DIR}/akonadi_search_version.h" PACKAGE_VERSION_FILE "${CMAKE_CURRENT_BINARY_DIR}/KF5AkonadiSearchConfigVersion.cmake" SOVERSION 5 ) # Compiler flags add_definitions(-DQT_NO_KEYWORDS) add_definitions(-DQT_NO_URL_CAST_FROM_STRING) add_definitions(-DQT_USE_QSTRINGBUILDER) add_definitions(-DTRANSLATION_DOMAIN=\"akonadi_search\") add_definitions("-DQT_NO_CAST_FROM_ASCII -DQT_NO_CAST_TO_ASCII") #add_definitions(-DQT_DISABLE_DEPRECATED_BEFORE=0x060000) # Turn exceptions on kde_enable_exceptions() include_directories( ${XAPIAN_INCLUDE_DIR} ) # Targets add_subdirectory(src) #add_subdirectory(xapian) #add_subdirectory(core) #add_subdirectory(agent) #add_subdirectory(lib) add_subdirectory(akonadiplugin) #add_subdirectory(search) #add_subdirectory(debug) if (NOT WIN32) #add_subdirectory(runner) endif() if (BUILD_TESTING) add_subdirectory(autotests) add_subdirectory(tests) endif() # Config files set(CMAKECONFIG_INSTALL_DIR "${KDE_INSTALL_CMAKEPACKAGEDIR}/KF5AkonadiSearch") configure_package_config_file( "${CMAKE_CURRENT_SOURCE_DIR}/KF5AkonadiSearchConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/KF5AkonadiSearchConfig.cmake" INSTALL_DESTINATION ${CMAKECONFIG_INSTALL_DIR} ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/KF5AkonadiSearchConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/KF5AkonadiSearchConfigVersion.cmake" DESTINATION ${CMAKECONFIG_INSTALL_DIR} COMPONENT devel ) install(EXPORT KF5AkonadiSearchTargets NAMESPACE KF5:: DESTINATION "${CMAKECONFIG_INSTALL_DIR}" FILE KF5AkonadiSearchTargets.cmake) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/akonadi_search_version.h" DESTINATION ${KDE_INSTALL_INCLUDEDIR_KF5} COMPONENT Devel ) install( FILES akonadi-search.renamecategories akonadi-search.categories DESTINATION ${KDE_INSTALL_CONFDIR} ) feature_summary(WHAT ALL INCLUDE_QUIET_PACKAGES FATAL_ON_MISSING_REQUIRED_PACKAGES ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 661c770..559bd92 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,99 +1,101 @@ kde_enable_exceptions() include_directories( ${XAPIAN_INCLUDE_DIR} ) set(AKONADISEARCH_LIB_SRCS contactcompleter.cpp + indexeditems.cpp indexer.cpp querymapper.cpp querymapper_p.cpp querypropertymapper.cpp resultiterator.cpp searchrunner.cpp store.cpp xapiandatabase.cpp xapiandocument.cpp xapiantermgenerator.cpp utils.cpp email/emailindexer.cpp email/emailquerymapper.cpp email/emailquerypropertymapper.cpp email/emailstore.cpp emailcontacts/emailcontactsindexer.cpp emailcontacts/emailcontactsstore.cpp contact/contactgroupindexer.cpp contact/contactindexer.cpp contact/contactquerymapper.cpp contact/contactquerypropertymapper.cpp contact/contactstore.cpp incidence/incidenceindexer.cpp incidence/incidencequerymapper.cpp incidence/incidencequerypropertymapper.cpp incidence/incidencestore.cpp collection/collectionindexer.cpp collection/collectionquerymapper.cpp collection/collectionquerypropertymapper.cpp collection/collectionstore.cpp note/noteindexer.cpp note/notequerymapper.cpp note/notequerypropertymapper.cpp note/notestore.cpp ) ecm_generate_headers(AkonadiSearch_LIB_HEADERS HEADER_NAMES ContactCompleter + IndexedItems Indexer ObjectCache QueryMapper ResultIterator SearchRunner Store REQUIRED_HEADERS AkonadiSearch_LIB_HEADERS ) ecm_qt_declare_logging_category( AKONADISEARCH_LIB_SRCS HEADER akonadisearch_debug.h IDENTIFIER AKONADISEARCH_LOG CATEGORY_NAME org.kde.pim.akonadisearch_lib) add_library(KF5AkonadiSearch ${AKONADISEARCH_LIB_SRCS}) add_library(KF5::AkonadiSearch ALIAS KF5AkonadiSearch) generate_export_header( KF5AkonadiSearch BASE_NAME AKONADISEARCH EXPORT_FILE_NAME akonadisearch_export.h) target_link_libraries(KF5AkonadiSearch PUBLIC Qt5::Core Qt5::Gui KF5::AkonadiCore PRIVATE KF5::Mime KF5::AkonadiMime KF5::Contacts KF5::CalendarCore KF5::Codecs Qt5::Concurrent ${XAPIAN_LIBRARIES} ) set_target_properties(KF5AkonadiSearch PROPERTIES VERSION ${AKONADISEARCH_VERSION_STRING} SOVERSION ${AKONADISEARCH_SOVERSION} EXPORT_NAME AkonadiSearch ) install(TARGETS KF5AkonadiSearch EXPORT KF5AkonadiSearchTargets ${KF5_INSTALL_TARGETS_DEFAULT_ARGS}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/akonadisearch_export.h ${AkonadiSearch_LIB_HEADERS} DESTINATION ${KDE_INSTALL_INCLUDEDIR_KF5}/AkonadiSearch COMPONENT Devel ) diff --git a/src/indexeditems.cpp b/src/indexeditems.cpp new file mode 100644 index 0000000..5d8a682 --- /dev/null +++ b/src/indexeditems.cpp @@ -0,0 +1,118 @@ +/* + * This file is part of the KDE Akonadi Search Project + * Copyright (C) 2016-2018 Laurent Montel + * Copyright (C) 2018 Daniel Vrátil + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) version 3, or any + * later version accepted by the membership of KDE e.V. (or its + * successor approved by the membership of KDE e.V.), which shall + * act as a proxy defined in Section 6 of version 3 of the license. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + */ + +#include + +#include "indexeditems.h" +#include "akonadisearch_debug.h" +#include "contact/contactstore.h" +#include "email/emailstore.h" +#include "incidence/incidencestore.h" +#include "note/notestore.h" +#include "querymapper.h" +#include "resultiterator.h" +#include "querypropertymapper_p.h" + +#include + +#include +#include +#include + +#include +#include + +using namespace Akonadi::Search; + +namespace { + +class HelperQueryPropertyMapper : public QueryPropertyMapper { +public: + HelperQueryPropertyMapper() = default; +}; + +class HelperQueryMapper : public QueryMapper { +public: + HelperQueryMapper() = default; + + const QueryPropertyMapper &propertyMapper() override { + static HelperQueryPropertyMapper mapper; + return mapper; + } +}; + +} + + +class Akonadi::Search::IndexedItems::Private +{ +public: + Private() {} + + void findIndexedInStore(QSet &indexed, Akonadi::Collection::Id collectionId, + const std::unique_ptr &store) { + Akonadi::SearchQuery query; + query.addTerm({Akonadi::SearchTerm::Collection, collectionId}); + auto iter = store->search(HelperQueryMapper().map(query)); + while (iter.next()) { + indexed.insert(iter.id()); + } + } +}; + +IndexedItems::IndexedItems() + : d(new Private()) +{ +} + +IndexedItems::~IndexedItems() +{ +} + +qint64 IndexedItems::indexedItems(Akonadi::Collection::Id id) +{ + std::vectorstores{ EmailStore(), ContactStore(), IncidenceStore(), NoteStore() }; + // Qt seems to struggle with deducing the return type of the Map lambda, hence the + // std::function wrapper. + return QtConcurrent::blockingMappedReduced(stores.begin(), stores.end(), + std::function([id](Store &store) { return store.indexedItems(id); }), + [](qint64 &result, qint64 val) -> qint64 { return result += val; }); +} + +QSet IndexedItems::findIndexedForType(Akonadi::Collection::Id collectionId, const QString &mimeType) +{ + QSet indexed; + d->findIndexedInStore(indexed, collectionId, std::unique_ptr(Store::create(mimeType))); + return indexed; +} + +QSet IndexedItems::findIndexed(Akonadi::Collection::Id collectionId) +{ + // TODO: Paralellize + QSet indexed; + d->findIndexedInStore(indexed, collectionId, std::make_unique()); + d->findIndexedInStore(indexed, collectionId, std::make_unique()); + d->findIndexedInStore(indexed, collectionId, std::make_unique()); + d->findIndexedInStore(indexed, collectionId, std::make_unique()); + return indexed; +} diff --git a/src/indexeditems.h b/src/indexeditems.h new file mode 100644 index 0000000..3b8e131 --- /dev/null +++ b/src/indexeditems.h @@ -0,0 +1,53 @@ +/* + * This file is part of the KDE Akonadi Search Project + * Copyright (C) 2016-2018 Laurent Montel + * Copyright (C) 2018 Daniel Vrátil + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) version 3, or any + * later version accepted by the membership of KDE e.V. (or its + * successor approved by the membership of KDE e.V.), which shall + * act as a proxy defined in Section 6 of version 3 of the license. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library. If not, see . + * + */ +#ifndef AKONADISEARCH_INDEXEDITEMS_H +#define AKONADISEARCH_INDEXEDITEMS_H + +#include +#include + +#include "akonadisearch_export.h" + +namespace Akonadi +{ +namespace Search +{ +class AKONADISEARCH_EXPORT IndexedItems +{ +public: + explicit IndexedItems(); + ~IndexedItems(); + + qint64 indexedItems(Akonadi::Collection::Id collectionId); + + QSet findIndexedForType(Akonadi::Collection::Id collectionId, const QString &mimeType); + QSet findIndexed(Akonadi::Collection::Id collectionId); + +private: + class Private; + QScopedPointer const d; +}; + +} +} +#endif // AKONADISEARCH_INDEXEDITEMS_H diff --git a/src/store.cpp b/src/store.cpp index a579b34..d67da2c 100644 --- a/src/store.cpp +++ b/src/store.cpp @@ -1,320 +1,348 @@ /* * Copyright (C) 2017 Daniel Vrátil * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include #include "store.h" #include "store_p.h" #include "akonadisearch_debug.h" #include "registrar_p.h" #include "resultiterator.h" #include "resultiterator_p.h" #include "xapiandatabase.h" #include "xapiandocument.h" #include "utils.h" +#include "querypropertymapper_p.h" #include "email/emailstore.h" #include "emailcontacts/emailcontactsstore.h" #include "contact/contactstore.h" #include "incidence/incidencestore.h" #include "note/notestore.h" #include "collection/collectionstore.h" +#include #include #include #include #include #include using namespace Akonadi::Search; namespace { Q_GLOBAL_STATIC(Registrar, sStores) static const unsigned int MaxQueryLimit = 10^6; } StorePrivate::StorePrivate(Store *q) : q(q) { commitTimer = new QTimer; commitTimer->setSingleShot(true); QObject::connect(commitTimer, &QTimer::timeout, [=]() { q->commit(); }); } StorePrivate::~StorePrivate() { delete commitTimer; if (db) { if (openMode == Store::WriteOnly) { db->commit(); } delete db; } } bool StorePrivate::ensureDb() { if (!db) { db = new XapianDatabase(dbPath(dbName), openMode == Store::ReadOnly); } return db && db->db(); } void StorePrivate::newChange() { ++changeCount; if (changeCount >= commitChangeCount) { q->commit(); } else { commitTimer->start(); } } QString StorePrivate::dbPath(const QString &dbName) const { // First look into the old location from Baloo times in ~/.local/share/baloo, // because we don't migrate the database files automatically. QString basePath; bool hasInstanceIdentifier = Akonadi::ServerManager::hasInstanceIdentifier(); if (hasInstanceIdentifier) { basePath = QStringLiteral("baloo/instances/%1").arg(Akonadi::ServerManager::instanceIdentifier()); } else { basePath = QStringLiteral("baloo"); } QString dbPath = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + QStringLiteral("/%1/%2/").arg(basePath, dbName); if (QDir(dbPath).exists()) { return dbPath; } // If the database does not exist in old Baloo folders, than use the new // location in Akonadi's datadir in ~/.local/share/akonadi/search_db. if (hasInstanceIdentifier) { basePath = QStringLiteral("akonadi/instance/%1/search_db").arg(Akonadi::ServerManager::instanceIdentifier()); } else { basePath = QStringLiteral("akonadi/search_db"); } dbPath = QStandardPaths::writableLocation(QStandardPaths::GenericDataLocation) + QStringLiteral("/%1/%2/").arg(basePath, dbName); QDir().mkpath(dbPath); return dbPath; } Store *Store::create(const QString &mimeType) { if (!sStores.exists()) { sStores->registerForType(); sStores->registerForType(); sStores->registerForType(); sStores->registerForType(); sStores->registerForType(); sStores->registerForType(); } return sStores->instantiate(mimeType); } Store::Store() : d(new StorePrivate(this)) { } Store::~Store() { delete d; } void Store::setAutoCommit(int changeCount, int timeoutMs) { d->commitChangeCount = changeCount; if (timeoutMs > 0) { d->commitTimer->setInterval(timeoutMs); d->commitTimer->start(); } else { d->commitTimer->stop(); } } QString Store::dbName() const { return d->dbName; } void Store::setDbName(const QString &name) { d->dbName = name; } Store::OpenMode Store::openMode() const { return d->openMode; } void Store::setOpenMode(OpenMode openMode) { d->openMode = openMode; } bool Store::index(qint64 id, const QByteArray &serializedIndex) { if (!d->ensureDb()) { return false; } QDataStream stream(serializedIndex); while (!stream.atEnd()) { qint64 documentId; stream >> documentId; if (documentId == -1) { documentId = id; } Xapian::Document document; stream >> document; d->newChange(); // FIXME: Xapian allows up to 2^32 documents, while Akonadi can deal with IDs // up to 2^64. However it's very unlikely that someone will ever have a // problem with running out of 32bit Item Ids.... if (!d->db->replaceDocument(static_cast(documentId), document)) { return false; } } return true; } bool Store::removeItem(qint64 id) { if (!d->ensureDb()) { return false; } d->newChange(); return d->db->deleteDocument(static_cast(id)); } bool Store::removeCollection(qint64 id) { if (!d->ensureDb()) { return false; } try { Xapian::Database *db = d->db->db(); Xapian::Query query(XapianDocument::collectionId(id).constData()); Xapian::Enquire enquire(*db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, db->get_doccount()); Xapian::MSetIterator end(mset.end()); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { removeItem(*it); } d->newChange(); return true; } catch (const Xapian::DocNotFoundError &) { return true; } catch (const Xapian::DatabaseError &err) { qCWarning(AKONADISEARCH_LOG) << "Error when removing from DB:" << err.get_error_string(); return false; } } bool Store::move(const qint64 id, qint64 srcCollection, qint64 destCollection) { if (!d->ensureDb()) { return false; } XapianDocument doc(d->db->document(id)); doc.removeTerm(XapianDocument::collectionId(srcCollection)); doc.addCollectionTerm(destCollection); d->newChange(); return d->db->replaceDocument(id, doc); } bool Store::copy(qint64 id, qint64 srcCollection, qint64 destId, qint64 destCollection) { if (!d->ensureDb()) { return false; } XapianDocument doc(d->db->document(id)); doc.removeTerm(XapianDocument::collectionId(srcCollection)); doc.addCollectionTerm(destCollection); d->newChange(); return d->db->replaceDocument(destId, doc); } bool Store::commit() { if (!d->ensureDb()) { return false; } d->changeCount = 0; if (d->commitTimer->isActive()) { d->commitTimer->stop(); } return d->db->commit(); } ResultIterator Store::search(const QByteArray &serializedQuery, unsigned int limit) { if (!d->ensureDb()) { return ResultIterator(); } Xapian::Enquire enq(*d->db->db()); const auto query = Xapian::Query::unserialise({ serializedQuery.constData(), static_cast(serializedQuery.size()) }); enq.set_query(query); Xapian::MSet mset; try { mset = enq.get_mset(0, limit > 0 ? limit : MaxQueryLimit); } catch (const Xapian::InvalidArgumentError &err) { qCWarning(AKONADISEARCH_LOG) << "Xapian query error:" << err.get_error_string(); qCWarning(AKONADISEARCH_LOG) << err.get_msg().c_str() << err.get_context().c_str() << err.get_description().c_str(); return {}; } ResultIterator iter; iter.d->init(mset); return iter; } + +namespace { + +class HelperPropertyMapper : public QueryPropertyMapper +{ +public: + HelperPropertyMapper() = default; +}; + +} + +qint64 Store::indexedItems(qint64 collectionId) +{ + if (!d->ensureDb()) { + return 0; + } + + const std::string term = HelperPropertyMapper().prefix(Akonadi::SearchTerm::Collection) + std::to_string(collectionId); + try { + return d->db->db()->get_termfreq(term); + } catch (const Xapian::Error &err) { + qCWarning(AKONADISEARCH_LOG, "Xapian termfreq error: %s", err.get_error_string()); + qCWarning(AKONADISEARCH_LOG, "%s: %s (%s)", err.get_msg().c_str(), err.get_context().c_str(), err.get_description().c_str()); + return 0; + } +} diff --git a/src/store.h b/src/store.h index 1a2007e..494f0b6 100644 --- a/src/store.h +++ b/src/store.h @@ -1,74 +1,76 @@ /* * Copyright (C) 2017 Daniel Vrátil * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #ifndef AKONADISEARCH_STORE_H_ #define AKONADISEARCH_STORE_H_ #include "akonadisearch_export.h" #include namespace Akonadi { namespace Search { class ResultIterator; class StorePrivate; class AKONADISEARCH_EXPORT Store { public: enum OpenMode { ReadOnly, WriteOnly }; static Store *create(const QString &mimeType); virtual ~Store(); QString dbName() const; OpenMode openMode() const; void setOpenMode(OpenMode openMode); void setAutoCommit(int changeCount, int timeoutMs); virtual bool index(qint64 id, const QByteArray &serializedIndex); virtual bool move(qint64 id, qint64 srcCollection, qint64 destCollection); virtual bool copy(qint64 id, qint64 srcCollection, qint64 destId, qint64 destCollection); virtual bool removeItem(qint64 id); virtual bool removeCollection(qint64 id); ResultIterator search(const QByteArray &serializedQuery, unsigned int limit = 0); + qint64 indexedItems(qint64 collectionId); + bool commit(); protected: explicit Store(); void setDbName(const QString &name); StorePrivate * const d; }; } } #endif