diff --git a/agent/collectionindexingjob.h b/agent/collectionindexingjob.h index 11fab74..9bcfe54 100644 --- a/agent/collectionindexingjob.h +++ b/agent/collectionindexingjob.h @@ -1,74 +1,74 @@ /* * Copyright 2014 Christian Mollekopf * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of * the License or (at your option) version 3 or any later version * accepted by the membership of KDE e.V. (or its successor approved * by the membership of KDE e.V.), which shall act as a proxy * defined in Section 14 of version 3 of the license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . * */ #ifndef COLLECTIONINDEXINGJOB_H #define COLLECTIONINDEXINGJOB_H #include #include #include -#include +#include #include "index.h" /** * Indexing Job that ensure a collections is fully indexed. * The following steps are required to bring the index up-to date: * 1. Index pending items * 2. Check if indexed item == local items (optimization) * 3. Make a full diff if necessary */ class CollectionIndexingJob : public KJob { Q_OBJECT public: explicit CollectionIndexingJob(Index &index, const Akonadi::Collection &col, const QList &pending, QObject *parent = nullptr); void setFullSync(bool); void start() override; Q_SIGNALS: void status(int, const QString &); void percent(int); private Q_SLOTS: void slotOnCollectionFetched(KJob *); void slotPendingItemsReceived(const Akonadi::Item::List &items); void slotPendingIndexed(KJob *); void slotUnindexedItemsReceived(const Akonadi::Item::List &items); void slotFoundUnindexed(KJob *); private: void findUnindexed(); void indexItems(const QList &itemIds); Akonadi::Collection m_collection; QList m_pending; QSet m_indexedItems; QList m_needsIndexing; Index &m_index; - QTime m_time; + QElapsedTimer m_time; bool m_reindexingLock; bool m_fullSync; int m_progressCounter; int m_progressTotal; }; #endif diff --git a/agent/emailindexer.cpp b/agent/emailindexer.cpp index 7c84222..49bd4b7 100644 --- a/agent/emailindexer.cpp +++ b/agent/emailindexer.cpp @@ -1,424 +1,424 @@ /* * This file is part of the KDE Akonadi Search Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "emailindexer.h" #include "akonadi_indexer_agent_debug.h" #include #include #include #include EmailIndexer::EmailIndexer(const QString &path, const QString &contactDbPath): AbstractIndexer(), m_doc(nullptr), m_termGen(nullptr), m_contactDb(nullptr) { try { m_db = new Xapian::WritableDatabase(path.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); } catch (const Xapian::DatabaseCorruptError &err) { qCWarning(AKONADI_INDEXER_AGENT_LOG) << "Database Corrupted - What did you do?"; qCWarning(AKONADI_INDEXER_AGENT_LOG) << err.get_error_string(); m_db = nullptr; } catch (const Xapian::Error &e) { qCWarning(AKONADI_INDEXER_AGENT_LOG) << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); m_db = nullptr; } try { m_contactDb = new Xapian::WritableDatabase(contactDbPath.toUtf8().constData(), Xapian::DB_CREATE_OR_OPEN); } catch (const Xapian::DatabaseCorruptError &err) { qCWarning(AKONADI_INDEXER_AGENT_LOG) << "Database Corrupted - What did you do?"; qCWarning(AKONADI_INDEXER_AGENT_LOG) << err.get_error_string(); m_contactDb = nullptr; } catch (const Xapian::Error &e) { qCWarning(AKONADI_INDEXER_AGENT_LOG) << QString::fromStdString(e.get_type()) << QString::fromStdString(e.get_description()); m_contactDb = nullptr; } } EmailIndexer::~EmailIndexer() { if (m_db) { m_db->commit(); delete m_db; } if (m_contactDb) { m_contactDb->commit(); delete m_contactDb; } } QStringList EmailIndexer::mimeTypes() const { return QStringList() << KMime::Message::mimeType(); } void EmailIndexer::index(const Akonadi::Item &item) { qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing item" << item.id(); if (!m_db) { return; } Akonadi::MessageStatus status; status.setStatusFromFlags(item.flags()); if (status.isSpam()) { return; } KMime::Message::Ptr msg; try { msg = item.payload(); } catch (const Akonadi::PayloadException &) { return; } m_doc = new Xapian::Document(); m_termGen = new Xapian::TermGenerator(); m_termGen->set_document(*m_doc); m_termGen->set_database(*m_db); processMessageStatus(status); process(msg); // Size m_doc->add_value(1, QString::number(item.size()).toStdString()); // Parent collection Q_ASSERT_X(item.parentCollection().isValid(), "Akonadi::Search::EmailIndexer::index", "Item does not have a valid parent collection"); Akonadi::Collection::Id colId = item.parentCollection().id(); QByteArray term = 'C' + QByteArray::number(colId); m_doc->add_boolean_term(term.data()); m_db->replace_document(item.id(), *m_doc); delete m_doc; delete m_termGen; m_doc = nullptr; m_termGen = nullptr; qCDebug(AKONADI_INDEXER_AGENT_LOG) << "DONE Indexing item" << item.id(); } void EmailIndexer::insert(const QByteArray &key, KMime::Headers::Base *unstructured) { if (unstructured) { m_termGen->index_text_without_positions(unstructured->asUnicodeString().toUtf8().constData(), 1, key.data()); } } void EmailIndexer::insert(const QByteArray &key, KMime::Headers::Generics::MailboxList *mlist) { if (mlist) { insert(key, mlist->mailboxes()); } } void EmailIndexer::insert(const QByteArray &key, KMime::Headers::Generics::AddressList *alist) { if (alist) { insert(key, alist->mailboxes()); } } namespace { // Does some extra stuff such as lower casing the email, removing all quotes // and removing extra spaces // TODO: Move this into KMime? // TODO: If name is all upper/lower then try to captialize it? QString prettyAddress(const KMime::Types::Mailbox &mbox) { const QString name = mbox.name().simplified(); const QByteArray email = mbox.address().simplified().toLower(); return KEmailAddress::normalizedAddress(name, QString::fromUtf8(email)); } } // Add once with a prefix and once without void EmailIndexer::insert(const QByteArray &key, const KMime::Types::Mailbox::List &list) { if (!m_contactDb) { return; } for (const KMime::Types::Mailbox &mbox : list) { std::string name(mbox.name().toUtf8().constData()); m_termGen->index_text_without_positions(name, 1, key.data()); m_termGen->index_text_without_positions(name, 1); m_termGen->index_text_without_positions(mbox.address().data(), 1, key.data()); m_termGen->index_text_without_positions(mbox.address().data(), 1); m_doc->add_term(QByteArray(key + mbox.address()).data()); m_doc->add_term(mbox.address().data()); // // Add emails for email auto-completion // const QString pa = prettyAddress(mbox); int id = qHash(pa); try { Xapian::Document doc = m_contactDb->get_document(id); continue; } catch (const Xapian::DocNotFoundError &) { Xapian::Document doc; std::string pretty(pa.toUtf8().constData()); doc.set_data(pretty); Xapian::TermGenerator termGen; termGen.set_document(doc); termGen.index_text(pretty); doc.add_term(mbox.address().data()); m_contactDb->replace_document(id, doc); } } } // FIXME: Only index properties that are actually searched! void EmailIndexer::process(const KMime::Message::Ptr &msg) { // // Process Headers // (Give the subject a higher priority) KMime::Headers::Subject *subject = msg->subject(false); if (subject) { std::string str(subject->asUnicodeString().toUtf8().constData()); qCDebug(AKONADI_INDEXER_AGENT_LOG) << "Indexing" << str.c_str(); m_termGen->index_text_without_positions(str, 1, "SU"); m_termGen->index_text_without_positions(str, 100); m_doc->set_data(str); } KMime::Headers::Date *date = msg->date(false); if (date) { - const QString str = QString::number(date->dateTime().toTime_t()); + const QString str = QString::number(date->dateTime().toSecsSinceEpoch()); m_doc->add_value(0, str.toStdString()); const QString julianDay = QString::number(date->dateTime().date().toJulianDay()); m_doc->add_value(2, julianDay.toStdString()); } insert("F", msg->from(false)); insert("T", msg->to(false)); insert("CC", msg->cc(false)); insert("BC", msg->bcc(false)); insert("O", msg->organization(false)); insert("RT", msg->replyTo(false)); insert("RF", msg->headerByType("Resent-From")); insert("LI", msg->headerByType("List-Id")); insert("XL", msg->headerByType("X-Loop")); insert("XML", msg->headerByType("X-Mailing-List")); insert("XSF", msg->headerByType("X-Spam-Flag")); // // Process Plain Text Content // //Index all headers m_termGen->index_text_without_positions(std::string(msg->head().constData()), 1, "HE"); KMime::Content *mainBody = msg->mainBodyPart("text/plain"); if (mainBody) { const std::string text(mainBody->decodedText().toUtf8().constData()); m_termGen->index_text_without_positions(text); m_termGen->index_text_without_positions(text, 1, "BO"); } else { processPart(msg.data(), nullptr); } } void EmailIndexer::processPart(KMime::Content *content, KMime::Content *mainContent) { if (content == mainContent) { return; } KMime::Headers::ContentType *type = content->contentType(false); if (type) { if (type->isMultipart()) { if (type->isSubtype("encrypted")) { return; } for (KMime::Content *c : content->contents()) { processPart(c, mainContent); } } // Only get HTML content, if no plain text content if (!mainContent && type->isHTMLText()) { QTextDocument doc; doc.setHtml(content->decodedText()); const std::string text(doc.toPlainText().toUtf8().constData()); m_termGen->index_text_without_positions(text); } } // FIXME: Handle attachments? } void EmailIndexer::processMessageStatus(const Akonadi::MessageStatus &status) { insertBool('R', status.isRead()); insertBool('A', status.hasAttachment()); insertBool('I', status.isImportant()); insertBool('W', status.isWatched()); insertBool('T', status.isToAct()); insertBool('D', status.isDeleted()); insertBool('S', status.isSpam()); insertBool('E', status.isReplied()); insertBool('G', status.isIgnored()); insertBool('F', status.isForwarded()); insertBool('N', status.isSent()); insertBool('Q', status.isQueued()); insertBool('H', status.isHam()); insertBool('C', status.isEncrypted()); insertBool('V', status.hasInvitation()); } void EmailIndexer::insertBool(char key, bool value) { QByteArray term("B"); if (value) { term.append(key); } else { term.append('N'); term.append(key); } m_doc->add_boolean_term(term.data()); } void EmailIndexer::toggleFlag(Xapian::Document &doc, const char *remove, const char *add) { try { doc.remove_term(remove); } catch (const Xapian::InvalidArgumentError &e) { // The previous flag state was not indexed, continue } doc.add_term(add); } void EmailIndexer::updateFlags(const Akonadi::Item &item, const QSet &added, const QSet &removed) { if (!m_db) { return; } Xapian::Document doc; try { doc = m_db->get_document(item.id()); } catch (const Xapian::DocNotFoundError &) { return; } for (const QByteArray &flag : removed) { if (flag == Akonadi::MessageFlags::Seen) { toggleFlag(doc, "BR", "BNR"); } else if (flag == Akonadi::MessageFlags::Flagged) { toggleFlag(doc, "BI", "BNI"); } else if (flag == Akonadi::MessageFlags::Watched) { toggleFlag(doc, "BW", "BNW"); } } for (const QByteArray &flag : added) { if (flag == Akonadi::MessageFlags::Seen) { toggleFlag(doc, "BNR", "BR"); } else if (flag == Akonadi::MessageFlags::Flagged) { toggleFlag(doc, "BNI", "BI"); } else if (flag == Akonadi::MessageFlags::Watched) { toggleFlag(doc, "BNW", "BW"); } } m_db->replace_document(doc.get_docid(), doc); } void EmailIndexer::remove(const Akonadi::Item &item) { if (!m_db) { return; } try { m_db->delete_document(item.id()); //TODO remove contacts from contact db? } catch (const Xapian::DocNotFoundError &) { return; } } void EmailIndexer::remove(const Akonadi::Collection &collection) { if (!m_db) { return; } try { Xapian::Query query('C' + QString::number(collection.id()).toStdString()); Xapian::Enquire enquire(*m_db); enquire.set_query(query); Xapian::MSet mset = enquire.get_mset(0, m_db->get_doccount()); Xapian::MSetIterator end = mset.end(); for (Xapian::MSetIterator it = mset.begin(); it != end; ++it) { const qint64 id = *it; remove(Akonadi::Item(id)); } } catch (const Xapian::DocNotFoundError &) { return; } } void EmailIndexer::move(Akonadi::Item::Id itemId, Akonadi::Collection::Id from, Akonadi::Collection::Id to) { if (!m_db) { return; } Xapian::Document doc; try { doc = m_db->get_document(itemId); } catch (const Xapian::DocNotFoundError &) { return; } const QByteArray ft = 'C' + QByteArray::number(from); const QByteArray tt = 'C' + QByteArray::number(to); doc.remove_term(ft.data()); doc.add_boolean_term(tt.data()); m_db->replace_document(doc.get_docid(), doc); } void EmailIndexer::commit() { if (m_db) { m_db->commit(); } if (m_contactDb) { m_contactDb->commit(); } } diff --git a/agent/tests/emailtest.cpp b/agent/tests/emailtest.cpp index 5868926..94ab015 100644 --- a/agent/tests/emailtest.cpp +++ b/agent/tests/emailtest.cpp @@ -1,205 +1,206 @@ /* * This file is part of the KDE Akonadi Search Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "emailindexer.h" #include #include #include #include +#include #include #include #include #include #include class App : public QApplication { Q_OBJECT public: App(int &argc, char **argv, int flags = ApplicationFlags); private Q_SLOTS: void main(); void slotRootCollectionsFetched(KJob *job); void indexNextCollection(); void itemReceived(const Akonadi::Item::List &item); void slotIndexed(); void slotCommitTimerElapsed(); private: Akonadi::Collection::List m_collections; EmailIndexer m_indexer; - QTime m_totalTime; + QElapsedTimer m_totalTime; int m_indexTime; int m_numEmails; QTimer m_commitTimer; }; int main(int argc, char **argv) { App app(argc, argv); return app.exec(); } App::App(int &argc, char **argv, int flags) : QApplication(argc, argv, flags) , m_indexer(QStringLiteral("/tmp/xap"), QStringLiteral("/tmp/xapC")) { QTimer::singleShot(0, this, &App::main); } void App::main() { m_commitTimer.setInterval(1000); connect(&m_commitTimer, &QTimer::timeout, this, &App::slotCommitTimerElapsed); m_commitTimer.start(); Akonadi::CollectionFetchJob *job = new Akonadi::CollectionFetchJob(Akonadi::Collection::root(), Akonadi::CollectionFetchJob::Recursive); connect(job, &Akonadi::CollectionFetchJob::finished, this, &App::slotRootCollectionsFetched); job->start(); m_numEmails = 0; m_indexTime = 0; m_totalTime.start(); } void App::slotRootCollectionsFetched(KJob *kjob) { Akonadi::CollectionFetchJob *job = qobject_cast(kjob); m_collections = job->collections(); QMutableVectorIterator it(m_collections); while (it.hasNext()) { const Akonadi::Collection &c = it.next(); const QStringList mimeTypes = c.contentMimeTypes(); if (!c.contentMimeTypes().contains(QLatin1String("message/rfc822"))) { it.remove(); } } if (m_collections.size()) { indexNextCollection(); } else { qDebug() << "No collections to index"; } } void App::indexNextCollection() { Akonadi::ItemFetchJob *fetchJob = new Akonadi::ItemFetchJob(m_collections.takeFirst(), this); fetchJob->fetchScope().fetchAllAttributes(true); fetchJob->fetchScope().fetchFullPayload(true); fetchJob->fetchScope().setFetchModificationTime(false); fetchJob->fetchScope().setAncestorRetrieval(Akonadi::ItemFetchScope::Parent); fetchJob->setDeliveryOption(Akonadi::ItemFetchJob::EmitItemsIndividually); connect(fetchJob, &Akonadi::ItemFetchJob::itemsReceived, this, &App::itemReceived); connect(fetchJob, &Akonadi::ItemFetchJob::result, this, &App::slotIndexed); } void App::itemReceived(const Akonadi::Item::List &itemList) { - QTime timer; + QElapsedTimer timer; timer.start(); for (const Akonadi::Item &item : itemList) { m_indexer.index(item); } m_indexTime += timer.elapsed(); m_numEmails += itemList.size(); } void App::slotCommitTimerElapsed() { - QTime timer; + QElapsedTimer timer; timer.start(); m_indexer.commit(); m_indexTime += timer.elapsed(); qDebug() << "Emails:" << m_numEmails; qDebug() << "Total Time:" << m_totalTime.elapsed() / 1000.0 << " seconds"; qDebug() << "Index Time:" << m_indexTime / 1000.0 << " seconds"; } void App::slotIndexed() { if (!m_collections.isEmpty()) { QTimer::singleShot(0, this, &App::indexNextCollection); return; } m_indexer.commit(); qDebug() << "Emails:" << m_numEmails; qDebug() << "Total Time:" << m_totalTime.elapsed() / 1000.0 << " seconds"; qDebug() << "Index Time:" << m_indexTime / 1000.0 << " seconds"; // Print the io usage QFile file(QStringLiteral("/proc/self/io")); file.open(QIODevice::ReadOnly | QIODevice::Text); QTextStream fs(&file); QString str = fs.readAll(); qDebug() << "------- IO ---------"; QTextStream stream(&str); while (!stream.atEnd()) { QString str = stream.readLine(); QString rchar(QStringLiteral("rchar: ")); if (str.startsWith(rchar)) { ulong amt = str.midRef(rchar.size()).toULong(); qDebug() << "Read:" << amt / 1024 << "kb"; } QString wchar(QStringLiteral("wchar: ")); if (str.startsWith(wchar)) { ulong amt = str.midRef(wchar.size()).toULong(); qDebug() << "Write:" << amt / 1024 << "kb"; } QString read(QStringLiteral("read_bytes: ")); if (str.startsWith(read)) { ulong amt = str.midRef(read.size()).toULong(); qDebug() << "Actual Reads:" << amt / 1024 << "kb"; } QString write(QStringLiteral("write_bytes: ")); if (str.startsWith(write)) { ulong amt = str.midRef(write.size()).toULong(); qDebug() << "Actual Writes:" << amt / 1024 << "kb"; } } qDebug() << "\nThe actual read/writes may be 0 because of an existing" << "cache and /tmp being memory mapped"; quit(); } #include "emailtest.moc" diff --git a/akonadiplugin/searchplugin.cpp b/akonadiplugin/searchplugin.cpp index 072cdb5..b4353da 100644 --- a/akonadiplugin/searchplugin.cpp +++ b/akonadiplugin/searchplugin.cpp @@ -1,376 +1,376 @@ /* * This file is part of the KDE Akonadi Search Project * Copyright (C) 2014 Christian Mollekopf * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "searchplugin.h" #include "query.h" #include "term.h" #include "resultiterator.h" #include #include "akonadiplugin_indexer_debug.h" #include #include #include #include using namespace Akonadi::Search; static Term::Operation mapRelation(Akonadi::SearchTerm::Relation relation) { if (relation == Akonadi::SearchTerm::RelAnd) { return Term::And; } return Term::Or; } static Term::Comparator mapComparator(Akonadi::SearchTerm::Condition comparator) { if (comparator == Akonadi::SearchTerm::CondContains) { return Term::Contains; } if (comparator == Akonadi::SearchTerm::CondGreaterOrEqual) { return Term::GreaterEqual; } if (comparator == Akonadi::SearchTerm::CondGreaterThan) { return Term::Greater; } if (comparator == Akonadi::SearchTerm::CondEqual) { return Term::Equal; } if (comparator == Akonadi::SearchTerm::CondLessOrEqual) { return Term::LessEqual; } if (comparator == Akonadi::SearchTerm::CondLessThan) { return Term::Less; } return Term::Auto; } static Term getTerm(const Akonadi::SearchTerm &term, const QString &property) { Term t(property, term.value().toString(), mapComparator(term.condition())); t.setNegation(term.isNegated()); return t; } Term recursiveEmailTermMapping(const Akonadi::SearchTerm &term) { if (!term.subTerms().isEmpty()) { Term t(mapRelation(term.relation())); const auto subTermsResult = term.subTerms(); for (const Akonadi::SearchTerm &subterm : subTermsResult) { const Term newTerm = recursiveEmailTermMapping(subterm); if (newTerm.isValid()) { t.addSubTerm(newTerm); } } return t; } else { // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << term.key() << term.value(); const Akonadi::EmailSearchTerm::EmailSearchField field = Akonadi::EmailSearchTerm::fromKey(term.key()); switch (field) { case Akonadi::EmailSearchTerm::Message: { Term s(Term::Or); s.setNegation(term.isNegated()); s.addSubTerm(Term(QStringLiteral("body"), term.value(), mapComparator(term.condition()))); s.addSubTerm(Term(QStringLiteral("headers"), term.value(), mapComparator(term.condition()))); return s; } case Akonadi::EmailSearchTerm::Body: return getTerm(term, QStringLiteral("body")); case Akonadi::EmailSearchTerm::Headers: return getTerm(term, QStringLiteral("headers")); case Akonadi::EmailSearchTerm::ByteSize: return getTerm(term, QStringLiteral("size")); case Akonadi::EmailSearchTerm::HeaderDate: { - Term s(QStringLiteral("date"), QString::number(term.value().toDateTime().toTime_t()), mapComparator(term.condition())); + Term s(QStringLiteral("date"), QString::number(term.value().toDateTime().toSecsSinceEpoch()), mapComparator(term.condition())); s.setNegation(term.isNegated()); return s; } case Akonadi::EmailSearchTerm::HeaderOnlyDate: { Term s(QStringLiteral("onlydate"), QString::number(term.value().toDate().toJulianDay()), mapComparator(term.condition())); s.setNegation(term.isNegated()); return s; } case Akonadi::EmailSearchTerm::Subject: return getTerm(term, QStringLiteral("subject")); case Akonadi::EmailSearchTerm::HeaderFrom: return getTerm(term, QStringLiteral("from")); case Akonadi::EmailSearchTerm::HeaderTo: return getTerm(term, QStringLiteral("to")); case Akonadi::EmailSearchTerm::HeaderCC: return getTerm(term, QStringLiteral("cc")); case Akonadi::EmailSearchTerm::HeaderBCC: return getTerm(term, QStringLiteral("bcc")); case Akonadi::EmailSearchTerm::MessageStatus: if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Flagged)) { return Term(QStringLiteral("isimportant"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::ToAct)) { return Term(QStringLiteral("istoact"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Watched)) { return Term(QStringLiteral("iswatched"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Deleted)) { return Term(QStringLiteral("isdeleted"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Spam)) { return Term(QStringLiteral("isspam"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Replied)) { return Term(QStringLiteral("isreplied"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Ignored)) { return Term(QStringLiteral("isignored"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Forwarded)) { return Term(QStringLiteral("isforwarded"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Sent)) { return Term(QStringLiteral("issent"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Queued)) { return Term(QStringLiteral("isqueued"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Ham)) { return Term(QStringLiteral("isham"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Seen)) { return Term(QStringLiteral("isread"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::HasAttachment)) { return Term(QStringLiteral("hasattachment"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::Encrypted)) { return Term(QStringLiteral("isencrypted"), !term.isNegated()); } if (term.value().toString() == QString::fromLatin1(Akonadi::MessageFlags::HasInvitation)) { return Term(QStringLiteral("hasinvitation"), !term.isNegated()); } break; case Akonadi::EmailSearchTerm::MessageTag: //search directly in akonadi? or index tags. break; case Akonadi::EmailSearchTerm::HeaderReplyTo: return getTerm(term, QStringLiteral("replyto")); case Akonadi::EmailSearchTerm::HeaderOrganization: return getTerm(term, QStringLiteral("organization")); case Akonadi::EmailSearchTerm::HeaderListId: return getTerm(term, QStringLiteral("listid")); case Akonadi::EmailSearchTerm::HeaderResentFrom: return getTerm(term, QStringLiteral("resentfrom")); case Akonadi::EmailSearchTerm::HeaderXLoop: return getTerm(term, QStringLiteral("xloop")); case Akonadi::EmailSearchTerm::HeaderXMailingList: return getTerm(term, QStringLiteral("xmailinglist")); case Akonadi::EmailSearchTerm::HeaderXSpamFlag: return getTerm(term, QStringLiteral("xspamflag")); case Akonadi::EmailSearchTerm::Attachment: return Term(QStringLiteral("hasattachment"), !term.isNegated()); case Akonadi::EmailSearchTerm::Unknown: default: if (!term.key().isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "unknown term " << term.key(); } } } return Term(); } Term recursiveCalendarTermMapping(const Akonadi::SearchTerm &term) { if (!term.subTerms().isEmpty()) { Term t(mapRelation(term.relation())); for (const Akonadi::SearchTerm &subterm : term.subTerms()) { const Term newTerm = recursiveCalendarTermMapping(subterm); if (newTerm.isValid()) { t.addSubTerm(newTerm); } } return t; } else { // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << term.key() << term.value(); const Akonadi::IncidenceSearchTerm::IncidenceSearchField field = Akonadi::IncidenceSearchTerm::fromKey(term.key()); switch (field) { case Akonadi::IncidenceSearchTerm::Organizer: return getTerm(term, QStringLiteral("organizer")); case Akonadi::IncidenceSearchTerm::Summary: return getTerm(term, QStringLiteral("summary")); case Akonadi::IncidenceSearchTerm::Location: return getTerm(term, QStringLiteral("location")); case Akonadi::IncidenceSearchTerm::PartStatus: { Term t(QStringLiteral("partstatus"), term.value().toString(), Term::Equal); t.setNegation(term.isNegated()); return t; } default: if (!term.key().isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "unknown term " << term.key(); } } } return Term(); } Term recursiveNoteTermMapping(const Akonadi::SearchTerm &term) { if (!term.subTerms().isEmpty()) { Term t(mapRelation(term.relation())); for (const Akonadi::SearchTerm &subterm : term.subTerms()) { const Term newTerm = recursiveNoteTermMapping(subterm); if (newTerm.isValid()) { t.addSubTerm(newTerm); } } return t; } else { // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << term.key() << term.value(); const Akonadi::EmailSearchTerm::EmailSearchField field = Akonadi::EmailSearchTerm::fromKey(term.key()); switch (field) { case Akonadi::EmailSearchTerm::Subject: return getTerm(term, QStringLiteral("subject")); case Akonadi::EmailSearchTerm::Body: return getTerm(term, QStringLiteral("body")); default: if (!term.key().isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "unknown term " << term.key(); } } } return Term(); } Term recursiveContactTermMapping(const Akonadi::SearchTerm &term) { if (!term.subTerms().isEmpty()) { Term t(mapRelation(term.relation())); for (const Akonadi::SearchTerm &subterm : term.subTerms()) { const Term newTerm = recursiveContactTermMapping(subterm); if (newTerm.isValid()) { t.addSubTerm(newTerm); } } return t; } else { // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << term.key() << term.value(); const Akonadi::ContactSearchTerm::ContactSearchField field = Akonadi::ContactSearchTerm::fromKey(term.key()); switch (field) { case Akonadi::ContactSearchTerm::Name: return getTerm(term, QStringLiteral("name")); case Akonadi::ContactSearchTerm::Email: return getTerm(term, QStringLiteral("email")); case Akonadi::ContactSearchTerm::Nickname: return getTerm(term, QStringLiteral("nick")); case Akonadi::ContactSearchTerm::Uid: return getTerm(term, QStringLiteral("uid")); case Akonadi::ContactSearchTerm::Unknown: default: if (!term.key().isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "unknown term " << term.key(); } } } return Term(); } QSet SearchPlugin::search(const QString &akonadiQuery, const QVector &collections, const QStringList &mimeTypes) { if (akonadiQuery.isEmpty() && collections.isEmpty() && mimeTypes.isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "empty query"; return {}; } Akonadi::SearchQuery searchQuery; if (!akonadiQuery.isEmpty()) { searchQuery = Akonadi::SearchQuery::fromJSON(akonadiQuery.toLatin1()); if (searchQuery.isNull() && collections.isEmpty() && mimeTypes.isEmpty()) { return {}; } } const Akonadi::SearchTerm term = searchQuery.term(); Query query; Term t; if (mimeTypes.contains(QLatin1String("message/rfc822"))) { // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << "mail query"; query.setType(QStringLiteral("Email")); t = recursiveEmailTermMapping(term); } else if (mimeTypes.contains(KContacts::Addressee::mimeType()) || mimeTypes.contains(KContacts::ContactGroup::mimeType())) { query.setType(QStringLiteral("Contact")); t = recursiveContactTermMapping(term); } else if (mimeTypes.contains(QLatin1String("text/x-vnd.akonadi.note"))) { query.setType(QStringLiteral("Note")); t = recursiveNoteTermMapping(term); } else if (mimeTypes.contains(QLatin1String("application/x-vnd.akonadi.calendar.event")) || mimeTypes.contains(QLatin1String("application/x-vnd.akonadi.calendar.todo")) || mimeTypes.contains(QLatin1String("application/x-vnd.akonadi.calendar.journal")) || mimeTypes.contains(QLatin1String("application/x-vnd.akonadi.calendar.freebusy"))) { query.setType(QStringLiteral("Calendar")); t = recursiveCalendarTermMapping(term); } else { // Unknown type return {}; } if (searchQuery.limit() > 0) { query.setLimit(searchQuery.limit()); } //Filter by collection if not empty if (!collections.isEmpty()) { Term parentTerm(Term::And); Term collectionTerm(Term::Or); for (const qint64 col : collections) { collectionTerm.addSubTerm(Term(QStringLiteral("collection"), QString::number(col), Term::Equal)); } if (t.isEmpty()) { query.setTerm(collectionTerm); } else { parentTerm.addSubTerm(collectionTerm); parentTerm.addSubTerm(t); query.setTerm(parentTerm); } } else { if (t.subTerms().isEmpty()) { qCWarning(AKONADIPLUGIN_INDEXER_LOG) << "no terms added"; return QSet(); } query.setTerm(t); } QSet resultSet; // qCDebug(AKONADIPLUGIN_INDEXER_LOG) << query.toJSON(); ResultIterator iter = query.exec(); while (iter.next()) { const QByteArray id = iter.id(); const int fid = deserialize("akonadi", id); resultSet << fid; } qCDebug(AKONADIPLUGIN_INDEXER_LOG) << "Got" << resultSet.count() << "results"; return resultSet; } diff --git a/lib/tests/contactcompletiontest.cpp b/lib/tests/contactcompletiontest.cpp index 885a9bd..f9267fd 100644 --- a/lib/tests/contactcompletiontest.cpp +++ b/lib/tests/contactcompletiontest.cpp @@ -1,83 +1,84 @@ /* * This file is part of the KDE Akonadi Search Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "../contactcompleter.h" #include #include #include +#include #include #include #include #include using namespace Akonadi::Search::PIM; class App : public QCoreApplication { Q_OBJECT public: App(int &argc, char **argv, int flags = ApplicationFlags); QString m_query; private Q_SLOTS: void main(); }; int main(int argc, char **argv) { App app(argc, argv); if (argc != 2) { qWarning() << "Proper args required"; exit(0); } app.m_query = QString::fromUtf8(argv[1]); return app.exec(); } App::App(int &argc, char **argv, int flags): QCoreApplication(argc, argv, flags) { QTimer::singleShot(0, this, &App::main); } void App::main() { ContactCompleter com(m_query, 100); - QTime timer; + QElapsedTimer timer; timer.start(); const QStringList emails = com.complete(); for (const QString &em : qAsConst(emails)) { std::cout << em.toUtf8().data() << std::endl; } qDebug() << timer.elapsed(); quit(); } #include "contactcompletiontest.moc" diff --git a/search/email/agepostingsource.cpp b/search/email/agepostingsource.cpp index 0bbf004..3c93c7a 100644 --- a/search/email/agepostingsource.cpp +++ b/search/email/agepostingsource.cpp @@ -1,72 +1,72 @@ /* * This file is part of the KDE Akonadi Search Project * Copyright (C) 2013 Vishesh Handa * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) version 3, or any * later version accepted by the membership of KDE e.V. (or its * successor approved by the membership of KDE e.V.), which shall * act as a proxy defined in Section 6 of version 3 of the license. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library. If not, see . * */ #include "agepostingsource.h" #include #include #include using namespace Akonadi::Search; AgePostingSource::AgePostingSource(Xapian::valueno slot_) : Xapian::ValuePostingSource(slot_) { - m_currentTime_t = QDateTime::currentDateTimeUtc().toTime_t(); + m_currentTime_t = QDateTime::currentDateTimeUtc().toSecsSinceEpoch(); } Xapian::weight AgePostingSource::get_weight() const { std::string s = *value_it; QString str = QString::fromUtf8(s.c_str(), s.length()); bool ok = false; uint time = str.toUInt(&ok); if (!ok) { return 0.0; } uint diff = m_currentTime_t - time; // Each day is given a penalty of penalty of 1.0 double penalty = 1.0 / (24 * 60 * 60); double result = 1000.0 - (diff * penalty); if (result < 0.0) { return 0.0; } return result; } Xapian::PostingSource *AgePostingSource::clone() const { return new AgePostingSource(slot); } void AgePostingSource::init(const Xapian::Database &db_) { Xapian::ValuePostingSource::init(db_); set_maxweight(1000.0); }