diff --git a/messagecomposer/src/helper/messagefactoryng.cpp b/messagecomposer/src/helper/messagefactoryng.cpp index 1051135d..07fe4c32 100644 --- a/messagecomposer/src/helper/messagefactoryng.cpp +++ b/messagecomposer/src/helper/messagefactoryng.cpp @@ -1,1006 +1,1006 @@ /* Copyright (C) 2010 Klarälvdalens Datakonsult AB, a KDAB Group company, info@kdab.com Copyright (c) 2010 Leo Franchi Copyright (C) 2017-2020 Laurent Montel This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "messagefactoryng.h" #include "settings/messagecomposersettings.h" #include "messagefactoryforwardjob.h" #include "messagefactoryreplyjob.h" #include "MessageComposer/Util" #include #ifndef QT_NO_CURSOR #include #endif #include #include #include #include #include #include #include #include "helper/messagehelper.h" #include #include "messagecomposer_debug.h" #include #include using namespace MessageComposer; namespace KMime { namespace Types { static bool operator==(const KMime::Types::Mailbox &left, const KMime::Types::Mailbox &right) { return left.addrSpec().asString() == right.addrSpec().asString(); } } } /** * Strips all the user's addresses from an address list. This is used * when replying. */ static KMime::Types::Mailbox::List stripMyAddressesFromAddressList(const KMime::Types::Mailbox::List &list, const KIdentityManagement::IdentityManager *manager) { KMime::Types::Mailbox::List addresses(list); for (KMime::Types::Mailbox::List::Iterator it = addresses.begin(); it != addresses.end();) { if (manager->thatIsMe(it->prettyAddress())) { it = addresses.erase(it); } else { ++it; } } return addresses; } MessageFactoryNG::MessageFactoryNG(const KMime::Message::Ptr &origMsg, Akonadi::Item::Id id, const Akonadi::Collection &col, QObject *parent) : QObject(parent) , m_identityManager(nullptr) , m_origMsg(origMsg) , m_folderId(0) , m_parentFolderId(0) , m_collection(col) , m_replyStrategy(MessageComposer::ReplySmart) , m_quote(true) , m_id(id) { } MessageFactoryNG::~MessageFactoryNG() { } // Return the addresses to use when replying to the author of msg. // See . static KMime::Types::Mailbox::List authorMailboxes( - const KMime::Message::Ptr &msg, KMime::Types::Mailbox::List mailingLists) + const KMime::Message::Ptr &msg, const KMime::Types::Mailbox::List &mailingLists) { if (auto mrt = msg->headerByType("Mail-Reply-To")) { return KMime::Types::Mailbox::listFrom7BitString(mrt->as7BitString(false)); } if (auto rt = msg->replyTo(false)) { // Did a mailing list munge Reply-To? auto mboxes = rt->mailboxes(); for (const auto &list : mailingLists) { mboxes.removeAll(list); } if (!mboxes.isEmpty()) { return mboxes; } } return msg->from(true)->mailboxes(); } void MessageFactoryNG::slotCreateReplyDone(const KMime::Message::Ptr &msg, bool replyAll) { applyCharset(msg); MessageComposer::Util::addLinkInformation(msg, m_id, Akonadi::MessageStatus::statusReplied()); if (m_parentFolderId > 0) { KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-Fcc"); header->fromUnicodeString(QString::number(m_parentFolderId), "utf-8"); msg->setHeader(header); } if (auto hrd = m_origMsg->headerByType("X-KMail-EncryptActionEnabled")) { if (hrd->as7BitString(false).contains("true")) { auto header = new KMime::Headers::Generic("X-KMail-EncryptActionEnabled"); header->fromUnicodeString(QStringLiteral("true"), "utf-8"); msg->setHeader(header); } } msg->assemble(); MessageReply reply; reply.msg = msg; reply.replyAll = replyAll; Q_EMIT createReplyDone(reply); } void MessageFactoryNG::createReplyAsync() { KMime::Message::Ptr msg(new KMime::Message); QByteArray refStr; bool replyAll = true; KMime::Types::Mailbox::List toList; KMime::Types::Mailbox::List replyToList; const uint originalIdentity = identityUoid(m_origMsg); MessageHelper::initFromMessage(msg, m_origMsg, m_identityManager, originalIdentity); replyToList = m_origMsg->replyTo()->mailboxes(); msg->contentType()->setCharset("utf-8"); if (auto hdr = m_origMsg->headerByType("List-Post")) { const QString hdrListPost = hdr->asUnicodeString(); if (hdrListPost.contains(QLatin1String("mailto:"), Qt::CaseInsensitive)) { QRegExp rx(QStringLiteral("]+)@([^>]+)>"), Qt::CaseInsensitive); if (rx.indexIn(hdrListPost, 0) != -1) { // matched KMime::Types::Mailbox mailbox; mailbox.fromUnicodeString(rx.cap(1) + QLatin1Char('@') + rx.cap(2)); m_mailingListAddresses << mailbox; } } } switch (m_replyStrategy) { case MessageComposer::ReplySmart: { if (auto hdr = m_origMsg->headerByType("Mail-Followup-To")) { toList << KMime::Types::Mailbox::listFrom7BitString(hdr->as7BitString(false)); } else if (!m_mailingListAddresses.isEmpty()) { if (replyToList.isEmpty()) { toList = (KMime::Types::Mailbox::List() << m_mailingListAddresses.at(0)); } else { toList = replyToList; } } else { // Doesn't seem to be a mailing list. auto originalFromList = m_origMsg->from()->mailboxes(); auto originalToList = m_origMsg->to()->mailboxes(); if (m_identityManager->thatIsMe(KMime::Types::Mailbox::listToUnicodeString(originalFromList)) && !m_identityManager->thatIsMe(KMime::Types::Mailbox::listToUnicodeString(originalToList)) ) { // Sender seems to be one of our own identities and recipient is not, // so we assume that this is a reply to a "sent" mail where the user // wants to add additional information for the recipient. toList = originalToList; } else { // "Normal" case: reply to sender. toList = authorMailboxes(m_origMsg, m_mailingListAddresses); } replyAll = false; } // strip all my addresses from the list of recipients const KMime::Types::Mailbox::List recipients = toList; toList = stripMyAddressesFromAddressList(recipients, m_identityManager); // ... unless the list contains only my addresses (reply to self) if (toList.isEmpty() && !recipients.isEmpty()) { toList << recipients.first(); } break; } case MessageComposer::ReplyList: { if (auto hdr = m_origMsg->headerByType("Mail-Followup-To")) { KMime::Types::Mailbox mailbox; mailbox.from7BitString(hdr->as7BitString(false)); toList << mailbox; } else if (!m_mailingListAddresses.isEmpty()) { toList << m_mailingListAddresses[ 0 ]; } else if (!replyToList.isEmpty()) { // assume a Reply-To header mangling mailing list toList = replyToList; } // strip all my addresses from the list of recipients const KMime::Types::Mailbox::List recipients = toList; toList = stripMyAddressesFromAddressList(recipients, m_identityManager); break; } case MessageComposer::ReplyAll: if (auto hdr = m_origMsg->headerByType("Mail-Followup-To")) { toList = KMime::Types::Mailbox::listFrom7BitString(hdr->as7BitString(false)); } else { auto ccList = stripMyAddressesFromAddressList(m_origMsg->cc()->mailboxes(), m_identityManager); if (!m_mailingListAddresses.isEmpty()) { toList = stripMyAddressesFromAddressList(m_origMsg->to()->mailboxes(), m_identityManager); bool addMailingList = true; - for (const KMime::Types::Mailbox &m : m_mailingListAddresses) { + for (const KMime::Types::Mailbox &m : qAsConst(m_mailingListAddresses)) { if (toList.contains(m)) { addMailingList = false; break; } } if (addMailingList) { toList += m_mailingListAddresses.front(); } ccList += authorMailboxes(m_origMsg, m_mailingListAddresses); } else { // Doesn't seem to be a mailing list. auto originalFromList = m_origMsg->from()->mailboxes(); auto originalToList = m_origMsg->to()->mailboxes(); if (m_identityManager->thatIsMe(KMime::Types::Mailbox::listToUnicodeString(originalFromList)) && !m_identityManager->thatIsMe(KMime::Types::Mailbox::listToUnicodeString(originalToList)) ) { // Sender seems to be one of our own identities and recipient is not, // so we assume that this is a reply to a "sent" mail where the user // wants to add additional information for the recipient. toList = originalToList; } else { // "Normal" case: reply to sender. toList = stripMyAddressesFromAddressList(m_origMsg->to()->mailboxes(), m_identityManager); toList += authorMailboxes(m_origMsg, m_mailingListAddresses); } } - for (const KMime::Types::Mailbox &mailbox : ccList) { + for (const KMime::Types::Mailbox &mailbox : qAsConst(ccList)) { msg->cc()->addAddress(mailbox); } } break; case MessageComposer::ReplyAuthor: toList = authorMailboxes(m_origMsg, m_mailingListAddresses); replyAll = false; break; case MessageComposer::ReplyNone: // the addressees will be set by the caller break; default: Q_UNREACHABLE(); } for (const KMime::Types::Mailbox &mailbox : qAsConst(toList)) { msg->to()->addAddress(mailbox); } refStr = getRefStr(m_origMsg); if (!refStr.isEmpty()) { msg->references()->fromUnicodeString(QString::fromLocal8Bit(refStr), "utf-8"); } //In-Reply-To = original msg-id msg->inReplyTo()->from7BitString(m_origMsg->messageID()->as7BitString(false)); msg->subject()->fromUnicodeString(MessageCore::StringUtil::replySubject(m_origMsg.data()), "utf-8"); // If the reply shouldn't be blank, apply the template to the message if (m_quote) { MessageFactoryReplyJob *job = new MessageFactoryReplyJob; connect(job, &MessageFactoryReplyJob::replyDone, this, &MessageFactoryNG::slotCreateReplyDone); job->setMsg(msg); job->setReplyAll(replyAll); job->setIdentityManager(m_identityManager); job->setSelection(m_selection); job->setTemplate(m_template); job->setOrigMsg(m_origMsg); job->setCollection(m_collection); job->start(); } else { slotCreateReplyDone(msg, replyAll); } } void MessageFactoryNG::slotCreateForwardDone(const KMime::Message::Ptr &msg) { applyCharset(msg); MessageComposer::Util::addLinkInformation(msg, m_id, Akonadi::MessageStatus::statusForwarded()); msg->assemble(); Q_EMIT createForwardDone(msg); } void MessageFactoryNG::createForwardAsync() { KMime::Message::Ptr msg(new KMime::Message); // This is a non-multipart, non-text mail (e.g. text/calendar). Construct // a multipart/mixed mail and add the original body as an attachment. if (!m_origMsg->contentType()->isMultipart() && (!m_origMsg->contentType()->isText() || (m_origMsg->contentType()->isText() && m_origMsg->contentType()->subType() != "html" && m_origMsg->contentType()->subType() != "plain"))) { const uint originalIdentity = identityUoid(m_origMsg); MessageHelper::initFromMessage(msg, m_origMsg, m_identityManager, originalIdentity); msg->removeHeader(); msg->removeHeader(); msg->contentType()->setMimeType("multipart/mixed"); //TODO: Andras: somebody should check if this is correct. :) // empty text part KMime::Content *msgPart = new KMime::Content; msgPart->contentType()->setMimeType("text/plain"); msg->addContent(msgPart); // the old contents of the mail KMime::Content *secondPart = new KMime::Content; secondPart->contentType()->setMimeType(m_origMsg->contentType()->mimeType()); secondPart->setBody(m_origMsg->body()); // use the headers of the original mail secondPart->setHead(m_origMsg->head()); msg->addContent(secondPart); msg->assemble(); } // Normal message (multipart or text/plain|html) // Just copy the message, the template parser will do the hard work of // replacing the body text in TemplateParser::addProcessedBodyToMessage() else { //TODO Check if this is ok msg->setHead(m_origMsg->head()); msg->setBody(m_origMsg->body()); QString oldContentType = msg->contentType()->asUnicodeString(); const uint originalIdentity = identityUoid(m_origMsg); MessageHelper::initFromMessage(msg, m_origMsg, m_identityManager, originalIdentity); // restore the content type, MessageHelper::initFromMessage() sets the contents type to // text/plain, via initHeader(), for unclear reasons msg->contentType()->fromUnicodeString(oldContentType, "utf-8"); msg->assemble(); } msg->subject()->fromUnicodeString(MessageCore::StringUtil::forwardSubject(m_origMsg.data()), "utf-8"); MessageFactoryForwardJob *job = new MessageFactoryForwardJob; connect(job, &MessageFactoryForwardJob::forwardDone, this, &MessageFactoryNG::slotCreateForwardDone); job->setIdentityManager(m_identityManager); job->setMsg(msg); job->setSelection(m_selection); job->setTemplate(m_template); job->setOrigMsg(m_origMsg); job->setCollection(m_collection); job->start(); } QPair< KMime::Message::Ptr, QVector< KMime::Content * > > MessageFactoryNG::createAttachedForward(const Akonadi::Item::List &items) { // create forwarded message with original message as attachment // remove headers that shouldn't be forwarded KMime::Message::Ptr msg(new KMime::Message); QVector< KMime::Content * > attachments; const int numberOfItems(items.count()); if (numberOfItems >= 2) { // don't respect X-KMail-Identity headers because they might differ for // the selected mails MessageHelper::initHeader(msg, m_identityManager, 0); } else if (numberOfItems == 1) { KMime::Message::Ptr firstMsg = MessageComposer::Util::message(items.first()); const uint originalIdentity = identityUoid(firstMsg); MessageHelper::initFromMessage(msg, firstMsg, m_identityManager, originalIdentity); msg->subject()->fromUnicodeString(MessageCore::StringUtil::forwardSubject(firstMsg.data()), "utf-8"); } MessageHelper::setAutomaticFields(msg, true); #ifndef QT_NO_CURSOR KPIM::KCursorSaver busy(KPIM::KBusyPtr::busy()); #endif if (numberOfItems == 0) { attachments << createForwardAttachmentMessage(m_origMsg); MessageComposer::Util::addLinkInformation(msg, m_id, Akonadi::MessageStatus::statusForwarded()); } else { // iterate through all the messages to be forwarded attachments.reserve(items.count()); for (const Akonadi::Item &item : qAsConst(items)) { attachments << createForwardAttachmentMessage(MessageComposer::Util::message(item)); MessageComposer::Util::addLinkInformation(msg, item.id(), Akonadi::MessageStatus::statusForwarded()); } } applyCharset(msg); //msg->assemble(); return QPair< KMime::Message::Ptr, QVector< KMime::Content * > >(msg, QVector< KMime::Content * >() << attachments); } KMime::Content *MessageFactoryNG::createForwardAttachmentMessage(const KMime::Message::Ptr &fwdMsg) { // remove headers that shouldn't be forwarded MessageCore::StringUtil::removePrivateHeaderFields(fwdMsg); fwdMsg->removeHeader(); fwdMsg->assemble(); // set the part KMime::Content *msgPart = new KMime::Content(fwdMsg.data()); msgPart->contentType()->setMimeType("message/rfc822"); msgPart->contentDisposition()->setParameter(QStringLiteral("filename"), i18n("forwarded message")); msgPart->contentDisposition()->setDisposition(KMime::Headers::CDinline); msgPart->contentDescription()->fromUnicodeString(fwdMsg->from()->asUnicodeString() + QLatin1String(": ") + fwdMsg->subject()->asUnicodeString(), "utf-8"); msgPart->setBody(fwdMsg->encodedContent()); msgPart->assemble(); MessageComposer::Util::addLinkInformation(fwdMsg, 0, Akonadi::MessageStatus::statusForwarded()); return msgPart; } KMime::Message::Ptr MessageFactoryNG::createResend() { KMime::Message::Ptr msg(new KMime::Message); msg->setContent(m_origMsg->encodedContent()); msg->parse(); msg->removeHeader(); uint originalIdentity = identityUoid(m_origMsg); // Set the identity from above KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-Identity"); header->fromUnicodeString(QString::number(originalIdentity), "utf-8"); msg->setHeader(header); // Restore the original bcc field as this is overwritten in applyIdentity msg->bcc(m_origMsg->bcc()); return msg; } KMime::Message::Ptr MessageFactoryNG::createRedirect(const QString &toStr, const QString &ccStr, const QString &bccStr, int transportId, const QString &fcc, int identity) { if (!m_origMsg) { return KMime::Message::Ptr(); } // copy the message 1:1 KMime::Message::Ptr msg(new KMime::Message); msg->setContent(m_origMsg->encodedContent()); msg->parse(); uint id = identity; if (identity == -1) { if (auto hrd = msg->headerByType("X-KMail-Identity")) { const QString strId = hrd->asUnicodeString().trimmed(); if (!strId.isEmpty()) { id = strId.toUInt(); } } } const KIdentityManagement::Identity &ident = m_identityManager->identityForUoidOrDefault(id); // X-KMail-Redirect-From: content const QString strByWayOf = QString::fromLocal8Bit("%1 (by way of %2 <%3>)") .arg(m_origMsg->from()->asUnicodeString(), ident.fullName(), ident.primaryEmailAddress()); // Resent-From: content const QString strFrom = QString::fromLocal8Bit("%1 <%2>") .arg(ident.fullName(), ident.primaryEmailAddress()); // format the current date to be used in Resent-Date: // FIXME: generate datetime the same way as KMime, otherwise we get inconsistency // in unit-tests. Unfortunatelly RFC2822Date is not enough for us, we need the // composition hack below const QDateTime dt = QDateTime::currentDateTime(); const QString newDate = QLocale::c().toString(dt, QStringLiteral("ddd, ")) +dt.toString(Qt::RFC2822Date); // Clean up any resent headers msg->removeHeader("Resent-Cc"); msg->removeHeader("Resent-Bcc"); msg->removeHeader("Resent-Sender"); // date, from to and id will be set anyway // prepend Resent-*: headers (c.f. RFC2822 3.6.6) QString msgIdSuffix; if (MessageComposer::MessageComposerSettings::useCustomMessageIdSuffix()) { msgIdSuffix = MessageComposer::MessageComposerSettings::customMsgIDSuffix(); } KMime::Headers::Generic *header = new KMime::Headers::Generic("Resent-Message-ID"); header->fromUnicodeString(MessageCore::StringUtil::generateMessageId(msg->sender()->asUnicodeString(), msgIdSuffix), "utf-8"); msg->setHeader(header); header = new KMime::Headers::Generic("Resent-Date"); header->fromUnicodeString(newDate, "utf-8"); msg->setHeader(header); header = new KMime::Headers::Generic("Resent-From"); header->fromUnicodeString(strFrom, "utf-8"); msg->setHeader(header); if (msg->to(false)) { KMime::Headers::To *headerT = new KMime::Headers::To; headerT->fromUnicodeString(m_origMsg->to()->asUnicodeString(), "utf-8"); msg->setHeader(headerT); } header = new KMime::Headers::Generic("Resent-To"); header->fromUnicodeString(toStr, "utf-8"); msg->setHeader(header); if (!ccStr.isEmpty()) { header = new KMime::Headers::Generic("Resent-Cc"); header->fromUnicodeString(ccStr, "utf-8"); msg->setHeader(header); } if (!bccStr.isEmpty()) { header = new KMime::Headers::Generic("Resent-Bcc"); header->fromUnicodeString(bccStr, "utf-8"); msg->setHeader(header); } header = new KMime::Headers::Generic("X-KMail-Redirect-From"); header->fromUnicodeString(strByWayOf, "utf-8"); msg->setHeader(header); if (transportId != -1) { header = new KMime::Headers::Generic("X-KMail-Transport"); header->fromUnicodeString(QString::number(transportId), "utf-8"); msg->setHeader(header); } if (!fcc.isEmpty()) { header = new KMime::Headers::Generic("X-KMail-Fcc"); header->fromUnicodeString(fcc, "utf-8"); msg->setHeader(header); } const bool fccIsDisabled = ident.disabledFcc(); if (fccIsDisabled) { KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-FccDisabled"); header->fromUnicodeString(QStringLiteral("true"), "utf-8"); msg->setHeader(header); } else { msg->removeHeader("X-KMail-FccDisabled"); } msg->assemble(); MessageComposer::Util::addLinkInformation(msg, m_id, Akonadi::MessageStatus::statusForwarded()); return msg; } KMime::Message::Ptr MessageFactoryNG::createDeliveryReceipt() { QString receiptTo; if (auto hrd = m_origMsg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return KMime::Message::Ptr(); } receiptTo.remove(QChar::fromLatin1('\n')); KMime::Message::Ptr receipt(new KMime::Message); const uint originalIdentity = identityUoid(m_origMsg); MessageHelper::initFromMessage(receipt, m_origMsg, m_identityManager, originalIdentity); receipt->to()->fromUnicodeString(receiptTo, QStringLiteral("utf-8").toLatin1()); receipt->subject()->fromUnicodeString(i18n("Receipt: ") + m_origMsg->subject()->asUnicodeString(), "utf-8"); QString str = QStringLiteral("Your message was successfully delivered."); str += QLatin1String("\n\n---------- Message header follows ----------\n"); str += QString::fromLatin1(m_origMsg->head()); str += QLatin1String("--------------------------------------------\n"); // Conversion to toLatin1 is correct here as Mail headers should contain // ascii only receipt->setBody(str.toLatin1()); MessageHelper::setAutomaticFields(receipt); receipt->assemble(); return receipt; } KMime::Message::Ptr MessageFactoryNG::createMDN(KMime::MDN::ActionMode a, KMime::MDN::DispositionType d, KMime::MDN::SendingMode s, int mdnQuoteOriginal, const QVector &m) { // extract where to send to: QString receiptTo; if (auto hrd = m_origMsg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return KMime::Message::Ptr(new KMime::Message); } receiptTo.remove(QChar::fromLatin1('\n')); QString special; // fill in case of error, warning or failure // extract where to send from: QString finalRecipient = m_identityManager->identityForUoidOrDefault(identityUoid(m_origMsg)).fullEmailAddr(); // // Generate message: // KMime::Message::Ptr receipt(new KMime::Message()); const uint originalIdentity = identityUoid(m_origMsg); MessageHelper::initFromMessage(receipt, m_origMsg, m_identityManager, originalIdentity); receipt->contentType()->from7BitString("multipart/report"); receipt->contentType()->setBoundary(KMime::multiPartBoundary()); receipt->contentType()->setCharset("us-ascii"); receipt->removeHeader(); // Modify the ContentType directly (replaces setAutomaticFields(true)) receipt->contentType()->setParameter(QStringLiteral("report-type"), QStringLiteral("disposition-notification")); QString description = replaceHeadersInString(m_origMsg, KMime::MDN::descriptionFor(d, m)); // text/plain part: KMime::Content *firstMsgPart = new KMime::Content(m_origMsg.data()); firstMsgPart->contentType()->setMimeType("text/plain"); firstMsgPart->contentType()->setCharset("utf-8"); firstMsgPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE7Bit); firstMsgPart->setBody(description.toUtf8()); receipt->addContent(firstMsgPart); // message/disposition-notification part: KMime::Content *secondMsgPart = new KMime::Content(m_origMsg.data()); secondMsgPart->contentType()->setMimeType("message/disposition-notification"); secondMsgPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE7Bit); QByteArray originalRecipient = ""; if (auto hrd = m_origMsg->headerByType("Original-Recipient")) { originalRecipient = hrd->as7BitString(false); } secondMsgPart->setBody(KMime::MDN::dispositionNotificationBodyContent( finalRecipient, originalRecipient, m_origMsg->messageID()->as7BitString(false), /* Message-ID */ d, a, s, m, special)); receipt->addContent(secondMsgPart); if (mdnQuoteOriginal < 0 || mdnQuoteOriginal > 2) { mdnQuoteOriginal = 0; } /* 0=> Nothing, 1=>Full Message, 2=>HeadersOnly*/ KMime::Content *thirdMsgPart = new KMime::Content(m_origMsg.data()); switch (mdnQuoteOriginal) { case 1: thirdMsgPart->contentType()->setMimeType("message/rfc822"); thirdMsgPart->setBody(MessageCore::StringUtil::asSendableString(m_origMsg)); receipt->addContent(thirdMsgPart); break; case 2: thirdMsgPart->contentType()->setMimeType("text/rfc822-headers"); thirdMsgPart->setBody(MessageCore::StringUtil::headerAsSendableString(m_origMsg)); receipt->addContent(thirdMsgPart); break; case 0: default: delete thirdMsgPart; break; } receipt->to()->fromUnicodeString(receiptTo, "utf-8"); //Laurent: We don't translate subject ? receipt->subject()->from7BitString("Message Disposition Notification"); KMime::Headers::InReplyTo *header = new KMime::Headers::InReplyTo; header->fromUnicodeString(m_origMsg->messageID()->asUnicodeString(), "utf-8"); receipt->setHeader(header); receipt->references()->from7BitString(getRefStr(m_origMsg)); receipt->assemble(); qCDebug(MESSAGECOMPOSER_LOG) << "final message:" + receipt->encodedContent(); receipt->assemble(); return receipt; } QPair< KMime::Message::Ptr, KMime::Content * > MessageFactoryNG::createForwardDigestMIME(const Akonadi::Item::List &items) { KMime::Message::Ptr msg(new KMime::Message); KMime::Content *digest = new KMime::Content(msg.data()); QString mainPartText = i18n("\nThis is a MIME digest forward. The content of the" " message is contained in the attachment(s).\n\n\n"); digest->contentType()->setMimeType("multipart/digest"); digest->contentType()->setBoundary(KMime::multiPartBoundary()); digest->contentDescription()->fromUnicodeString(QStringLiteral("Digest of %1 messages.").arg(items.count()), "utf8"); digest->contentDisposition()->setFilename(QStringLiteral("digest")); digest->fromUnicodeString(mainPartText); int id = 0; for (const Akonadi::Item &item : qAsConst(items)) { KMime::Message::Ptr fMsg = MessageComposer::Util::message(item); if (id == 0) { if (auto hrd = fMsg->headerByType("X-KMail-Identity")) { id = hrd->asUnicodeString().toInt(); } } MessageCore::StringUtil::removePrivateHeaderFields(fMsg); fMsg->removeHeader(); fMsg->assemble(); KMime::Content *part = new KMime::Content(digest); part->contentType()->setMimeType("message/rfc822"); part->contentType()->setCharset(fMsg->contentType()->charset()); part->contentID()->setIdentifier(fMsg->contentID()->identifier()); part->contentDescription()->fromUnicodeString(fMsg->contentDescription()->asUnicodeString(), "utf8"); part->contentDisposition()->setParameter(QStringLiteral("name"), i18n("forwarded message")); part->fromUnicodeString(QString::fromLatin1(fMsg->encodedContent())); part->assemble(); MessageComposer::Util::addLinkInformation(msg, item.id(), Akonadi::MessageStatus::statusForwarded()); digest->addContent(part); } digest->assemble(); id = m_folderId; MessageHelper::initHeader(msg, m_identityManager, id); // qCDebug(MESSAGECOMPOSER_LOG) << "digest:" << digest->contents().size() << digest->encodedContent(); return QPair< KMime::Message::Ptr, KMime::Content * >(msg, digest); } void MessageFactoryNG::setIdentityManager(KIdentityManagement::IdentityManager *ident) { m_identityManager = ident; } void MessageFactoryNG::setReplyStrategy(MessageComposer::ReplyStrategy replyStrategy) { m_replyStrategy = replyStrategy; } void MessageFactoryNG::setSelection(const QString &selection) { m_selection = selection; } void MessageFactoryNG::setQuote(bool quote) { m_quote = quote; } void MessageFactoryNG::setTemplate(const QString &templ) { m_template = templ; } void MessageFactoryNG::setMailingListAddresses(const KMime::Types::Mailbox::List &listAddresses) { m_mailingListAddresses << listAddresses; } void MessageFactoryNG::setFolderIdentity(Akonadi::Collection::Id folderIdentityId) { m_folderId = folderIdentityId; } void MessageFactoryNG::putRepliesInSameFolder(Akonadi::Collection::Id parentColId) { m_parentFolderId = parentColId; } bool MessageFactoryNG::MDNRequested(const KMime::Message::Ptr &msg) { // extract where to send to: QString receiptTo; if (auto hrd = msg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return false; } receiptTo.remove(QChar::fromLatin1('\n')); return !receiptTo.isEmpty(); } bool MessageFactoryNG::MDNConfirmMultipleRecipients(const KMime::Message::Ptr &msg) { // extract where to send to: QString receiptTo; if (auto hrd = msg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return false; } receiptTo.remove(QChar::fromLatin1('\n')); // RFC 2298: [ Confirmation from the user SHOULD be obtained (or no // MDN sent) ] if there is more than one distinct address in the // Disposition-Notification-To header. qCDebug(MESSAGECOMPOSER_LOG) << "KEmailAddress::splitAddressList(receiptTo):" << KEmailAddress::splitAddressList(receiptTo).join(QLatin1Char('\n')); return KEmailAddress::splitAddressList(receiptTo).count() > 1; } bool MessageFactoryNG::MDNReturnPathEmpty(const KMime::Message::Ptr &msg) { // extract where to send to: QString receiptTo; if (auto hrd = msg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return false; } receiptTo.remove(QChar::fromLatin1('\n')); // RFC 2298: MDNs SHOULD NOT be sent automatically if the address in // the Disposition-Notification-To header differs from the address // in the Return-Path header. [...] Confirmation from the user // SHOULD be obtained (or no MDN sent) if there is no Return-Path // header in the message [...] KMime::Types::AddrSpecList returnPathList = MessageHelper::extractAddrSpecs(msg, "Return-Path"); QString returnPath = returnPathList.isEmpty() ? QString() : returnPathList.front().localPart + QChar::fromLatin1('@') + returnPathList.front().domain; qCDebug(MESSAGECOMPOSER_LOG) << "clean return path:" << returnPath; return returnPath.isEmpty(); } bool MessageFactoryNG::MDNReturnPathNotInRecieptTo(const KMime::Message::Ptr &msg) { // extract where to send to: QString receiptTo; if (auto hrd = msg->headerByType("Disposition-Notification-To")) { receiptTo = hrd->asUnicodeString(); } if (receiptTo.trimmed().isEmpty()) { return false; } receiptTo.remove(QChar::fromLatin1('\n')); // RFC 2298: MDNs SHOULD NOT be sent automatically if the address in // the Disposition-Notification-To header differs from the address // in the Return-Path header. [...] Confirmation from the user // SHOULD be obtained (or no MDN sent) if there is no Return-Path // header in the message [...] KMime::Types::AddrSpecList returnPathList = MessageHelper::extractAddrSpecs(msg, QStringLiteral("Return-Path").toLatin1()); QString returnPath = returnPathList.isEmpty() ? QString() : returnPathList.front().localPart + QChar::fromLatin1('@') + returnPathList.front().domain; qCDebug(MESSAGECOMPOSER_LOG) << "clean return path:" << returnPath; return !receiptTo.contains(returnPath, Qt::CaseSensitive); } bool MessageFactoryNG::MDNMDNUnknownOption(const KMime::Message::Ptr &msg) { // RFC 2298: An importance of "required" indicates that // interpretation of the parameter is necessary for proper // generation of an MDN in response to this request. If a UA does // not understand the meaning of the parameter, it MUST NOT generate // an MDN with any disposition type other than "failed" in response // to the request. QString notificationOptions; if (auto hrd = msg->headerByType("Disposition-Notification-Options")) { notificationOptions = hrd->asUnicodeString(); } if (notificationOptions.contains(QLatin1String("required"), Qt::CaseSensitive)) { // ### hacky; should parse... // There is a required option that we don't understand. We need to // ask the user what we should do: return true; } return false; } uint MessageFactoryNG::identityUoid(const KMime::Message::Ptr &msg) { QString idString; if (auto hdr = msg->headerByType("X-KMail-Identity")) { idString = hdr->asUnicodeString().trimmed(); } bool ok = false; uint id = idString.toUInt(&ok); if (!ok || id == 0) { id = m_identityManager->identityForAddress(msg->to()->asUnicodeString() + QLatin1String(", ") + msg->cc()->asUnicodeString()).uoid(); } if (id == 0 && m_folderId > 0) { id = m_folderId; } return id; } QString MessageFactoryNG::replaceHeadersInString(const KMime::Message::Ptr &msg, const QString &s) { QString result = s; QRegExp rx(QStringLiteral("\\$\\{([a-z0-9-]+)\\}"), Qt::CaseInsensitive); Q_ASSERT(rx.isValid()); const QString sDate = KMime::DateFormatter::formatDate( KMime::DateFormatter::Localized, msg->date()->dateTime().toSecsSinceEpoch()); qCDebug(MESSAGECOMPOSER_LOG) << "creating mdn date:" << msg->date()->dateTime().toSecsSinceEpoch() << sDate; result.replace(QStringLiteral("${date}"), sDate); int idx = 0; while ((idx = rx.indexIn(result, idx)) != -1) { const QByteArray ba = rx.cap(1).toLatin1(); QString replacement; if (auto hrd = msg->headerByType(ba.constData())) { replacement = hrd->asUnicodeString(); } result.replace(idx, rx.matchedLength(), replacement); idx += replacement.length(); } return result; } void MessageFactoryNG::applyCharset(const KMime::Message::Ptr msg) { if (MessageComposer::MessageComposerSettings::forceReplyCharset()) { // first convert the body from its current encoding to unicode representation QTextCodec *bodyCodec = KCharsets::charsets()->codecForName(QString::fromLatin1(msg->contentType()->charset())); if (!bodyCodec) { bodyCodec = KCharsets::charsets()->codecForName(QStringLiteral("UTF-8")); } const QString body = bodyCodec->toUnicode(msg->body()); // then apply the encoding of the original message msg->contentType()->setCharset(m_origMsg->contentType()->charset()); QTextCodec *codec = KCharsets::charsets()->codecForName(QString::fromLatin1(msg->contentType()->charset())); if (!codec) { qCCritical(MESSAGECOMPOSER_LOG) << "Could not get text codec for charset" << msg->contentType()->charset(); } else if (!codec->canEncode(body)) { // charset can't encode body, fall back to preferred const QStringList charsets = MessageComposer::MessageComposerSettings::preferredCharsets(); QVector chars; chars.reserve(charsets.count()); for (const QString &charset : charsets) { chars << charset.toLatin1(); } QByteArray fallbackCharset = MessageComposer::Util::selectCharset(chars, body); if (fallbackCharset.isEmpty()) { // UTF-8 as fall-through fallbackCharset = "UTF-8"; } codec = KCharsets::charsets()->codecForName(QString::fromLatin1(fallbackCharset)); msg->setBody(codec->fromUnicode(body)); } else { msg->setBody(codec->fromUnicode(body)); } } } QByteArray MessageFactoryNG::getRefStr(const KMime::Message::Ptr &msg) { QByteArray firstRef, lastRef, refStr, retRefStr; int i, j; if (auto hdr = msg->references(false)) { refStr = hdr->as7BitString(false).trimmed(); } if (refStr.isEmpty()) { return msg->messageID()->as7BitString(false); } i = refStr.indexOf('<'); j = refStr.indexOf('>'); firstRef = refStr.mid(i, j - i + 1); if (!firstRef.isEmpty()) { retRefStr = firstRef + ' '; } i = refStr.lastIndexOf('<'); j = refStr.lastIndexOf('>'); lastRef = refStr.mid(i, j - i + 1); if (!lastRef.isEmpty() && lastRef != firstRef) { retRefStr += lastRef + ' '; } retRefStr += msg->messageID()->as7BitString(false); return retRefStr; } diff --git a/messagecomposer/src/snippet/convertsnippetvariablesjob.cpp b/messagecomposer/src/snippet/convertsnippetvariablesjob.cpp index 76ce7b52..cbcacffb 100644 --- a/messagecomposer/src/snippet/convertsnippetvariablesjob.cpp +++ b/messagecomposer/src/snippet/convertsnippetvariablesjob.cpp @@ -1,293 +1,293 @@ /* Copyright (C) 2019-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "convertsnippetvariablesjob.h" #include "messagecomposer_debug.h" #include "composer/composerviewinterface.h" #include #include #include #include using namespace MessageComposer; ConvertSnippetVariablesJob::ConvertSnippetVariablesJob(QObject *parent) : QObject(parent) { } ConvertSnippetVariablesJob::~ConvertSnippetVariablesJob() { delete mComposerViewInterface; } void ConvertSnippetVariablesJob::setText(const QString &str) { mText = str; } bool ConvertSnippetVariablesJob::canStart() const { if (mText.isEmpty() || !mComposerViewInterface) { return false; } return true; } void ConvertSnippetVariablesJob::start() { if (!canStart()) { Q_EMIT textConverted(QString()); deleteLater(); return; } Q_EMIT textConverted(convertVariables(mComposerViewInterface, mText)); deleteLater(); } QString ConvertSnippetVariablesJob::text() const { return mText; } MessageComposer::ComposerViewInterface *ConvertSnippetVariablesJob::composerViewInterface() const { return mComposerViewInterface; } void ConvertSnippetVariablesJob::setComposerViewInterface(MessageComposer::ComposerViewInterface *composerViewInterface) { mComposerViewInterface = composerViewInterface; } QString ConvertSnippetVariablesJob::convertVariables(const QString &cmd, int &i, QChar c) { QString result; if (cmd.startsWith(QLatin1String("LASTYEAR"))) { i += strlen("LASTYEAR"); const QDate date = QDate::currentDate(); result.append(QString::number(date.year() - 1)); } else if (cmd.startsWith(QLatin1String("NEXTYEAR"))) { i += strlen("NEXTYEAR"); const QDate date = QDate::currentDate(); result.append(QString::number(date.year() - 2)); } else if (cmd.startsWith(QLatin1String("MONTHNUMBER"))) { i += strlen("MONTHNUMBER"); const QDate date = QDate::currentDate(); result.append(QString::number(date.month())); } else if (cmd.startsWith(QLatin1String("DAYOFMONTH"))) { i += strlen("DAYOFMONTH"); const QDate date = QDate::currentDate(); result.append(QString::number(date.daysInMonth())); } else if (cmd.startsWith(QLatin1String("WEEKNUMBER"))) { i += strlen("WEEKNUMBER"); const QDate date = QDate::currentDate(); result.append(QString::number(date.weekNumber())); } else if (cmd.startsWith(QLatin1String("MONTHNAMESHORT"))) { i += strlen("MONTHNAMESHORT"); const QDate date = QDate::currentDate(); result.append(date.toString(QStringLiteral("MMM"))); } else if (cmd.startsWith(QLatin1String("MONTHNAMELONG"))) { i += strlen("MONTHNAMELONG"); const QDate date = QDate::currentDate(); result.append(date.toString(QStringLiteral("MMMM"))); } else if (cmd.startsWith(QLatin1String("DAYOFWEEKNAMESHORT"))) { i += strlen("DAYOFWEEKNAMESHORT"); const QDate date = QDate::currentDate(); result.append(date.toString(QStringLiteral("ddd"))); } else if (cmd.startsWith(QLatin1String("DAYOFWEEKNAMELONG"))) { i += strlen("DAYOFWEEKNAMELONG"); const QDate date = QDate::currentDate(); result.append(date.toString(QStringLiteral("dddd"))); } else if (cmd.startsWith(QLatin1String("YEARLASTMONTH"))) { i += strlen("YEARLASTMONTH"); const QDate date = QDate::currentDate().addMonths(-1); result.append(date.toString(QStringLiteral("yyyy-MMM"))); } else if (cmd.startsWith(QLatin1String("YEAR"))) { i += strlen("YEAR"); const QDate date = QDate::currentDate(); result.append(QString::number(date.year())); } else if (cmd.startsWith(QLatin1String("DAYOFWEEK"))) { i += strlen("DAYOFWEEK"); const QDate date = QDate::currentDate(); result.append(QString::number(date.dayOfWeek())); } else { result.append(c); } return result; } QString ConvertSnippetVariablesJob::convertVariables(MessageComposer::ComposerViewInterface *composerView, const QString &text) { QString result; const int tmpl_len = text.length(); for (int i = 0; i < tmpl_len; ++i) { const QChar c = text[i]; if (c == QLatin1Char('%')) { const QString cmd = text.mid(i + 1); if (composerView) { if (cmd.startsWith(QLatin1String("CCADDR"))) { i += strlen("CCADDR"); const QString str = composerView->cc(); result.append(str); } else if (cmd.startsWith(QLatin1String("CCFNAME"))) { i += strlen("CCFNAME"); const QString str = getFirstNameFromEmail(composerView->cc()); result.append(str); } else if (cmd.startsWith(QLatin1String("CCLNAME"))) { i += strlen("CCLNAME"); const QString str = getLastNameFromEmail(composerView->cc()); result.append(str); } else if (cmd.startsWith(QLatin1String("CCNAME"))) { i += strlen("CCNAME"); const QString str = getNameFromEmail(composerView->cc()); result.append(str); } else if (cmd.startsWith(QLatin1String("FULLSUBJECT"))) { i += strlen("FULLSUBJECT"); const QString str = composerView->subject(); result.append(str); } else if (cmd.startsWith(QLatin1String("TOADDR"))) { i += strlen("TOADDR"); const QString str = composerView->to(); result.append(str); } else if (cmd.startsWith(QLatin1String("TOFNAME"))) { i += strlen("TOFNAME"); const QString str = getFirstNameFromEmail(composerView->to()); result.append(str); } else if (cmd.startsWith(QLatin1String("TOLNAME"))) { i += strlen("TOLNAME"); const QString str = getLastNameFromEmail(composerView->to()); result.append(str); } else if (cmd.startsWith(QLatin1String("TONAME"))) { i += strlen("TONAME"); const QString str = getNameFromEmail(composerView->to()); result.append(str); } else if (cmd.startsWith(QLatin1String("FROMADDR"))) { i += strlen("FROMADDR"); const QString str = composerView->from(); result.append(str); } else if (cmd.startsWith(QLatin1String("FROMFNAME"))) { i += strlen("FROMFNAME"); const QString str = getFirstNameFromEmail(composerView->from()); result.append(str); } else if (cmd.startsWith(QLatin1String("FROMLNAME"))) { i += strlen("FROMLNAME"); const QString str = getLastNameFromEmail(composerView->from()); result.append(str); } else if (cmd.startsWith(QLatin1String("FROMNAME"))) { i += strlen("FROMNAME"); const QString str = getNameFromEmail(composerView->from()); result.append(str); } else if (cmd.startsWith(QLatin1String("DOW"))) { i += strlen("DOW"); const QString str = composerView->insertDayOfWeek(); result.append(str); } else if (cmd.startsWith(QLatin1String("DATE"))) { i += strlen("DATE"); const QString str = composerView->longDate(); result.append(str); } else if (cmd.startsWith(QLatin1String("SHORTDATE"))) { i += strlen("SHORTDATE"); const QString str = composerView->shortDate(); result.append(str); } else if (cmd.startsWith(QLatin1String("TIME"))) { i += strlen("TIME"); const QString str = composerView->shortTime(); result.append(str); } else if (cmd.startsWith(QLatin1String("TIMELONG"))) { i += strlen("TIMELONG"); const QString str = composerView->longTime(); result.append(str); } else if (cmd.startsWith(QLatin1String("ATTACHMENTCOUNT"))) { i += strlen("ATTACHMENTCOUNT"); const QString str = QString::number(composerView->attachments().count()); result.append(str); } else if (cmd.startsWith(QLatin1String("ATTACHMENTNAMES"))) { i += strlen("ATTACHMENTNAMES"); const QString str = composerView->attachments().names().join(QLatin1Char(',')); result.append(str); } else if (cmd.startsWith(QLatin1String("ATTACHMENTFILENAMES"))) { i += strlen("ATTACHMENTFILENAMES"); const QString str = composerView->attachments().fileNames().join(QLatin1Char(',')); result.append(str); } else if (cmd.startsWith(QLatin1String("ATTACHMENTNAMESANDSIZES"))) { i += strlen("ATTACHMENTNAMESANDSIZES"); const QString str = composerView->attachments().namesAndSize().join(QLatin1Char(',')); result.append(str); } else { result.append(convertVariables(cmd, i, c)); } } else { result.append(convertVariables(cmd, i, c)); } } else { result.append(c); } } return result; } -QString ConvertSnippetVariablesJob::getNameFromEmail(QString address) +QString ConvertSnippetVariablesJob::getNameFromEmail(const QString &address) { const QStringList lst = address.split(QStringLiteral(", ")); QStringList resultName; for (const QString &str : lst) { KMime::Types::Mailbox address; address.fromUnicodeString(KEmailAddress::normalizeAddressesAndEncodeIdn(str)); const QString firstName = address.name(); if (!firstName.isEmpty()) { resultName << firstName; } } const QString str = resultName.isEmpty() ? QString() : resultName.join(QStringLiteral(", ")); return str; } -QString ConvertSnippetVariablesJob::getFirstNameFromEmail(QString address) +QString ConvertSnippetVariablesJob::getFirstNameFromEmail(const QString &address) { const QStringList lst = address.split(QStringLiteral(", ")); QStringList resultName; for (const QString &str : lst) { KMime::Types::Mailbox address; address.fromUnicodeString(KEmailAddress::normalizeAddressesAndEncodeIdn(str)); const QString firstName = TemplateParser::Util::getLastNameFromEmail(address.name()); if (!firstName.isEmpty()) { resultName << firstName; } } const QString str = resultName.isEmpty() ? QString() : resultName.join(QStringLiteral(", ")); return str; } -QString ConvertSnippetVariablesJob::getLastNameFromEmail(QString address) +QString ConvertSnippetVariablesJob::getLastNameFromEmail(const QString &address) { const QStringList lst = address.split(QStringLiteral(", ")); QStringList resultName; for (const QString &str : lst) { KMime::Types::Mailbox address; address.fromUnicodeString(KEmailAddress::normalizeAddressesAndEncodeIdn(str)); const QString lastName = TemplateParser::Util::getLastNameFromEmail(address.name()); if (!lastName.isEmpty()) { resultName << lastName; } } const QString str = resultName.isEmpty() ? QString() : resultName.join(QStringLiteral(", ")); return str; } diff --git a/messagecomposer/src/snippet/convertsnippetvariablesjob.h b/messagecomposer/src/snippet/convertsnippetvariablesjob.h index 8c62fb14..e72f9d94 100644 --- a/messagecomposer/src/snippet/convertsnippetvariablesjob.h +++ b/messagecomposer/src/snippet/convertsnippetvariablesjob.h @@ -1,59 +1,59 @@ /* Copyright (C) 2019-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef CONVERTSNIPPETVARIABLESJOB_H #define CONVERTSNIPPETVARIABLESJOB_H #include #include "messagecomposer_export.h" namespace MessageComposer { class ComposerViewInterface; /** * @brief The ConvertSnippetVariablesJob class * @author Laurent Montel */ class MESSAGECOMPOSER_EXPORT ConvertSnippetVariablesJob : public QObject { Q_OBJECT public: explicit ConvertSnippetVariablesJob(QObject *parent = nullptr); ~ConvertSnippetVariablesJob(); void start(); void setText(const QString &str); Q_REQUIRED_RESULT QString text() const; MessageComposer::ComposerViewInterface *composerViewInterface() const; void setComposerViewInterface(MessageComposer::ComposerViewInterface *composerViewInterface); static Q_REQUIRED_RESULT QString convertVariables(MessageComposer::ComposerViewInterface *composerView, const QString &text); Q_REQUIRED_RESULT bool canStart() const; Q_SIGNALS: void textConverted(const QString &str); private: static Q_REQUIRED_RESULT QString convertVariables(const QString &cmd, int &i, QChar c); - static QString getFirstNameFromEmail(QString address); - static QString getLastNameFromEmail(QString address); - static QString getNameFromEmail(QString address); + static Q_REQUIRED_RESULT QString getFirstNameFromEmail(const QString &address); + static Q_REQUIRED_RESULT QString getLastNameFromEmail(const QString &address); + static Q_REQUIRED_RESULT QString getNameFromEmail(const QString &address); QString mText; MessageComposer::ComposerViewInterface *mComposerViewInterface = nullptr; }; } #endif // CONVERTVARIABLESJOB_H diff --git a/messagelist/src/core/model.cpp b/messagelist/src/core/model.cpp index cc437374..ff44bf72 100644 --- a/messagelist/src/core/model.cpp +++ b/messagelist/src/core/model.cpp @@ -1,4633 +1,4633 @@ /****************************************************************************** * * Copyright 2008 Szymon Tomasz Stefanek * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * *******************************************************************************/ // // This class is a rather huge monster. It's something that resembles a QAbstractItemModel // (because it has to provide the interface for a QTreeView) but isn't entirely one // (for optimization reasons). It basically manages a tree of items of two types: // GroupHeaderItem and MessageItem. Be sure to read the docs for ViewItemJob. // // A huge credit here goes to Till Adam which seems to have written most // (if not all) of the original KMail threading code. The KMHeaders implementation, // the documentation and his clever ideas were my starting points and essential tools. // This is why I'm adding his copyright entry (copied from headeritem.cpp) here even if // he didn't write a byte in this file until now :) // // Szymon Tomasz Stefanek, 03 Aug 2008 04:50 (am) // // This class contains ideas from: // // kmheaders.cpp / kmheaders.h, headeritem.cpp / headeritem.h // Copyright: (c) 2004 Till Adam < adam at kde dot org > // #include #include "core/model.h" #include "core/model_p.h" #include "core/view.h" #include "core/filter.h" #include "core/groupheaderitem.h" #include "core/item_p.h" #include "core/messageitem.h" #include "core/modelinvariantrowmapper.h" #include "core/storagemodelbase.h" #include "core/theme.h" #include "core/delegate.h" #include "core/manager.h" #include "core/messageitemsetmanager.h" #include "messagelist_debug.h" #include #include #include #include "MessageCore/StringUtil" #include #include #include #include #include #include #include #include #include namespace MessageList { namespace Core { Q_GLOBAL_STATIC(QTimer, _k_heartBeatTimer) /** * A job in a "View Fill" or "View Cleanup" or "View Update" task. * * For a "View Fill" task a job is a set of messages * that are contiguous in the storage. The set is expressed as a range * of row indexes. The task "sweeps" the storage in the specified * range, creates the appropriate Item instances and places them * in the right position in the tree. * * The idea is that in a single instance and for the same StorageModel * the jobs should never "cover" the same message twice. This assertion * is enforced all around this source file. * * For a "View Cleanup" task the job is a list of ModelInvariantIndex * objects (that are in fact MessageItem objects) that need to be removed * from the view. * * For a "View Update" task the job is a list of ModelInvariantIndex * objects (that are in fact MessageItem objects) that need to be updated. * * The interesting fact is that all the tasks need * very similar operations to be performed on the message tree. * * For a "View Fill" we have 5 passes. * * Pass 1 scans the underlying storage, creates the MessageItem objects * (which are subclasses of ModelInvariantIndex) and retrieves invariant * storage indexes for them. It also builds threading caches and * attempts to do some "easy" threading. If it succeeds in threading * and some conditions apply then it also attaches the items to the view. * Any unattached message is placed in a list. * * Pass 2 scans the list of messages that haven't been attached in * the first pass and performs perfect and reference based threading. * Since grouping of messages may depend on the "shape" of the thread * then certain threads aren't attached to the view yet. * Unassigned messages get stuffed into a list waiting for Pass3 * or directly to a list waiting for Pass4 (that is, Pass3 may be skipped * if there is no hope to find an imperfect parent by subject based threading). * * Pass 3 scans the list of messages that haven't been attached in * the first and second passes and performs subject based threading. * Since grouping of messages may depend on the "shape" of the thread * then certain threads aren't attached to the view yet. * Anything unattached gets stuffed into the list waiting for Pass4. * * Pass 4 scans the unattached threads and puts them in the appropriate * groups. After this pass nothing is unattached. * * Pass 5 eventually re-sorts the groups and removes the empty ones. * * For a "View Cleanup" we still have 5 passes. * * Pass 1 scans the list of invalidated ModelInvariantIndex-es, casts * them to MessageItem objects and detaches them from the view. * The orphan children of the destroyed items get stuffed in the list * of unassigned messages that has been used also in the "View Fill" task above. * * Pass 2, 3, 4 and 5: same as "View Fill", just operating on the "orphaned" * messages that need to be reattached to the view. * * For a "View Update" we still have 5 passes. * * Pass 1 scans the list of ModelInvariantIndex-es that need an update, casts * them to MessageItem objects and handles the updates from storage. * The updates may cause a regrouping so items might be stuffed in one * of the lists for pass 4 or 5. * * Pass 2, 3 and 4 are simply empty. * * Pass 5: same as "View Fill", just operating on groups that require updates * after the messages have been moved in pass 1. * * That's why we in fact have Pass1Fill, Pass1Cleanup, Pass1Update, Pass2, Pass3, Pass4 and Pass5 below. * Pass1Fill, Pass1Cleanup and Pass1Update are exclusive and all of them proceed with Pass2 when finished. */ class ViewItemJob { public: enum Pass { Pass1Fill = 0, ///< Build threading caches, *TRY* to do some threading, try to attach something to the view Pass1Cleanup = 1, ///< Kill messages, build list of orphans Pass1Update = 2, ///< Update messages Pass2 = 3, ///< Thread everything by using caches, try to attach more to the view Pass3 = 4, ///< Do more threading (this time try to guess), try to attach more to the view Pass4 = 5, ///< Attach anything is still unattached Pass5 = 6, ///< Eventually Re-sort group headers and remove the empty ones LastIndex = 7 ///< Keep this at the end, needed to get the size of the enum }; private: // Data for "View Fill" jobs int mStartIndex; ///< The first index (in the underlying storage) of this job int mCurrentIndex; ///< The current index (in the underlying storage) of this job int mEndIndex; ///< The last index (in the underlying storage) of this job // Data for "View Cleanup" jobs QList< ModelInvariantIndex * > *mInvariantIndexList; ///< Owned list of shallow pointers // Common data // The maximum time that we can spend "at once" inside viewItemJobStep() (milliseconds) // The bigger this value, the larger chunks of work we do at once and less the time // we loose in "breaking and resuming" the job. On the other side large values tend // to make the view less responsive up to a "freeze" perception if this value is larger // than 2000. int mChunkTimeout; // The interval between two fillView steps. The larger the interval, the more interactivity // we have. The shorter the interval the more work we get done per second. int mIdleInterval; // The minimum number of messages we process in every viewItemJobStep() call // The larger this value the less time we loose in checking the timeout every N messages. // On the other side, making this very large may make the view less responsive // if we're processing very few messages at a time and very high values (say > 10000) may // eventually make our job unbreakable until the end. int mMessageCheckCount; Pass mCurrentPass; // If this parameter is true then this job uses a "disconnected" UI. // It's FAR faster since we don't need to call beginInsertRows()/endInsertRows() // and we simply Q_EMIT a layoutChanged() at the end. It can be done only as the first // job though: subsequent jobs can't use layoutChanged() as it looses the expanded // state of items. bool mDisconnectUI; public: /** * Creates a "View Fill" operation job */ ViewItemJob(int startIndex, int endIndex, int chunkTimeout, int idleInterval, int messageCheckCount, bool disconnectUI = false) : mStartIndex(startIndex) , mCurrentIndex(startIndex) , mEndIndex(endIndex) , mInvariantIndexList(nullptr) , mChunkTimeout(chunkTimeout) , mIdleInterval(idleInterval) , mMessageCheckCount(messageCheckCount) , mCurrentPass(Pass1Fill) , mDisconnectUI(disconnectUI) { } /** * Creates a "View Cleanup" or "View Update" operation job */ ViewItemJob(Pass pass, QList< ModelInvariantIndex * > *invariantIndexList, int chunkTimeout, int idleInterval, int messageCheckCount) : mStartIndex(0) , mCurrentIndex(0) , mEndIndex(invariantIndexList->count() - 1) , mInvariantIndexList(invariantIndexList) , mChunkTimeout(chunkTimeout) , mIdleInterval(idleInterval) , mMessageCheckCount(messageCheckCount) , mCurrentPass(pass) , mDisconnectUI(false) { } ~ViewItemJob() { delete mInvariantIndexList; } public: int startIndex() const { return mStartIndex; } void setStartIndex(int startIndex) { mStartIndex = startIndex; mCurrentIndex = startIndex; } int currentIndex() const { return mCurrentIndex; } void setCurrentIndex(int currentIndex) { mCurrentIndex = currentIndex; } int endIndex() const { return mEndIndex; } void setEndIndex(int endIndex) { mEndIndex = endIndex; } Pass currentPass() const { return mCurrentPass; } void setCurrentPass(Pass pass) { mCurrentPass = pass; } int idleInterval() const { return mIdleInterval; } int chunkTimeout() const { return mChunkTimeout; } int messageCheckCount() const { return mMessageCheckCount; } QList< ModelInvariantIndex * > *invariantIndexList() const { return mInvariantIndexList; } bool disconnectUI() const { return mDisconnectUI; } }; } // namespace Core } // namespace MessageList using namespace MessageList::Core; Model::Model(View *pParent) : QAbstractItemModel(pParent) , d(new ModelPrivate(this)) { d->mRecursionCounterForReset = 0; d->mStorageModel = nullptr; d->mView = pParent; d->mAggregation = nullptr; d->mTheme = nullptr; d->mSortOrder = nullptr; d->mFilter = nullptr; d->mPersistentSetManager = nullptr; d->mInLengthyJobBatch = false; d->mLastSelectedMessageInFolder = nullptr; d->mLoading = false; d->mRootItem = new Item(Item::InvisibleRoot); d->mRootItem->setViewable(nullptr, true); d->mFillStepTimer.setSingleShot(true); d->mInvariantRowMapper = new ModelInvariantRowMapper(); d->mModelForItemFunctions = this; connect(&d->mFillStepTimer, &QTimer::timeout, this, [this]() { d->viewItemJobStep(); }); d->mCachedTodayLabel = i18n("Today"); d->mCachedYesterdayLabel = i18n("Yesterday"); d->mCachedUnknownLabel = i18nc("Unknown date", "Unknown"); d->mCachedLastWeekLabel = i18n("Last Week"); d->mCachedTwoWeeksAgoLabel = i18n("Two Weeks Ago"); d->mCachedThreeWeeksAgoLabel = i18n("Three Weeks Ago"); d->mCachedFourWeeksAgoLabel = i18n("Four Weeks Ago"); d->mCachedFiveWeeksAgoLabel = i18n("Five Weeks Ago"); d->mCachedWatchedOrIgnoredStatusBits = Akonadi::MessageStatus::statusIgnored().toQInt32() | Akonadi::MessageStatus::statusWatched().toQInt32(); connect(_k_heartBeatTimer(), &QTimer::timeout, this, [this]() { d->checkIfDateChanged(); }); if (!_k_heartBeatTimer->isActive()) { // First model starts it _k_heartBeatTimer->start(60000); // 1 minute } } Model::~Model() { setStorageModel(nullptr); d->clearJobList(); d->mOldestItem = nullptr; d->mNewestItem = nullptr; d->clearUnassignedMessageLists(); d->clearOrphanChildrenHash(); d->clearThreadingCacheReferencesIdMD5ToMessageItem(); d->clearThreadingCacheMessageSubjectMD5ToMessageItem(); delete d->mPersistentSetManager; // Delete the invariant row mapper before removing the items. // It's faster since the items will not need to call the invariant delete d->mInvariantRowMapper; delete d->mRootItem; delete d; } void Model::setAggregation(const Aggregation *aggregation) { d->mAggregation = aggregation; d->mView->setRootIsDecorated((d->mAggregation->grouping() == Aggregation::NoGrouping) && (d->mAggregation->threading() != Aggregation::NoThreading)); } void Model::setTheme(const Theme *theme) { d->mTheme = theme; } void Model::setSortOrder(const SortOrder *sortOrder) { d->mSortOrder = sortOrder; } const SortOrder *Model::sortOrder() const { return d->mSortOrder; } void Model::setFilter(const Filter *filter) { d->mFilter = filter; if (d->mFilter) { connect(d->mFilter, &Filter::finished, this, [this]() { d->slotApplyFilter(); }); } d->slotApplyFilter(); } void ModelPrivate::slotApplyFilter() { auto childList = mRootItem->childItems(); if (!childList) { return; } QModelIndex idx; // invalid QApplication::setOverrideCursor(Qt::WaitCursor); for (const auto child : qAsConst(*childList)) { applyFilterToSubtree(child, idx); } QApplication::restoreOverrideCursor(); } bool ModelPrivate::applyFilterToSubtree(Item *item, const QModelIndex &parentIndex) { // This function applies the current filter (eventually empty) // to a message tree starting at "item". if (!mModelForItemFunctions) { qCWarning(MESSAGELIST_LOG) << "Cannot apply filter, the UI must be not disconnected."; return true; } Q_ASSERT(item); // the item must obviously be valid Q_ASSERT(item->isViewable()); // the item must be viewable // Apply to children first auto childList = item->childItems(); bool childrenMatch = false; QModelIndex thisIndex = q->index(item, 0); if (childList) { for (const auto child : qAsConst(*childList)) { if (applyFilterToSubtree(child, thisIndex)) { childrenMatch = true; } } } if (!mFilter) { // empty filter always matches (but does not expand items) mView->setRowHidden(thisIndex.row(), parentIndex, false); return true; } if (childrenMatch) { mView->setRowHidden(thisIndex.row(), parentIndex, false); if (!mView->isExpanded(thisIndex)) { mView->expand(thisIndex); } return true; } if (item->type() == Item::Message) { if (mFilter->match((MessageItem *)item)) { mView->setRowHidden(thisIndex.row(), parentIndex, false); return true; } } // else this is a group header and it never explicitly matches // filter doesn't match, hide the item mView->setRowHidden(thisIndex.row(), parentIndex, true); return false; } int Model::columnCount(const QModelIndex &parent) const { if (!d->mTheme) { return 0; } if (parent.column() > 0) { return 0; } return d->mTheme->columns().count(); } QVariant Model::data(const QModelIndex &index, int role) const { /// this is called only when Akonadi is using the selectionmodel /// for item actions. since akonadi uses the ETM ItemRoles, and the /// messagelist uses its own internal roles, here we respond /// to the ETM ones. auto item = static_cast(index.internalPointer()); switch (role) { /// taken from entitytreemodel.h case Qt::UserRole + 1: //EntityTreeModel::ItemIdRole if (item->type() == MessageList::Core::Item::Message) { auto mItem = static_cast(item); return QVariant::fromValue(mItem->akonadiItem().id()); } else { return QVariant(); } break; case Qt::UserRole + 2: //EntityTreeModel::ItemRole if (item->type() == MessageList::Core::Item::Message) { auto mItem = static_cast(item); return QVariant::fromValue(mItem->akonadiItem()); } else { return QVariant(); } break; case Qt::UserRole + 3: //EntityTreeModel::MimeTypeRole if (item->type() == MessageList::Core::Item::Message) { return QStringLiteral("message/rfc822"); } else { return QVariant(); } break; case Qt::AccessibleTextRole: if (item->type() == MessageList::Core::Item::Message) { auto mItem = static_cast(item); return mItem->accessibleText(d->mTheme, index.column()); } else if (item->type() == MessageList::Core::Item::GroupHeader) { if (index.column() > 0) { return QString(); } auto hItem = static_cast(item); return hItem->label(); } return QString(); break; default: return QVariant(); } } QVariant Model::headerData(int section, Qt::Orientation, int role) const { if (!d->mTheme) { return QVariant(); } auto column = d->mTheme->column(section); if (!column) { return QVariant(); } if (d->mStorageModel && column->isSenderOrReceiver() && (role == Qt::DisplayRole)) { if (d->mStorageModelContainsOutboundMessages) { return QVariant(i18n("Receiver")); } return QVariant(i18n("Sender")); } const bool columnPixmapEmpty(column->pixmapName().isEmpty()); if ((role == Qt::DisplayRole) && columnPixmapEmpty) { return QVariant(column->label()); } else if ((role == Qt::ToolTipRole) && !columnPixmapEmpty) { return QVariant(column->label()); } else if ((role == Qt::DecorationRole) && !columnPixmapEmpty) { return QVariant(QIcon::fromTheme(column->pixmapName())); } return QVariant(); } QModelIndex Model::index(Item *item, int column) const { if (!d->mModelForItemFunctions) { return QModelIndex(); // called with disconnected UI: the item isn't known on the Qt side, yet } if (!item) { return QModelIndex(); } // FIXME: This function is a bottleneck (the caching in indexOfChildItem only works 30% of the time) auto par = item->parent(); if (!par) { if (item != d->mRootItem) { item->dump(QString()); } return QModelIndex(); } const int index = par->indexOfChildItem(item); if (index < 0) { return QModelIndex(); // BUG } return createIndex(index, column, item); } QModelIndex Model::index(int row, int column, const QModelIndex &parent) const { if (!d->mModelForItemFunctions) { return QModelIndex(); // called with disconnected UI: the item isn't known on the Qt side, yet } #ifdef READD_THIS_IF_YOU_WANT_TO_PASS_MODEL_TEST if (column < 0) { return QModelIndex(); // senseless column (we could optimize by skipping this check but ModelTest from trolltech is pedantic) } #endif const Item *item; if (parent.isValid()) { item = static_cast(parent.internalPointer()); if (!item) { return QModelIndex(); // should never happen } } else { item = d->mRootItem; } if (parent.column() > 0) { return QModelIndex(); // parent column is not 0: shouldn't have children (as per Qt documentation) } Item *child = item->childItem(row); if (!child) { return QModelIndex(); // no such row in parent } return createIndex(row, column, child); } QModelIndex Model::parent(const QModelIndex &modelIndex) const { Q_ASSERT(d->mModelForItemFunctions); // should be never called with disconnected UI if (!modelIndex.isValid()) { return QModelIndex(); // should never happen } auto item = static_cast(modelIndex.internalPointer()); if (!item) { return QModelIndex(); } auto par = item->parent(); if (!par) { return QModelIndex(); // should never happen } //return index( par, modelIndex.column() ); return index(par, 0); // parents are always in column 0 (as per Qt documentation) } int Model::rowCount(const QModelIndex &parent) const { if (!d->mModelForItemFunctions) { return 0; // called with disconnected UI } const Item *item; if (parent.isValid()) { item = static_cast(parent.internalPointer()); if (!item) { return 0; // should never happen } } else { item = d->mRootItem; } if (!item->isViewable()) { return 0; } return item->childItemCount(); } class RecursionPreventer { public: RecursionPreventer(int &counter) : mCounter(counter) { mCounter++; } ~RecursionPreventer() { mCounter--; } bool isRecursive() const { return mCounter > 1; } private: int &mCounter; }; StorageModel *Model::storageModel() const { return d->mStorageModel; } void ModelPrivate::clear() { q->beginResetModel(); if (mFillStepTimer.isActive()) { mFillStepTimer.stop(); } // Kill pre-selection at this stage mPreSelectionMode = PreSelectNone; mLastSelectedMessageInFolder = nullptr; mOldestItem = nullptr; mNewestItem = nullptr; // Reset the row mapper before removing items // This is faster since the items don't need to access the mapper. mInvariantRowMapper->modelReset(); clearJobList(); clearUnassignedMessageLists(); clearOrphanChildrenHash(); mGroupHeaderItemHash.clear(); mGroupHeadersThatNeedUpdate.clear(); mThreadingCacheMessageIdMD5ToMessageItem.clear(); mThreadingCacheMessageInReplyToIdMD5ToMessageItem.clear(); clearThreadingCacheReferencesIdMD5ToMessageItem(); clearThreadingCacheMessageSubjectMD5ToMessageItem(); mViewItemJobStepChunkTimeout = 100; mViewItemJobStepIdleInterval = 10; mViewItemJobStepMessageCheckCount = 10; delete mPersistentSetManager; mPersistentSetManager = nullptr; mCurrentItemToRestoreAfterViewItemJobStep = nullptr; mTodayDate = QDate::currentDate(); // FIXME: CLEAR THE FILTER HERE AS WE CAN'T APPLY IT WITH UI DISCONNECTED! mRootItem->killAllChildItems(); q->endResetModel(); //Q_EMIT headerDataChanged(); mView->selectionModel()->clearSelection(); } void Model::setStorageModel(StorageModel *storageModel, PreSelectionMode preSelectionMode) { // Prevent a case of recursion when opening a folder that has a message and the folder was // never opened before. RecursionPreventer preventer(d->mRecursionCounterForReset); if (preventer.isRecursive()) { return; } d->clear(); if (d->mStorageModel) { // Disconnect all signals from old storageModel std::for_each(d->mStorageModelConnections.cbegin(), d->mStorageModelConnections.cend(), [](const QMetaObject::Connection &c) -> bool { return QObject::disconnect(c); }); d->mStorageModelConnections.clear(); } const bool isReload = (d->mStorageModel == storageModel); d->mStorageModel = storageModel; if (!d->mStorageModel) { return; // no folder: nothing to fill } // Save threading cache of the previous folder, but only if the cache was // enabled and a different folder is being loaded - reload of the same folder // means change in aggregation in which case we will have to re-build the // cache so there's no point saving the current threading cache. if (d->mThreadingCache.isEnabled() && !isReload) { d->mThreadingCache.save(); } else { if (isReload) { qCDebug(MESSAGELIST_LOG) << "Identical folder reloaded, not saving old threading cache"; } else { qCDebug(MESSAGELIST_LOG) << "Threading disabled in previous folder, not saving threading cache"; } } // Load threading cache for the new folder, but only if threading is enabled, // otherwise we would just be caching a flat list. if (d->mAggregation->threading() != Aggregation::NoThreading) { d->mThreadingCache.setEnabled(true); d->mThreadingCache.load(d->mStorageModel->id(), d->mAggregation); } else { // No threading, no cache - don't even bother inserting entries into the // cache or trying to look them up there d->mThreadingCache.setEnabled(false); qCDebug(MESSAGELIST_LOG) << "Threading disabled in folder" << d->mStorageModel->id() << ", not using threading cache"; } d->mPreSelectionMode = preSelectionMode; d->mStorageModelContainsOutboundMessages = d->mStorageModel->containsOutboundMessages(); d->mStorageModelConnections = { connect(d->mStorageModel, &StorageModel::rowsInserted, this, [this](const QModelIndex &parent, int first, int last) { d->slotStorageModelRowsInserted(parent, first, last); }), connect(d->mStorageModel, &StorageModel::rowsRemoved, this, [this](const QModelIndex &parent, int first, int last) { d->slotStorageModelRowsRemoved(parent, first, last); }), connect(d->mStorageModel, &StorageModel::layoutChanged, this, [this]() { d->slotStorageModelLayoutChanged(); }), connect(d->mStorageModel, &StorageModel::modelReset, this, [this]() { d->slotStorageModelLayoutChanged(); }), connect(d->mStorageModel, &StorageModel::dataChanged, this, [this](const QModelIndex &topLeft, const QModelIndex &bottomRight) { d->slotStorageModelDataChanged(topLeft, bottomRight); }), connect(d->mStorageModel, &StorageModel::headerDataChanged, this, [this](Qt::Orientation orientation, int first, int last) { d->slotStorageModelHeaderDataChanged(orientation, first, last); }) }; if (d->mStorageModel->rowCount() == 0) { return; // folder empty: nothing to fill } // Here we use different strategies based on user preference and the folder size. // The knobs we can tune are: // // - The number of jobs used to scan the whole folder and their order // // There are basically two approaches to this. One is the "single big job" // approach. It scans the folder from the beginning to the end in a single job // entry. The job passes are done only once. It's advantage is that it's simplier // and it's less likely to generate imperfect parent threadings. The bad // side is that since the folders are "sort of" date ordered then the most interesting // messages show up at the end of the work. Not nice for large folders. // The other approach uses two jobs. This is a bit slower but smarter strategy. // First we scan the latest 1000 messages and *then* take care of the older ones. // This will show up the most interesting messages almost immediately. (Well... // All this assuming that the underlying storage always appends the newly arrived messages) // The strategy is slower since it generates some imperfect parent threadings which must be // adjusted by the second job. For instance, in my kernel mailing list folder this "smart" approach // generates about 150 additional imperfectly threaded children... but the "today" // messages show up almost immediately. The two-chunk job also makes computing // the percentage user feedback a little harder and might break some optimization // in the insertions (we're able to optimize appends and prepends but a chunked // job is likely to split our work at a boundary where messages are always inserted // in the middle of the list). // // - The maximum time to spend inside a single job step // // The larger this time, the greater the number of messages per second that this // engine can process but also greater time with frozen UI -> less interactivity. // Reasonable values start at 50 msecs. Values larger than 300 msecs are very likely // to be perceived by the user as UI non-reactivity. // // - The number of messages processed in each job step subchunk. // // A job subchunk is processed without checking the maximum time above. This means // that each job step will process at least the number of messages specified by this value. // Very low values mean that we respect the maximum time very carefully but we also // waste time to check if we ran out of time :) // Very high values are likely to cause the engine to not respect the maximum step time. // Reasonable values go from 5 to 100. // // - The "idle" time between two steps // // The lower this time, the greater the number of messages per second that this // engine can process but also lower time for the UI to process events -> less interactivity. // A value of 0 here means that Qt will trigger the timer as soon as it has some // idle time to spend. UI events will be still processed but slowdowns are possible. // 0 is reasonable though. Values larger than 200 will tend to make the total job // completion times high. // // If we have no filter it seems that we can apply a huge optimization. // We disconnect the UI for the first huge filling job. This allows us // to save the extremely expensive beginInsertRows()/endInsertRows() calls // and call a single layoutChanged() at the end. This slows down a lot item // expansion. But on the other side if only few items need to be expanded // then this strategy is better. If filtering is enabled then this strategy // isn't applicable (because filtering requires interaction with the UI // while the data is loading). // So... // For the very first small chunk it's ok to work with disconnected UI as long // as we have no filter. The first small chunk is always 1000 messages, so // even if all of them are expanded, it's still somewhat acceptable. bool canDoFirstSmallChunkWithDisconnectedUI = !d->mFilter; // Larger works need a bigger condition: few messages must be expanded in the end. bool canDoJobWithDisconnectedUI =// we have no filter !d->mFilter && ( // we do no threading at all (d->mAggregation->threading() == Aggregation::NoThreading) || // or we never expand threads (d->mAggregation->threadExpandPolicy() == Aggregation::NeverExpandThreads) || // or we expand threads but we'll be going to expand really only a few ( // so we don't expand them all (d->mAggregation->threadExpandPolicy() != Aggregation::AlwaysExpandThreads) && // and we'd expand only a few in fact (d->mStorageModel->initialUnreadRowCountGuess() < 1000) ) ); switch (d->mAggregation->fillViewStrategy()) { case Aggregation::FavorInteractivity: // favor interactivity if ((!canDoJobWithDisconnectedUI) && (d->mStorageModel->rowCount() > 3000)) { // empiric value // First a small job with the most recent messages. Large chunk, small (but non zero) idle interval // and a larger number of messages to process at once. auto job1 = new ViewItemJob(d->mStorageModel->rowCount() - 1000, d->mStorageModel->rowCount() - 1, 200, 20, 100, canDoFirstSmallChunkWithDisconnectedUI); d->mViewItemJobs.append(job1); // Then a larger job with older messages. Small chunk, bigger idle interval, small number of messages to // process at once. auto job2 = new ViewItemJob(0, d->mStorageModel->rowCount() - 1001, 100, 50, 10, false); d->mViewItemJobs.append(job2); // We could even extremize this by splitting the folder in several // chunks and scanning them from the newest to the oldest... but the overhead // due to imperfectly threaded children would be probably too big. } else { // small folder or can be done with disconnected UI: single chunk work. // Lag the CPU a bit more but not too much to destroy even the earliest interactivity. auto job = new ViewItemJob(0, d->mStorageModel->rowCount() - 1, 150, 30, 30, canDoJobWithDisconnectedUI); d->mViewItemJobs.append(job); } break; case Aggregation::FavorSpeed: // More batchy jobs, still interactive to a certain degree if ((!canDoJobWithDisconnectedUI) && (d->mStorageModel->rowCount() > 3000)) { // empiric value // large folder, but favor speed auto job1 = new ViewItemJob(d->mStorageModel->rowCount() - 1000, d->mStorageModel->rowCount() - 1, 250, 0, 100, canDoFirstSmallChunkWithDisconnectedUI); d->mViewItemJobs.append(job1); auto job2 = new ViewItemJob(0, d->mStorageModel->rowCount() - 1001, 200, 0, 10, false); d->mViewItemJobs.append(job2); } else { // small folder or can be done with disconnected UI and favor speed: single chunk work. // Lag the CPU more, get more work done auto job = new ViewItemJob(0, d->mStorageModel->rowCount() - 1, 250, 0, 100, canDoJobWithDisconnectedUI); d->mViewItemJobs.append(job); } break; case Aggregation::BatchNoInteractivity: { // one large job, never interrupt, block UI auto job = new ViewItemJob(0, d->mStorageModel->rowCount() - 1, 60000, 0, 100000, canDoJobWithDisconnectedUI); d->mViewItemJobs.append(job); break; } default: qCWarning(MESSAGELIST_LOG) << "Unrecognized fill view strategy"; Q_ASSERT(false); break; } d->mLoading = true; d->viewItemJobStep(); } void ModelPrivate::checkIfDateChanged() { // This function is called by MessageList::Core::Manager once in a while (every 1 minute or sth). // It is used to check if the current date has changed (with respect to mTodayDate). // // Our message items cache the formatted dates (as formatting them // on the fly would be too expensive). We also cache the labels of the groups which often display dates. // When the date changes we would need to fix all these strings. // // A dedicated algorithm to refresh the labels of the items would be either too complex // or would block on large trees. Fixing the labels of the groups is also quite hard... // // So to keep the things simple we just reload the view. if (!mStorageModel) { return; // nothing to do } if (mLoading) { return; // not now } if (!mViewItemJobs.isEmpty()) { return; // not now } if (mTodayDate == QDate::currentDate()) { return; // date not changed } // date changed, reload the view (and try to preserve the current selection) q->setStorageModel(mStorageModel, PreSelectLastSelected); } void Model::setPreSelectionMode(PreSelectionMode preSelect) { d->mPreSelectionMode = preSelect; d->mLastSelectedMessageInFolder = nullptr; } // // The "view fill" algorithm implemented in the functions below is quite smart but also quite complex. // It's governed by the following goals: // // - Be flexible: allow different configurations from "unsorted flat list" to a "grouped and threaded // list with different sorting algorithms applied to each aggregation level" // - Be reasonably fast // - Be non blocking: UI shouldn't freeze while the algorithm is running // - Be interruptible: user must be able to abort the execution and just switch to another folder in the middle // void ModelPrivate::clearUnassignedMessageLists() { // This is a bit tricky... // The three unassigned message lists contain messages that have been created // but not yet attached to the view. There may be two major cases for a message: // - it has no parent -> it must be deleted and it will delete its children too // - it has a parent -> it must NOT be deleted since it will be deleted by its parent. // Sometimes the things get a little complicated since in Pass2 and Pass3 // we have transitional states in that the MessageItem object can be in two of these lists. // WARNING: This function does NOT fixup mNewestItem and mOldestItem. If one of these // two messages is in the lists below, it's deleted and the member becomes a dangling pointer. // The caller must ensure that both mNewestItem and mOldestItem are set to 0 // and this is enforced in the assert below to avoid errors. This basically means // that this function should be called only when the storage model changes or // when the model is destroyed. Q_ASSERT((mOldestItem == nullptr) && (mNewestItem == nullptr)); if (!mUnassignedMessageListForPass2.isEmpty()) { // We're actually in Pass1* or Pass2: everything is mUnassignedMessageListForPass2 // Something may *also* be in mUnassignedMessageListForPass3 and mUnassignedMessageListForPass4 // but that are duplicates for sure. // We can't just sweep the list and delete parentless items since each delete // could kill children which are somewhere AFTER in the list: accessing the children // would then lead to a SIGSEGV. We first sweep the list gathering parentless // items and *then* delete them without accessing the parented ones. QList parentless; for (const auto mi : qAsConst(mUnassignedMessageListForPass2)) { if (!mi->parent()) { parentless.append(mi); } } for (const auto mi : qAsConst(parentless)) { delete mi; } mUnassignedMessageListForPass2.clear(); // Any message these list contain was also in mUnassignedMessageListForPass2 mUnassignedMessageListForPass3.clear(); mUnassignedMessageListForPass4.clear(); return; } // mUnassignedMessageListForPass2 is empty if (!mUnassignedMessageListForPass3.isEmpty()) { // We're actually at the very end of Pass2 or inside Pass3 // Pass2 pushes stuff in mUnassignedMessageListForPass3 *or* mUnassignedMessageListForPass4 // Pass3 pushes stuff from mUnassignedMessageListForPass3 to mUnassignedMessageListForPass4 // So if we're in Pass2 then the two lists contain distinct messages but if we're in Pass3 // then the two lists may contain the same messages. if (!mUnassignedMessageListForPass4.isEmpty()) { // We're actually in Pass3: the messiest one. QSet itemsToDelete; for (const auto mi : qAsConst(mUnassignedMessageListForPass3)) { if (!mi->parent()) { itemsToDelete.insert(mi); } } for (const auto mi : qAsConst(mUnassignedMessageListForPass4)) { if (!mi->parent()) { itemsToDelete.insert(mi); } } for (const auto mi : qAsConst(itemsToDelete)) { delete mi; } mUnassignedMessageListForPass3.clear(); mUnassignedMessageListForPass4.clear(); return; } // mUnassignedMessageListForPass4 is empty so we must be at the end of a very special kind of Pass2 // We have the same problem as in mUnassignedMessageListForPass2. QList parentless; for (const auto mi : qAsConst(mUnassignedMessageListForPass3)) { if (!mi->parent()) { parentless.append(mi); } } for (const auto mi : qAsConst(parentless)) { delete mi; } mUnassignedMessageListForPass3.clear(); return; } // mUnassignedMessageListForPass3 is empty if (!mUnassignedMessageListForPass4.isEmpty()) { // we're in Pass4.. this is easy. // We have the same problem as in mUnassignedMessageListForPass2. QList parentless; for (const auto mi : qAsConst(mUnassignedMessageListForPass4)) { if (!mi->parent()) { parentless.append(mi); } } for (const auto mi : qAsConst(parentless)) { delete mi; } mUnassignedMessageListForPass4.clear(); return; } } void ModelPrivate::clearThreadingCacheReferencesIdMD5ToMessageItem() { qDeleteAll(mThreadingCacheMessageReferencesIdMD5ToMessageItem); mThreadingCacheMessageReferencesIdMD5ToMessageItem.clear(); } void ModelPrivate::clearThreadingCacheMessageSubjectMD5ToMessageItem() { qDeleteAll(mThreadingCacheMessageSubjectMD5ToMessageItem); mThreadingCacheMessageSubjectMD5ToMessageItem.clear(); } void ModelPrivate::clearOrphanChildrenHash() { qDeleteAll(mOrphanChildrenHash); mOrphanChildrenHash.clear(); } void ModelPrivate::clearJobList() { if (mViewItemJobs.isEmpty()) { return; } if (mInLengthyJobBatch) { mInLengthyJobBatch = false; } qDeleteAll(mViewItemJobs); mViewItemJobs.clear(); mModelForItemFunctions = q; // make sure it's true, as there remains no job with disconnected UI } void ModelPrivate::attachGroup(GroupHeaderItem *ghi) { if (ghi->parent()) { if ( ((ghi)->childItemCount() > 0) // has children && (ghi)->isViewable() // is actually attached to the viewable root && mModelForItemFunctions // the UI is not disconnected && mView->isExpanded(q->index(ghi, 0)) // is actually expanded ) { saveExpandedStateOfSubtree(ghi); } // FIXME: This *WILL* break selection and current index... :/ ghi->parent()->takeChildItem(mModelForItemFunctions, ghi); } ghi->setParent(mRootItem); // I'm using a macro since it does really improve readability. // I'm NOT using a helper function since gcc will refuse to inline some of // the calls because they make this function grow too much. #define INSERT_GROUP_WITH_COMPARATOR(_ItemComparator) \ switch (mSortOrder->groupSortDirection()) \ { \ case SortOrder::Ascending: \ mRootItem->d_ptr->insertChildItem< _ItemComparator, true >(mModelForItemFunctions, ghi); \ break; \ case SortOrder::Descending: \ mRootItem->d_ptr->insertChildItem< _ItemComparator, false >(mModelForItemFunctions, ghi); \ break; \ default: /* should never happen... */ \ mRootItem->appendChildItem(mModelForItemFunctions, ghi); \ break; \ } switch (mSortOrder->groupSorting()) { case SortOrder::SortGroupsByDateTime: INSERT_GROUP_WITH_COMPARATOR(ItemDateComparator) break; case SortOrder::SortGroupsByDateTimeOfMostRecent: INSERT_GROUP_WITH_COMPARATOR(ItemMaxDateComparator) break; case SortOrder::SortGroupsBySenderOrReceiver: INSERT_GROUP_WITH_COMPARATOR(ItemSenderOrReceiverComparator) break; case SortOrder::SortGroupsBySender: INSERT_GROUP_WITH_COMPARATOR(ItemSenderComparator) break; case SortOrder::SortGroupsByReceiver: INSERT_GROUP_WITH_COMPARATOR(ItemReceiverComparator) break; case SortOrder::NoGroupSorting: mRootItem->appendChildItem(mModelForItemFunctions, ghi); break; default: // should never happen mRootItem->appendChildItem(mModelForItemFunctions, ghi); break; } if (ghi->initialExpandStatus() == Item::ExpandNeeded) { // this actually is a "non viewable expanded state" if (ghi->childItemCount() > 0) { if (mModelForItemFunctions) { // the UI is not disconnected syncExpandedStateOfSubtree(ghi); } } } // A group header is always viewable, when attached: apply the filter, if we have it. if (mFilter) { Q_ASSERT(mModelForItemFunctions); // UI must be NOT disconnected // apply the filter to subtree applyFilterToSubtree(ghi, QModelIndex()); } } void ModelPrivate::saveExpandedStateOfSubtree(Item *root) { Q_ASSERT(mModelForItemFunctions); // UI must be NOT disconnected here Q_ASSERT(root); root->setInitialExpandStatus(Item::ExpandNeeded); auto children = root->childItems(); if (!children) { return; } for (const auto mi : qAsConst(*children)) { if (mi->childItemCount() > 0 // has children && mi->isViewable() // is actually attached to the viewable root && mView->isExpanded(q->index(mi, 0))) { // is actually expanded saveExpandedStateOfSubtree(mi); } } } void ModelPrivate::syncExpandedStateOfSubtree(Item *root) { Q_ASSERT(mModelForItemFunctions); // UI must be NOT disconnected here // WE ASSUME that: // - the item is viewable // - its initialExpandStatus() is Item::ExpandNeeded // - it has at least one children (well.. this is not a strict requirement, but it's a waste of resources to expand items that don't have children) QModelIndex idx = q->index(root, 0); //if ( !mView->isExpanded( idx ) ) // this is O(logN!) in Qt.... very ugly... but it should never happen here mView->expand(idx); // sync the real state in the view root->setInitialExpandStatus(Item::ExpandExecuted); auto children = root->childItems(); if (!children) { return; } for (const auto mi : qAsConst(*children)) { if (mi->initialExpandStatus() == Item::ExpandNeeded) { if (mi->childItemCount() > 0) { syncExpandedStateOfSubtree(mi); } } } } void ModelPrivate::attachMessageToGroupHeader(MessageItem *mi) { QString groupLabel; time_t date; // compute the group header label and the date switch (mAggregation->grouping()) { case Aggregation::GroupByDate: case Aggregation::GroupByDateRange: { if (mAggregation->threadLeader() == Aggregation::MostRecentMessage) { date = mi->maxDate(); } else { date = mi->date(); } QDateTime dt; dt.setSecsSinceEpoch(date); QDate dDate = dt.date(); int daysAgo = -1; const int daysInWeek = 7; if (dDate.isValid() && mTodayDate.isValid()) { daysAgo = dDate.daysTo(mTodayDate); } if ((daysAgo < 0) // In the future || (static_cast< uint >(date) == static_cast< uint >(-1))) { // Invalid groupLabel = mCachedUnknownLabel; } else if (daysAgo == 0) { // Today groupLabel = mCachedTodayLabel; } else if (daysAgo == 1) { // Yesterday groupLabel = mCachedYesterdayLabel; } else if (daysAgo > 1 && daysAgo < daysInWeek) { // Within last seven days auto dayName = mCachedDayNameLabel.find(dDate.dayOfWeek()); // non-const call, but non-shared container if (dayName == mCachedDayNameLabel.end()) { dayName = mCachedDayNameLabel.insert(dDate.dayOfWeek(), QLocale::system().standaloneDayName(dDate.dayOfWeek())); } groupLabel = *dayName; } else if (mAggregation->grouping() == Aggregation::GroupByDate) { // GroupByDate seven days or more ago groupLabel = QLocale::system().toString(dDate, QLocale::ShortFormat); } else if (dDate.month() == mTodayDate.month() // GroupByDateRange within this month && dDate.year() == mTodayDate.year()) { int startOfWeekDaysAgo = (daysInWeek + mTodayDate.dayOfWeek() - QLocale().firstDayOfWeek()) % daysInWeek; int weeksAgo = ((daysAgo - startOfWeekDaysAgo) / daysInWeek) + 1; switch (weeksAgo) { case 0: // This week groupLabel = QLocale::system().standaloneDayName(dDate.dayOfWeek()); break; case 1: // 1 week ago groupLabel = mCachedLastWeekLabel; break; case 2: groupLabel = mCachedTwoWeeksAgoLabel; break; case 3: groupLabel = mCachedThreeWeeksAgoLabel; break; case 4: groupLabel = mCachedFourWeeksAgoLabel; break; case 5: groupLabel = mCachedFiveWeeksAgoLabel; break; default: // should never happen groupLabel = mCachedUnknownLabel; } } else if (dDate.year() == mTodayDate.year()) { // GroupByDateRange within this year auto monthName = mCachedMonthNameLabel.find(dDate.month()); // non-const call, but non-shared container if (monthName == mCachedMonthNameLabel.end()) { monthName = mCachedMonthNameLabel.insert(dDate.month(), QLocale::system().standaloneMonthName(dDate.month())); } groupLabel = *monthName; } else { // GroupByDateRange in previous years auto monthName = mCachedMonthNameLabel.find(dDate.month()); // non-const call, but non-shared container if (monthName == mCachedMonthNameLabel.end()) { monthName = mCachedMonthNameLabel.insert(dDate.month(), QLocale::system().standaloneMonthName(dDate.month())); } groupLabel = i18nc("Message Aggregation Group Header: Month name and Year number", "%1 %2", *monthName, QLocale::system().toString(dDate, QLatin1String("yyyy"))); } break; } case Aggregation::GroupBySenderOrReceiver: date = mi->date(); groupLabel = mi->displaySenderOrReceiver(); break; case Aggregation::GroupBySender: date = mi->date(); groupLabel = mi->displaySender(); break; case Aggregation::GroupByReceiver: date = mi->date(); groupLabel = mi->displayReceiver(); break; case Aggregation::NoGrouping: // append directly to root attachMessageToParent(mRootItem, mi); return; default: // should never happen attachMessageToParent(mRootItem, mi); return; } GroupHeaderItem *ghi; ghi = mGroupHeaderItemHash.value(groupLabel, nullptr); if (!ghi) { // not found ghi = new GroupHeaderItem(groupLabel); ghi->initialSetup(date, mi->size(), mi->sender(), mi->receiver(), mi->useReceiver()); switch (mAggregation->groupExpandPolicy()) { case Aggregation::NeverExpandGroups: // nothing to do break; case Aggregation::AlwaysExpandGroups: // expand always ghi->setInitialExpandStatus(Item::ExpandNeeded); break; case Aggregation::ExpandRecentGroups: // expand only if "close" to today if (mViewItemJobStepStartTime > ghi->date()) { if ((mViewItemJobStepStartTime - ghi->date()) < (3600 * 72)) { ghi->setInitialExpandStatus(Item::ExpandNeeded); } } else { if ((ghi->date() - mViewItemJobStepStartTime) < (3600 * 72)) { ghi->setInitialExpandStatus(Item::ExpandNeeded); } } break; default: // b0rken break; } attachMessageToParent(ghi, mi); attachGroup(ghi); // this will expand the group if required mGroupHeaderItemHash.insert(groupLabel, ghi); } else { // the group was already there (certainly viewable) // This function may be also called to re-group a message. // That is, to eventually find a new group for a message that has changed // its properties (but was already attached to a group). // So it may happen that we find out that in fact re-grouping wasn't really // needed because the message is already in the correct group. if (mi->parent() == ghi) { return; // nothing to be done } attachMessageToParent(ghi, mi); } // Remember this message as a thread leader mThreadingCache.updateParent(mi, nullptr); } MessageItem *ModelPrivate::findMessageParent(MessageItem *mi) { Q_ASSERT(mAggregation->threading() != Aggregation::NoThreading); // caller must take care of this // This function attempts to find a thread parent for the item "mi" // which actually may already have a children subtree. // Forged or plain broken message trees are dangerous here. // For example, a message tree with circular references like // // Message mi, Id=1, In-Reply-To=2 // Message childOfMi, Id=2, In-Reply-To=1 // // is perfectly possible and will cause us to find childOfMi // as parent of mi. This will then create a loop in the message tree // (which will then no longer be a tree in fact) and cause us to freeze // once we attempt to climb the parents. We need to take care of that. bool bMessageWasThreadable = false; MessageItem *pParent; // First of all try to find a "perfect parent", that is the message for that // we have the ID in the "In-Reply-To" field. This is actually done by using // MD5 caches of the message ids because of speed. Collisions are very unlikely. QByteArray md5 = mi->inReplyToIdMD5(); if (!md5.isEmpty()) { // have an In-Reply-To field MD5 pParent = mThreadingCacheMessageIdMD5ToMessageItem.value(md5, nullptr); if (pParent) { // Take care of circular references if ( (mi == pParent) // self referencing message || ( (mi->childItemCount() > 0) // mi already has children, this is fast to determine && pParent->hasAncestor(mi) // pParent is in the mi's children tree ) ) { qCWarning(MESSAGELIST_LOG) << "Circular In-Reply-To reference loop detected in the message tree"; mi->setThreadingStatus(MessageItem::NonThreadable); return nullptr; // broken message: throw it away } mi->setThreadingStatus(MessageItem::PerfectParentFound); return pParent; // got a perfect parent for this message } // got no perfect parent bMessageWasThreadable = true; // but the message was threadable } if (mAggregation->threading() == Aggregation::PerfectOnly) { mi->setThreadingStatus(bMessageWasThreadable ? MessageItem::ParentMissing : MessageItem::NonThreadable); return nullptr; // we're doing only perfect parent matches } // Try to use the "References" field. In fact we have the MD5 of the // (n-1)th entry in References. // // Original rationale from KMHeaders: // // If we don't have a replyToId, or if we have one and the // corresponding message is not in this folder, as happens // if you keep your outgoing messages in an OUTBOX, for // example, try the list of references, because the second // to last will likely be in this folder. replyToAuxIdMD5 // contains the second to last one. md5 = mi->referencesIdMD5(); if (!md5.isEmpty()) { pParent = mThreadingCacheMessageIdMD5ToMessageItem.value(md5, nullptr); if (pParent) { // Take care of circular references if ( (mi == pParent) // self referencing message || ( (mi->childItemCount() > 0) // mi already has children, this is fast to determine && pParent->hasAncestor(mi) // pParent is in the mi's children tree ) ) { qCWarning(MESSAGELIST_LOG) << "Circular reference loop detected in the message tree"; mi->setThreadingStatus(MessageItem::NonThreadable); return nullptr; // broken message: throw it away } mi->setThreadingStatus(MessageItem::ImperfectParentFound); return pParent; // got an imperfect parent for this message } auto messagesWithTheSameReferences = mThreadingCacheMessageReferencesIdMD5ToMessageItem.value(md5, nullptr); if (messagesWithTheSameReferences) { Q_ASSERT(!messagesWithTheSameReferences->isEmpty()); pParent = messagesWithTheSameReferences->first(); if (mi != pParent && (mi->childItemCount() == 0 || !pParent->hasAncestor(mi))) { mi->setThreadingStatus(MessageItem::ImperfectParentFound); return pParent; } } // got no imperfect parent bMessageWasThreadable = true; // but the message was threadable } if (mAggregation->threading() == Aggregation::PerfectAndReferences) { mi->setThreadingStatus(bMessageWasThreadable ? MessageItem::ParentMissing : MessageItem::NonThreadable); return nullptr; // we're doing only perfect parent matches } Q_ASSERT(mAggregation->threading() == Aggregation::PerfectReferencesAndSubject); // We are supposed to do subject based threading but we can't do it now. // This is because the subject based threading *may* be wrong and waste // time by creating circular references (that we'd need to detect and fix). // We first try the perfect and references based threading on all the messages // and then run subject based threading only on the remaining ones. mi->setThreadingStatus((bMessageWasThreadable || mi->subjectIsPrefixed()) ? MessageItem::ParentMissing : MessageItem::NonThreadable); return nullptr; } // Subject threading cache stuff #if 0 // Debug helpers void dump_iterator_and_list(QList< MessageItem * >::Iterator &iter, QList< MessageItem * > *list) { qCDebug(MESSAGELIST_LOG) << "Threading cache part dump"; if (iter == list->end()) { qCDebug(MESSAGELIST_LOG) << "Iterator pointing to end of the list"; } else { qCDebug(MESSAGELIST_LOG) << "Iterator pointing to " << *iter << " subject [" << (*iter)->subject() << "] date [" << (*iter)->date() << "]"; } for (QList< MessageItem * >::Iterator it = list->begin(); it != list->end(); ++it) { qCDebug(MESSAGELIST_LOG) << "List element " << *it << " subject [" << (*it)->subject() << "] date [" << (*it)->date() << "]"; } qCDebug(MESSAGELIST_LOG) << "End of threading cache part dump"; } void dump_list(QList< MessageItem * > *list) { qCDebug(MESSAGELIST_LOG) << "Threading cache part dump"; for (QList< MessageItem * >::Iterator it = list->begin(); it != list->end(); ++it) { qCDebug(MESSAGELIST_LOG) << "List element " << *it << " subject [" << (*it)->subject() << "] date [" << (*it)->date() << "]"; } qCDebug(MESSAGELIST_LOG) << "End of threading cache part dump"; } #endif // debug helpers // a helper class used in a qLowerBound() call below class MessageLessThanByDate { public: inline bool operator()(const MessageItem *mi1, const MessageItem *mi2) const { if (mi1->date() < mi2->date()) { // likely return true; } if (mi1->date() > mi2->date()) { // likely return false; } // dates are equal, compare by pointer return mi1 < mi2; } }; void ModelPrivate::addMessageToReferencesBasedThreadingCache(MessageItem *mi) { // Messages in this cache are sorted by date, and if dates are equal then they are sorted by pointer value. // Sorting by date is used to optimize the parent lookup in guessMessageParent() below. // WARNING: If the message date changes for some reason (like in the "update" step) // then the cache may become unsorted. For this reason the message about to // be changed must be first removed from the cache and then reinserted. auto messagesWithTheSameReference = mThreadingCacheMessageReferencesIdMD5ToMessageItem.value(mi->referencesIdMD5(), nullptr); if (!messagesWithTheSameReference) { messagesWithTheSameReference = new QList< MessageItem * >(); mThreadingCacheMessageReferencesIdMD5ToMessageItem.insert(mi->referencesIdMD5(), messagesWithTheSameReference); messagesWithTheSameReference->append(mi); return; } // Found: assert that we have no duplicates in the cache. Q_ASSERT(!messagesWithTheSameReference->contains(mi)); // Ordered insert: first by date then by pointer value. auto it = std::lower_bound(messagesWithTheSameReference->begin(), messagesWithTheSameReference->end(), mi, MessageLessThanByDate()); messagesWithTheSameReference->insert(it, mi); } void ModelPrivate::removeMessageFromReferencesBasedThreadingCache(MessageItem *mi) { // We assume that the caller knows what he is doing and the message is actually in the cache. // If the message isn't in the cache then we should not be called at all. auto messagesWithTheSameReference = mThreadingCacheMessageReferencesIdMD5ToMessageItem.value(mi->referencesIdMD5(), nullptr); // We assume that the message is there so the list must be non null. Q_ASSERT(messagesWithTheSameReference); // The cache *MUST* be ordered first by date then by pointer value auto it = std::lower_bound(messagesWithTheSameReference->begin(), messagesWithTheSameReference->end(), mi, MessageLessThanByDate()); // The binary based search must have found a message Q_ASSERT(it != messagesWithTheSameReference->end()); // and it must have found exactly the message requested Q_ASSERT(*it == mi); // Kill it messagesWithTheSameReference->erase(it); // And kill the list if it was the last one if (messagesWithTheSameReference->isEmpty()) { mThreadingCacheMessageReferencesIdMD5ToMessageItem.remove(mi->referencesIdMD5()); delete messagesWithTheSameReference; } } void ModelPrivate::addMessageToSubjectBasedThreadingCache(MessageItem *mi) { // Messages in this cache are sorted by date, and if dates are equal then they are sorted by pointer value. // Sorting by date is used to optimize the parent lookup in guessMessageParent() below. // WARNING: If the message date changes for some reason (like in the "update" step) // then the cache may become unsorted. For this reason the message about to // be changed must be first removed from the cache and then reinserted. // Lookup the list of messages with the same stripped subject auto messagesWithTheSameStrippedSubject = mThreadingCacheMessageSubjectMD5ToMessageItem.value(mi->strippedSubjectMD5(), nullptr); if (!messagesWithTheSameStrippedSubject) { // Not there yet: create it and append. messagesWithTheSameStrippedSubject = new QList< MessageItem * >(); mThreadingCacheMessageSubjectMD5ToMessageItem.insert(mi->strippedSubjectMD5(), messagesWithTheSameStrippedSubject); messagesWithTheSameStrippedSubject->append(mi); return; } // Found: assert that we have no duplicates in the cache. Q_ASSERT(!messagesWithTheSameStrippedSubject->contains(mi)); // Ordered insert: first by date then by pointer value. auto it = std::lower_bound(messagesWithTheSameStrippedSubject->begin(), messagesWithTheSameStrippedSubject->end(), mi, MessageLessThanByDate()); messagesWithTheSameStrippedSubject->insert(it, mi); } void ModelPrivate::removeMessageFromSubjectBasedThreadingCache(MessageItem *mi) { // We assume that the caller knows what he is doing and the message is actually in the cache. // If the message isn't in the cache then we should not be called at all. // // The game is called "performance" // Grab the list of all the messages with the same stripped subject (all potential parents) auto messagesWithTheSameStrippedSubject = mThreadingCacheMessageSubjectMD5ToMessageItem.value(mi->strippedSubjectMD5(), nullptr); // We assume that the message is there so the list must be non null. Q_ASSERT(messagesWithTheSameStrippedSubject); // The cache *MUST* be ordered first by date then by pointer value auto it = std::lower_bound(messagesWithTheSameStrippedSubject->begin(), messagesWithTheSameStrippedSubject->end(), mi, MessageLessThanByDate()); // The binary based search must have found a message Q_ASSERT(it != messagesWithTheSameStrippedSubject->end()); // and it must have found exactly the message requested Q_ASSERT(*it == mi); // Kill it messagesWithTheSameStrippedSubject->erase(it); // And kill the list if it was the last one if (messagesWithTheSameStrippedSubject->isEmpty()) { mThreadingCacheMessageSubjectMD5ToMessageItem.remove(mi->strippedSubjectMD5()); delete messagesWithTheSameStrippedSubject; } } MessageItem *ModelPrivate::guessMessageParent(MessageItem *mi) { // This function implements subject based threading // It attempts to guess a thread parent for the item "mi" // which actually may already have a children subtree. // We have all the problems of findMessageParent() plus the fact that // we're actually guessing (and often we may be *wrong*). Q_ASSERT(mAggregation->threading() == Aggregation::PerfectReferencesAndSubject); // caller must take care of this Q_ASSERT(mi->subjectIsPrefixed()); // caller must take care of this Q_ASSERT(mi->threadingStatus() == MessageItem::ParentMissing); // Do subject based threading const QByteArray md5 = mi->strippedSubjectMD5(); if (!md5.isEmpty()) { auto messagesWithTheSameStrippedSubject = mThreadingCacheMessageSubjectMD5ToMessageItem.value(md5, nullptr); if (messagesWithTheSameStrippedSubject) { Q_ASSERT(!messagesWithTheSameStrippedSubject->isEmpty()); // Need to find the message with the maximum date lower than the one of this message time_t maxTime = (time_t)0; MessageItem *pParent = nullptr; // Here'we re really guessing so circular references are possible // even on perfectly valid trees. This is why we don't consider it // an error but just continue searching. // FIXME: This might be speed up with an initial binary search (?) // ANSWER: No. We can't rely on date order (as it can be updated on the fly...) for (const auto it : qAsConst(*messagesWithTheSameStrippedSubject)) { int delta = mi->date() - it->date(); // We don't take into account messages with a delta smaller than 120. // Assuming that our date() values are correct (that is, they take into // account timezones etc..) then one usually needs more than 120 seconds // to answer to a message. Better safe than sorry. // This check also includes negative deltas so messages later than mi aren't considered if (delta < 120) { break; // The list is ordered by date (ascending) so we can stop searching here } // About the "magic" 3628899 value here comes a Till's comment from the original KMHeaders: // // "Parents more than six weeks older than the message are not accepted. The reasoning being // that if a new message with the same subject turns up after such a long time, the chances // that it is still part of the same thread are slim. The value of six weeks is chosen as a // result of a poll conducted on kde-devel, so it's probably bogus. :)" if (delta < 3628899) { // Compute the closest. if ((maxTime < it->date())) { // This algorithm *can* be (and often is) wrong. // Take care of circular threading which is really possible at this level. // If mi contains "it" inside its children subtree then we have // found such a circular threading problem. // Note that here we can't have it == mi because of the delta >= 120 check above. if ((mi->childItemCount() == 0) || !it->hasAncestor(mi)) { maxTime = it->date(); pParent = it; } } } } if (pParent) { mi->setThreadingStatus(MessageItem::ImperfectParentFound); return pParent; // got an imperfect parent for this message } } } return nullptr; } // // A little template helper, hopefully inlineable. // // Return true if the specified message item is in the wrong position // inside the specified parent and needs re-sorting. Return false otherwise. // Both parent and messageItem must not be null. // // Checking if a message needs re-sorting instead of just re-sorting it // is very useful since re-sorting is an expensive operation. // template< class ItemComparator > static bool messageItemNeedsReSorting(SortOrder::SortDirection messageSortDirection, ItemPrivate *parent, MessageItem *messageItem) { if ((messageSortDirection == SortOrder::Ascending) || (parent->mType == Item::Message)) { return parent->childItemNeedsReSorting< ItemComparator, true >(messageItem); } return parent->childItemNeedsReSorting< ItemComparator, false >(messageItem); } bool ModelPrivate::handleItemPropertyChanges(int propertyChangeMask, Item *parent, Item *item) { // The facts: // // - If dates changed: // - If we're sorting messages by min/max date then at each level the messages might need resorting. // - If the thread leader is the most recent message of a thread then the uppermost // message of the thread might need re-grouping. // - If the groups are sorted by min/max date then the group might need re-sorting too. // // This function explicitly doesn't re-apply the filter when ActionItemStatus changes. // This is because filters must be re-applied due to a broader range of status variations: // this is done in viewItemJobStepInternalForJobPass1Update() instead (which is the only // place in that ActionItemStatus may be set). if (parent->type() == Item::InvisibleRoot) { // item is either a message or a group attached to the root. // It might need resorting. if (item->type() == Item::GroupHeader) { // item is a group header attached to the root. if ( ( // max date changed (propertyChangeMask & MaxDateChanged) &&// groups sorted by max date (mSortOrder->groupSorting() == SortOrder::SortGroupsByDateTimeOfMostRecent) ) || ( // date changed (propertyChangeMask & DateChanged) &&// groups sorted by date (mSortOrder->groupSorting() == SortOrder::SortGroupsByDateTime) ) ) { // This group might need re-sorting. // Groups are large container of messages so it's likely that // another message inserted will cause this group to be marked again. // So we wait until the end to do the grand final re-sorting: it will be done in Pass4. mGroupHeadersThatNeedUpdate.insert(static_cast< GroupHeaderItem * >(item), static_cast< GroupHeaderItem * >(item)); } } else { // item is a message. It might need re-sorting. // Since sorting is an expensive operation, we first check if it's *really* needed. // Re-sorting will actually not change min/max dates at all and // will not climb up the parent's ancestor tree. switch (mSortOrder->messageSorting()) { case SortOrder::SortMessagesByDateTime: if (propertyChangeMask & DateChanged) { // date changed if (messageItemNeedsReSorting< ItemDateComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else date changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByDateTimeOfMostRecent: if (propertyChangeMask & MaxDateChanged) { // max date changed if (messageItemNeedsReSorting< ItemMaxDateComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else max date changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByActionItemStatus: if (propertyChangeMask & ActionItemStatusChanged) { // todo status changed if (messageItemNeedsReSorting< ItemActionItemStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else to do status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByUnreadStatus: if (propertyChangeMask & UnreadStatusChanged) { // new / unread status changed if (messageItemNeedsReSorting< ItemUnreadStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else new/unread status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByImportantStatus: if (propertyChangeMask & ImportantStatusChanged) { // important status changed if (messageItemNeedsReSorting< ItemImportantStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else new/unread status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByAttachmentStatus: if (propertyChangeMask & AttachmentStatusChanged) { // attachment status changed if (messageItemNeedsReSorting< ItemAttachmentStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else new/unread status changed, but it doesn't match sorting order: no need to re-sort break; default: // this kind of message sorting isn't affected by the property changes: nothing to do. break; } } return false; // the invisible root isn't affected by any change. } if (parent->type() == Item::GroupHeader) { // item is a message attached to a GroupHeader. // It might need re-grouping or re-sorting (within the same group) // Check re-grouping here. if ( ( // max date changed (propertyChangeMask & MaxDateChanged) &&// thread leader is most recent message (mAggregation->threadLeader() == Aggregation::MostRecentMessage) ) || ( // date changed (propertyChangeMask & DateChanged) &&// thread leader the topmost message (mAggregation->threadLeader() == Aggregation::TopmostMessage) ) ) { // Might really need re-grouping. // attachMessageToGroupHeader() will find the right group for this message // and if it's different than the current it will move it. attachMessageToGroupHeader(static_cast< MessageItem * >(item)); // Re-grouping fixes the properties of the involved group headers // so at exit of attachMessageToGroupHeader() the parent can't be affected // by the change anymore. return false; } // Re-grouping wasn't needed. Re-sorting might be. } // else item is a message attached to another message and might need re-sorting only. // Check if message needs re-sorting. switch (mSortOrder->messageSorting()) { case SortOrder::SortMessagesByDateTime: if (propertyChangeMask & DateChanged) { // date changed if (messageItemNeedsReSorting< ItemDateComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else date changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByDateTimeOfMostRecent: if (propertyChangeMask & MaxDateChanged) { // max date changed if (messageItemNeedsReSorting< ItemMaxDateComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else max date changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByActionItemStatus: if (propertyChangeMask & ActionItemStatusChanged) { // todo status changed if (messageItemNeedsReSorting< ItemActionItemStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else to do status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByUnreadStatus: if (propertyChangeMask & UnreadStatusChanged) { // new / unread status changed if (messageItemNeedsReSorting< ItemUnreadStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else new/unread status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByImportantStatus: if (propertyChangeMask & ImportantStatusChanged) { // important status changed if (messageItemNeedsReSorting< ItemImportantStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else important status changed, but it doesn't match sorting order: no need to re-sort break; case SortOrder::SortMessagesByAttachmentStatus: if (propertyChangeMask & AttachmentStatusChanged) { // attachment status changed if (messageItemNeedsReSorting< ItemAttachmentStatusComparator >(mSortOrder->messageSortDirection(), parent->d_ptr, static_cast< MessageItem * >(item))) { attachMessageToParent(parent, static_cast< MessageItem * >(item)); } } // else important status changed, but it doesn't match sorting order: no need to re-sort break; default: // this kind of message sorting isn't affected by property changes: nothing to do. break; } return true; // parent might be affected too. } void ModelPrivate::messageDetachedUpdateParentProperties(Item *oldParent, MessageItem *mi) { Q_ASSERT(oldParent); Q_ASSERT(mi); Q_ASSERT(oldParent != mRootItem); // oldParent might have its properties changed because of the child removal. // propagate the changes up. for (;;) { // pParent is not the root item now. This is assured by how we enter this loop // and by the fact that handleItemPropertyChanges returns false when grandParent // is Item::InvisibleRoot. We could actually assert it here... // Check if its dates need an update. int propertyChangeMask; if ((mi->maxDate() == oldParent->maxDate()) && oldParent->recomputeMaxDate()) { propertyChangeMask = MaxDateChanged; } else { break; // from the for(;;) loop } // One of the oldParent properties has changed for sure Item *grandParent = oldParent->parent(); // If there is no grandParent then oldParent isn't attached to the view. // Re-sorting / re-grouping isn't needed for sure. if (!grandParent) { break; // from the for(;;) loop } // The following function will return true if grandParent may be affected by the change. // If the grandParent isn't affected, we stop climbing. if (!handleItemPropertyChanges(propertyChangeMask, grandParent, oldParent)) { break; // from the for(;;) loop } // Now we need to climb up one level and check again. oldParent = grandParent; } // for(;;) loop // If the last message was removed from a group header then this group will need an update // for sure. We will need to remove it (unless a message is attached back to it) if (oldParent->type() == Item::GroupHeader) { if (oldParent->childItemCount() == 0) { mGroupHeadersThatNeedUpdate.insert(static_cast< GroupHeaderItem * >(oldParent), static_cast< GroupHeaderItem * >(oldParent)); } } } void ModelPrivate::propagateItemPropertiesToParent(Item *item) { Item *pParent = item->parent(); Q_ASSERT(pParent); Q_ASSERT(pParent != mRootItem); for (;;) { // pParent is not the root item now. This is assured by how we enter this loop // and by the fact that handleItemPropertyChanges returns false when grandParent // is Item::InvisibleRoot. We could actually assert it here... // Check if its dates need an update. int propertyChangeMask; if (item->maxDate() > pParent->maxDate()) { pParent->setMaxDate(item->maxDate()); propertyChangeMask = MaxDateChanged; } else { // No parent dates have changed: no further work is needed. Stop climbing here. break; // from the for(;;) loop } // One of the pParent properties has changed. Item *grandParent = pParent->parent(); // If there is no grandParent then pParent isn't attached to the view. // Re-sorting / re-grouping isn't needed for sure. if (!grandParent) { break; // from the for(;;) loop } // The following function will return true if grandParent may be affected by the change. // If the grandParent isn't affected, we stop climbing. if (!handleItemPropertyChanges(propertyChangeMask, grandParent, pParent)) { break; // from the for(;;) loop } // Now we need to climb up one level and check again. pParent = grandParent; } // for(;;) } void ModelPrivate::attachMessageToParent(Item *pParent, MessageItem *mi, AttachOptions attachOptions) { Q_ASSERT(pParent); Q_ASSERT(mi); // This function may be called to do a simple "re-sort" of the item inside the parent. // In that case mi->parent() is equal to pParent. bool oldParentWasTheSame; if (mi->parent()) { Item *oldParent = mi->parent(); // The item already had a parent and this means that we're moving it. oldParentWasTheSame = oldParent == pParent; // just re-sorting ? if (mi->isViewable()) { // is actually // The message is actually attached to the viewable root // Unfortunately we need to hack the model/view architecture // since it's somewhat flawed in this. At the moment of writing // there is simply no way to atomically move a subtree. // We must detach, call beginRemoveRows()/endRemoveRows(), // save the expanded state, save the selection, save the current item, // save the view position (YES! As we are removing items the view // will hopelessly jump around so we're just FORCED to break // the isolation from the view)... // ...*then* reattach, restore the expanded state, restore the selection, // restore the current item, restore the view position and pray // that nothing will fail in the (rather complicated) process.... // Yet more unfortunately, while saving the expanded state might stop // at a certain (unexpanded) point in the tree, saving the selection // is hopelessly recursive down to the bare leafs. // Furthermore the expansion of items is a common case while selection // in the subtree is rare, so saving it would be a huge cost with // a low revenue. // This is why we just let the selection screw up. I hereby refuse to call // yet another expensive recursive function here :D // The current item saving can be somewhat optimized doing it once for // a single job step... if ( ((mi)->childItemCount() > 0) // has children && mModelForItemFunctions // the UI is not actually disconnected && mView->isExpanded(q->index(mi, 0)) // is actually expanded ) { saveExpandedStateOfSubtree(mi); } } // If the parent is viewable (so mi was viewable too) then the beginRemoveRows() // and endRemoveRows() functions of this model will be called too. oldParent->takeChildItem(mModelForItemFunctions, mi); if ((!oldParentWasTheSame) && (oldParent != mRootItem)) { messageDetachedUpdateParentProperties(oldParent, mi); } } else { // The item had no parent yet. oldParentWasTheSame = false; } // Take care of perfect / imperfect threading. // Items that are now perfectly threaded, but already have a different parent // might have been imperfectly threaded before. Remove them from the caches. // Items that are now imperfectly threaded must be added to the caches. // // If we're just re-sorting the item inside the same parent then the threading // caches don't need to be updated (since they actually depend on the parent). if (!oldParentWasTheSame) { switch (mi->threadingStatus()) { case MessageItem::PerfectParentFound: if (!mi->inReplyToIdMD5().isEmpty()) { mThreadingCacheMessageInReplyToIdMD5ToMessageItem.remove(mi->inReplyToIdMD5(), mi); } if (attachOptions == StoreInCache && pParent->type() == Item::Message) { mThreadingCache.updateParent(mi, static_cast(pParent)); } break; case MessageItem::ImperfectParentFound: case MessageItem::ParentMissing: // may be: temporary or just fallback assignment if (!mi->inReplyToIdMD5().isEmpty()) { if (!mThreadingCacheMessageInReplyToIdMD5ToMessageItem.contains(mi->inReplyToIdMD5(), mi)) { mThreadingCacheMessageInReplyToIdMD5ToMessageItem.insert(mi->inReplyToIdMD5(), mi); } } break; case MessageItem::NonThreadable: // this also happens when we do no threading at all // make gcc happy Q_ASSERT(!mThreadingCacheMessageInReplyToIdMD5ToMessageItem.contains(mi->inReplyToIdMD5(), mi)); break; } } // Set the new parent mi->setParent(pParent); // Propagate watched and ignored status if ( (pParent->status().toQInt32() & mCachedWatchedOrIgnoredStatusBits) // unlikely && (pParent->type() == Item::Message) // likely ) { // the parent is either watched or ignored: propagate to the child if (pParent->status().isWatched()) { int row = mInvariantRowMapper->modelInvariantIndexToModelIndexRow(mi); mi->setStatus(Akonadi::MessageStatus::statusWatched()); mStorageModel->setMessageItemStatus(mi, row, Akonadi::MessageStatus::statusWatched()); } else if (pParent->status().isIgnored()) { int row = mInvariantRowMapper->modelInvariantIndexToModelIndexRow(mi); mi->setStatus(Akonadi::MessageStatus::statusIgnored()); mStorageModel->setMessageItemStatus(mi, row, Akonadi::MessageStatus::statusIgnored()); } } // And insert into its child list // If pParent is viewable then the insert/append functions will call this model's // beginInsertRows() and endInsertRows() functions. This is EXTREMELY // expensive and ugly but it's the only way with the Qt4 imposed Model/View method. // Dude... (citation from Lost, if it wasn't clear). // I'm using a macro since it does really improve readability. // I'm NOT using a helper function since gcc will refuse to inline some of // the calls because they make this function grow too much. #define INSERT_MESSAGE_WITH_COMPARATOR(_ItemComparator) \ if ((mSortOrder->messageSortDirection() == SortOrder::Ascending) \ || (pParent->type() == Item::Message)) \ { \ pParent->d_ptr->insertChildItem< _ItemComparator, true >(mModelForItemFunctions, mi); \ } \ else \ { \ pParent->d_ptr->insertChildItem< _ItemComparator, false >(mModelForItemFunctions, mi); \ } // If pParent is viewable then the insertion call will also set the child state to viewable. // Since mi MAY have children, then this call may make them viewable. switch (mSortOrder->messageSorting()) { case SortOrder::SortMessagesByDateTime: INSERT_MESSAGE_WITH_COMPARATOR(ItemDateComparator) break; case SortOrder::SortMessagesByDateTimeOfMostRecent: INSERT_MESSAGE_WITH_COMPARATOR(ItemMaxDateComparator) break; case SortOrder::SortMessagesBySize: INSERT_MESSAGE_WITH_COMPARATOR(ItemSizeComparator) break; case SortOrder::SortMessagesBySenderOrReceiver: INSERT_MESSAGE_WITH_COMPARATOR(ItemSenderOrReceiverComparator) break; case SortOrder::SortMessagesBySender: INSERT_MESSAGE_WITH_COMPARATOR(ItemSenderComparator) break; case SortOrder::SortMessagesByReceiver: INSERT_MESSAGE_WITH_COMPARATOR(ItemReceiverComparator) break; case SortOrder::SortMessagesBySubject: INSERT_MESSAGE_WITH_COMPARATOR(ItemSubjectComparator) break; case SortOrder::SortMessagesByActionItemStatus: INSERT_MESSAGE_WITH_COMPARATOR(ItemActionItemStatusComparator) break; case SortOrder::SortMessagesByUnreadStatus: INSERT_MESSAGE_WITH_COMPARATOR(ItemUnreadStatusComparator) break; case SortOrder::SortMessagesByImportantStatus: INSERT_MESSAGE_WITH_COMPARATOR(ItemImportantStatusComparator) break; case SortOrder::SortMessagesByAttachmentStatus: INSERT_MESSAGE_WITH_COMPARATOR(ItemAttachmentStatusComparator) break; case SortOrder::NoMessageSorting: pParent->appendChildItem(mModelForItemFunctions, mi); break; default: // should never happen pParent->appendChildItem(mModelForItemFunctions, mi); break; } // Decide if we need to expand parents bool childNeedsExpanding = (mi->initialExpandStatus() == Item::ExpandNeeded); if (pParent->initialExpandStatus() == Item::NoExpandNeeded) { switch (mAggregation->threadExpandPolicy()) { case Aggregation::NeverExpandThreads: // just do nothing unless this child has children and is already marked for expansion if (childNeedsExpanding) { pParent->setInitialExpandStatus(Item::ExpandNeeded); } break; case Aggregation::ExpandThreadsWithNewMessages: // No more new status. fall through to unread if it exists in config case Aggregation::ExpandThreadsWithUnreadMessages: // expand only if unread (or it has children marked for expansion) if (childNeedsExpanding || !mi->status().isRead()) { pParent->setInitialExpandStatus(Item::ExpandNeeded); } break; case Aggregation::ExpandThreadsWithUnreadOrImportantMessages: // expand only if unread, important or todo (or it has children marked for expansion) // FIXME: Wouldn't it be nice to be able to test for bitmasks in MessageStatus ? if (childNeedsExpanding || !mi->status().isRead() || mi->status().isImportant() || mi->status().isToAct()) { pParent->setInitialExpandStatus(Item::ExpandNeeded); } break; case Aggregation::AlwaysExpandThreads: // expand everything pParent->setInitialExpandStatus(Item::ExpandNeeded); break; default: // BUG break; } } // else it's already marked for expansion or expansion has been already executed // expand parent first, if possible if (pParent->initialExpandStatus() == Item::ExpandNeeded) { // If UI is not disconnected and parent is viewable, go up and expand if (mModelForItemFunctions && pParent->isViewable()) { // Now expand parents as needed Item *parentToExpand = pParent; while (parentToExpand) { if (parentToExpand == mRootItem) { break; // no need to set it expanded } // parentToExpand is surely viewable (because this item is) if (parentToExpand->initialExpandStatus() == Item::ExpandExecuted) { break; } mView->expand(q->index(parentToExpand, 0)); parentToExpand->setInitialExpandStatus(Item::ExpandExecuted); parentToExpand = parentToExpand->parent(); } } else { // It isn't viewable or UI is disconnected: climb up marking only Item *parentToExpand = pParent->parent(); while (parentToExpand) { if (parentToExpand == mRootItem) { break; // no need to set it expanded } parentToExpand->setInitialExpandStatus(Item::ExpandNeeded); parentToExpand = parentToExpand->parent(); } } } if (mi->isViewable()) { // mi is now viewable // sync subtree expanded status if (childNeedsExpanding) { if (mi->childItemCount() > 0) { if (mModelForItemFunctions) { // the UI is not disconnected syncExpandedStateOfSubtree(mi); // sync the real state in the view } } } // apply the filter, if needed if (mFilter) { Q_ASSERT(mModelForItemFunctions); // the UI must be NOT disconnected here // apply the filter to subtree if (applyFilterToSubtree(mi, q->index(pParent, 0))) { // mi matched, expand parents (unconditionally) mView->ensureDisplayedWithParentsExpanded(mi); } } } // Now we need to propagate the property changes the upper levels. // If we have just inserted a message inside the root then no work needs to be done: // no grouping is in effect and the message is already in the right place. if (pParent == mRootItem) { return; } // If we have just removed the item from this parent and re-inserted it // then this operation was a simple re-sort. The code above didn't update // the properties when removing the item so we don't actually need // to make the updates back. if (oldParentWasTheSame) { return; } // FIXME: OPTIMIZE THIS: First propagate changes THEN syncExpandedStateOfSubtree() // and applyFilterToSubtree... (needs some thinking though). // Time to propagate up. propagateItemPropertiesToParent(mi); // Aaah.. we're done. Time for a thea ? :) } // FIXME: ThreadItem ? // // Foo Bar, Joe Thommason, Martin Rox ... Eddie Maiden // Title , Last by xxx // // When messages are added, mark it as dirty only (?) -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass5(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass5(ViewItemJob *job, QElapsedTimer elapsedTimer) { // In this pass we scan the group headers that are in mGroupHeadersThatNeedUpdate. // Empty groups get deleted while the other ones are re-sorted. int curIndex = job->currentIndex(); auto it = mGroupHeadersThatNeedUpdate.begin(); auto end = mGroupHeadersThatNeedUpdate.end(); while (it != end) { if ((*it)->childItemCount() == 0) { // group with no children, kill it (*it)->parent()->takeChildItem(mModelForItemFunctions, *it); mGroupHeaderItemHash.remove((*it)->label()); // If we were going to restore its position after the job step, well.. we can't do it anymore. if (mCurrentItemToRestoreAfterViewItemJobStep == (*it)) { mCurrentItemToRestoreAfterViewItemJobStep = nullptr; } // bye bye delete *it; } else { // Group with children: probably needs re-sorting. // Re-sorting here is an expensive operation. // In fact groups have been put in the QHash above on the assumption // that re-sorting *might* be needed but no real (expensive) check // has been done yet. Also by sorting a single group we might actually // put the others in the right place. // So finally check if re-sorting is *really* needed. bool needsReSorting; // A macro really improves readability here. #define CHECK_IF_GROUP_NEEDS_RESORTING(_ItemDateComparator) \ switch (mSortOrder->groupSortDirection()) \ { \ case SortOrder::Ascending: \ needsReSorting = (*it)->parent()->d_ptr->childItemNeedsReSorting< _ItemDateComparator, true >(*it); \ break; \ case SortOrder::Descending: \ needsReSorting = (*it)->parent()->d_ptr->childItemNeedsReSorting< _ItemDateComparator, false >(*it); \ break; \ default: /* should never happen */ \ needsReSorting = false; \ break; \ } switch (mSortOrder->groupSorting()) { case SortOrder::SortGroupsByDateTime: CHECK_IF_GROUP_NEEDS_RESORTING(ItemDateComparator) break; case SortOrder::SortGroupsByDateTimeOfMostRecent: CHECK_IF_GROUP_NEEDS_RESORTING(ItemMaxDateComparator) break; case SortOrder::SortGroupsBySenderOrReceiver: CHECK_IF_GROUP_NEEDS_RESORTING(ItemSenderOrReceiverComparator) break; case SortOrder::SortGroupsBySender: CHECK_IF_GROUP_NEEDS_RESORTING(ItemSenderComparator) break; case SortOrder::SortGroupsByReceiver: CHECK_IF_GROUP_NEEDS_RESORTING(ItemReceiverComparator) break; case SortOrder::NoGroupSorting: needsReSorting = false; break; default: // Should never happen... just assume re-sorting is not needed needsReSorting = false; break; } if (needsReSorting) { attachGroup(*it); // it will first detach and then re-attach in the proper place } } it = mGroupHeadersThatNeedUpdate.erase(it); curIndex++; // FIXME: In fact a single update is likely to manipulate // a subtree with a LOT of messages inside. If interactivity is favored // we should check the time really more often. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (it != mGroupHeadersThatNeedUpdate.end()) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass4(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass4(ViewItemJob *job, QElapsedTimer elapsedTimer) { // In this pass we scan mUnassignedMessageListForPass4 which now // contains both items with parents and items without parents. // We scan mUnassignedMessageList for messages without parent (the ones that haven't been // attached to the viewable tree yet) and find a suitable group for them. Then we simply // clear mUnassignedMessageList. // We call this pass "Grouping" int curIndex = job->currentIndex(); int endIndex = job->endIndex(); while (curIndex <= endIndex) { MessageItem *mi = mUnassignedMessageListForPass4[curIndex]; if (!mi->parent()) { // Unassigned item: thread leader, insert into the proper group. // Locate the group (or root if no grouping requested) attachMessageToGroupHeader(mi); } else { // A parent was already assigned in Pass3: we have nothing to do here } curIndex++; // FIXME: In fact a single call to attachMessageToGroupHeader() is likely to manipulate // a subtree with a LOT of messages inside. If interactivity is favored // we should check the time really more often. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } mUnassignedMessageListForPass4.clear(); return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass3(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass3(ViewItemJob *job, QElapsedTimer elapsedTimer) { // In this pass we scan the mUnassignedMessageListForPass3 and try to do construct the threads // by using subject based threading. If subject based threading is not in effect then // this pass turns to a nearly-no-op: at the end of Pass2 we have swapped the lists // and mUnassignedMessageListForPass3 is actually empty. // We don't shrink the mUnassignedMessageListForPass3 for two reasons: // - It would mess up this chunked algorithm by shifting indexes // - mUnassignedMessageList is a QList which is basically an array. It's faster // to traverse an array of N entries than to remove K>0 entries one by one and // to traverse the remaining N-K entries. int curIndex = job->currentIndex(); int endIndex = job->endIndex(); while (curIndex <= endIndex) { // If we're here, then threading is requested for sure. auto mi = mUnassignedMessageListForPass3[curIndex]; if ((!mi->parent()) || (mi->threadingStatus() == MessageItem::ParentMissing)) { // Parent is missing (either "physically" with the item being not attached or "logically" // with the item being attached to a group or directly to the root. if (mi->subjectIsPrefixed()) { // We can try to guess it auto mparent = guessMessageParent(mi); if (mparent) { // imperfect parent found if (mi->isViewable()) { // mi was already viewable, we're just trying to re-parent it better... attachMessageToParent(mparent, mi); if (!mparent->isViewable()) { // re-attach it immediately (so current item is not lost) auto topmost = mparent->topmostMessage(); Q_ASSERT(!topmost->parent()); // groups are always viewable! topmost->setThreadingStatus(MessageItem::ParentMissing); attachMessageToGroupHeader(topmost); } } else { // mi wasn't viewable yet.. no need to attach parent attachMessageToParent(mparent, mi); } // and we're done for now } else { // so parent not found, (threadingStatus() is either MessageItem::ParentMissing or MessageItem::NonThreadable) Q_ASSERT((mi->threadingStatus() == MessageItem::ParentMissing) || (mi->threadingStatus() == MessageItem::NonThreadable)); mUnassignedMessageListForPass4.append(mi); // this is ~O(1) // and wait for Pass4 } } else { // can't guess the parent as the subject isn't prefixed Q_ASSERT((mi->threadingStatus() == MessageItem::ParentMissing) || (mi->threadingStatus() == MessageItem::NonThreadable)); mUnassignedMessageListForPass4.append(mi); // this is ~O(1) // and wait for Pass4 } } else { // Has a parent: either perfect parent already found or non threadable. // Since we don't end here if mi has status of parent missing then mi must not have imperfect parent. Q_ASSERT(mi->threadingStatus() != MessageItem::ImperfectParentFound); Q_ASSERT(mi->isViewable()); } curIndex++; // FIXME: In fact a single call to attachMessageToGroupHeader() is likely to manipulate // a subtree with a LOT of messages inside. If interactivity is favored // we should check the time really more often. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } mUnassignedMessageListForPass3.clear(); return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass2(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass2(ViewItemJob *job, QElapsedTimer elapsedTimer) { // In this pass we scan the mUnassignedMessageList and try to do construct the threads. // If some thread leader message got attached to the viewable tree in Pass1Fill then // we'll also attach all of its children too. The thread leaders we were unable // to attach in Pass1Fill and their children (which we find here) will make it to the small Pass3 // We don't shrink the mUnassignedMessageList for two reasons: // - It would mess up this chunked algorithm by shifting indexes // - mUnassignedMessageList is a QList which is basically an array. It's faster // to traverse an array of N entries than to remove K>0 entries one by one and // to traverse the remaining N-K entries. // We call this pass "Threading" int curIndex = job->currentIndex(); int endIndex = job->endIndex(); while (curIndex <= endIndex) { // If we're here, then threading is requested for sure. auto mi = mUnassignedMessageListForPass2[curIndex]; // The item may or may not have a parent. // If it has no parent or it has a temporary one (mi->parent() && mi->threadingStatus() == MessageItem::ParentMissing) // then we attempt to (re-)thread it. Otherwise we just do nothing (the job has already been done by the previous steps). if ((!mi->parent()) || (mi->threadingStatus() == MessageItem::ParentMissing)) { qint64 parentId; auto mparent = mThreadingCache.parentForItem(mi, parentId); if (mparent && !mparent->hasAncestor(mi)) { mi->setThreadingStatus(MessageItem::PerfectParentFound); attachMessageToParent(mparent, mi, SkipCacheUpdate); } else { if (parentId > 0) { // In second pass we have all available Items in mThreadingCache already. If // mThreadingCache.parentForItem() returns null, but returns valid parentId then // the Item was removed from Akonadi and our threading cache is out-of-date. mThreadingCache.expireParent(mi); mparent = findMessageParent(mi); } else if (parentId < 0) { mparent = findMessageParent(mi); } else { // parentId = 0: this message is a thread leader so don't // bother resolving parent, it will be moved directly to // Pass4 in the code below } if (mparent) { // parent found, either perfect or imperfect if (mi->isViewable()) { // mi was already viewable, we're just trying to re-parent it better... attachMessageToParent(mparent, mi); if (!mparent->isViewable()) { // re-attach it immediately (so current item is not lost) auto topmost = mparent->topmostMessage(); Q_ASSERT(!topmost->parent()); // groups are always viewable! topmost->setThreadingStatus(MessageItem::ParentMissing); attachMessageToGroupHeader(topmost); } } else { // mi wasn't viewable yet.. no need to attach parent attachMessageToParent(mparent, mi); } // and we're done for now } else { // so parent not found, (threadingStatus() is either MessageItem::ParentMissing or MessageItem::NonThreadable) switch (mi->threadingStatus()) { case MessageItem::ParentMissing: if (mAggregation->threading() == Aggregation::PerfectReferencesAndSubject) { // parent missing but still can be found in Pass3 mUnassignedMessageListForPass3.append(mi); // this is ~O(1) } else { // We're not doing subject based threading: will never be threaded, go straight to Pass4 mUnassignedMessageListForPass4.append(mi); // this is ~O(1) } break; case MessageItem::NonThreadable: // will never be threaded, go straight to Pass4 mUnassignedMessageListForPass4.append(mi); // this is ~O(1) break; default: // a bug for sure qCWarning(MESSAGELIST_LOG) << "ERROR: Invalid message threading status returned by findMessageParent()!"; Q_ASSERT(false); break; } } } } else { // Has a parent: either perfect parent already found or non threadable. // Since we don't end here if mi has status of parent missing then mi must not have imperfect parent. Q_ASSERT(mi->threadingStatus() != MessageItem::ImperfectParentFound); if (!mi->isViewable()) { qCWarning(MESSAGELIST_LOG) << "Non viewable message " << mi << " subject " << mi->subject().toUtf8().data(); Q_ASSERT(mi->isViewable()); } } curIndex++; // FIXME: In fact a single call to attachMessageToGroupHeader() is likely to manipulate // a subtree with a LOT of messages inside. If interactivity is favored // we should check the time really more often. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } mUnassignedMessageListForPass2.clear(); return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Fill(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Fill(ViewItemJob *job, QElapsedTimer elapsedTimer) { // In this pass we scan the a contiguous region of the underlying storage (that is // assumed to be FLAT) and create the corresponding MessageItem objects. // The deal is to show items to the user as soon as possible so in this pass we // *TRY* to attach them to the viewable tree (which is rooted on mRootItem). // Messages we're unable to attach for some reason (mainly due to threading) get appended // to mUnassignedMessageList and wait for Pass2. // We call this pass "Processing" // Should we use the receiver or the sender field for sorting ? bool bUseReceiver = mStorageModelContainsOutboundMessages; // The begin storage index of our work int curIndex = job->currentIndex(); // The end storage index of our work. int endIndex = job->endIndex(); unsigned long msgToSelect = mPreSelectionMode == PreSelectLastSelected ? mStorageModel->preSelectedMessage() : 0; MessageItem *mi = nullptr; while (curIndex <= endIndex) { // Create the message item with no parent: we'll set it later if (!mi) { mi = new MessageItem(); } else { // a MessageItem discarded by a previous iteration: reuse it. Q_ASSERT(mi->parent() == nullptr); } if (!mStorageModel->initializeMessageItem(mi, curIndex, bUseReceiver)) { // ugh qCWarning(MESSAGELIST_LOG) << "Fill of the MessageItem at storage row index " << curIndex << " failed"; curIndex++; continue; } // If we're supposed to pre-select a specific message, check if it's this one. if (msgToSelect != 0 && msgToSelect == mi->uniqueId()) { // Found, it's this one. // But actually it's not viewable (so not selectable). We must wait // until the end of the job to be 100% sure. So here we just translate // the unique id to a MessageItem pointer and wait. mLastSelectedMessageInFolder = mi; msgToSelect = 0; // already found, don't bother checking anymore } // Update the newest/oldest message, since we might be supposed to select those later if (mi->date() != static_cast(-1)) { if (!mOldestItem || mOldestItem->date() > mi->date()) { mOldestItem = mi; } if (!mNewestItem || mNewestItem->date() < mi->date()) { mNewestItem = mi; } } // Ok.. it passed the initial checks: we will not be discarding it. // Make this message item an invariant index to the underlying model storage. mInvariantRowMapper->createModelInvariantIndex(curIndex, mi); // Attempt to do threading as soon as possible (to display items to the user) if (mAggregation->threading() != Aggregation::NoThreading) { // Threading is requested // Fetch the data needed for proper threading // Add the item to the threading caches switch (mAggregation->threading()) { case Aggregation::PerfectReferencesAndSubject: mStorageModel->fillMessageItemThreadingData(mi, curIndex, StorageModel::PerfectThreadingReferencesAndSubject); // We also need to build the subject/reference-based threading cache addMessageToReferencesBasedThreadingCache(mi); addMessageToSubjectBasedThreadingCache(mi); break; case Aggregation::PerfectAndReferences: mStorageModel->fillMessageItemThreadingData(mi, curIndex, StorageModel::PerfectThreadingPlusReferences); addMessageToReferencesBasedThreadingCache(mi); break; default: mStorageModel->fillMessageItemThreadingData(mi, curIndex, StorageModel::PerfectThreadingOnly); break; } // Perfect/References threading cache mThreadingCacheMessageIdMD5ToMessageItem.insert(mi->messageIdMD5(), mi); // Register the current item into the threading cache mThreadingCache.addItemToCache(mi); // First of all look into the persistent cache qint64 parentId; Item *pParent = mThreadingCache.parentForItem(mi, parentId); if (pParent) { // We already have the parent MessageItem. Attach current message // to it and mark it as perfect mi->setThreadingStatus(MessageItem::PerfectParentFound); attachMessageToParent(pParent, mi); } else if (parentId > 0) { // We don't have the parent MessageItem yet, but we do know the // parent: delay for pass 2 when we will have the parent MessageItem // for sure. mi->setThreadingStatus(MessageItem::ParentMissing); mUnassignedMessageListForPass2.append(mi); } else if (parentId == 0) { // Message is a thread leader, skip straight to Pass4 mi->setThreadingStatus(MessageItem::NonThreadable); mUnassignedMessageListForPass4.append(mi); } else { // Check if this item is a perfect parent for some imperfectly threaded // message (that is actually attached to it, but not necessarily to the // viewable root). If it is, then remove the imperfect child from its // current parent rebuild the hierarchy on the fly. bool needsImmediateReAttach = false; if (!mThreadingCacheMessageInReplyToIdMD5ToMessageItem.isEmpty()) { // unlikely const auto lImperfectlyThreaded = mThreadingCacheMessageInReplyToIdMD5ToMessageItem.values(mi->messageIdMD5()); for (const auto it : lImperfectlyThreaded) { Q_ASSERT(it->parent()); Q_ASSERT(it->parent() != mi); if (!((it->threadingStatus() == MessageItem::ImperfectParentFound) || (it->threadingStatus() == MessageItem::ParentMissing))) { qCritical() << "Got message " << it << " with threading status" << it->threadingStatus(); Q_ASSERT_X(false, "ModelPrivate::viewItemJobStepInternalForJobPass1Fill", "Wrong threading status"); } // If the item was already attached to the view then // re-attach it immediately. This will avoid a message // being displayed for a short while in the view and then // disappear until a perfect parent isn't found. if (it->isViewable()) { needsImmediateReAttach = true; } it->setThreadingStatus(MessageItem::PerfectParentFound); attachMessageToParent(mi, it); } } // FIXME: Might look by "References" too, here... (?) // Attempt to do threading with anything we already have in caches until now // Note that this is likely to work since thread-parent messages tend // to come before thread-children messages in the folders (simply because of // date of arrival). // First of all try to find a "perfect parent", that is the message for that // we have the ID in the "In-Reply-To" field. This is actually done by using // MD5 caches of the message ids because of speed. Collisions are very unlikely. const QByteArray md5 = mi->inReplyToIdMD5(); if (!md5.isEmpty()) { // Have an In-Reply-To field MD5. // In well behaved mailing lists 70% of the threadable messages get a parent here :) pParent = mThreadingCacheMessageIdMD5ToMessageItem.value(md5, nullptr); if (pParent) { // very likely // Take care of self-referencing (which is always possible) // and circular In-Reply-To reference loops which are possible // in case this item was found to be a perfect parent for some // imperfectly threaded message just above. if ( (mi == pParent) // self referencing message || ( (mi->childItemCount() > 0) // mi already has children, this is fast to determine && pParent->hasAncestor(mi) // pParent is in the mi's children tree ) ) { // Bad, bad message.. it has In-Reply-To equal to Message-Id // or it's in a circular In-Reply-To reference loop. // Will wait for Pass2 with References-Id only qCWarning(MESSAGELIST_LOG) << "Circular In-Reply-To reference loop detected in the message tree"; mUnassignedMessageListForPass2.append(mi); } else { // wow, got a perfect parent for this message! mi->setThreadingStatus(MessageItem::PerfectParentFound); attachMessageToParent(pParent, mi); // we're done with this message (also for Pass2) } } else { // got no parent // will have to wait Pass2 mUnassignedMessageListForPass2.append(mi); } } else { // No In-Reply-To header. bool mightHaveOtherMeansForThreading; switch (mAggregation->threading()) { case Aggregation::PerfectReferencesAndSubject: mightHaveOtherMeansForThreading = mi->subjectIsPrefixed() || !mi->referencesIdMD5().isEmpty(); break; case Aggregation::PerfectAndReferences: mightHaveOtherMeansForThreading = !mi->referencesIdMD5().isEmpty(); break; case Aggregation::PerfectOnly: mightHaveOtherMeansForThreading = false; break; default: // BUG: there shouldn't be other values (NoThreading is excluded in an upper branch) Q_ASSERT(false); mightHaveOtherMeansForThreading = false; // make gcc happy break; } if (mightHaveOtherMeansForThreading) { // We might have other means for threading this message, wait until Pass2 mUnassignedMessageListForPass2.append(mi); } else { // No other means for threading this message. This is either // a standalone message or a thread leader. // If there is no grouping in effect or thread leaders are just the "topmost" // messages then we might be done with this one. if ( (mAggregation->grouping() == Aggregation::NoGrouping) || (mAggregation->threadLeader() == Aggregation::TopmostMessage) ) { // We're done with this message: it will be surely either toplevel (no grouping in effect) // or a thread leader with a well defined group. Do it :) //qCDebug(MESSAGELIST_LOG) << "Setting message status from " << mi->threadingStatus() << " to non threadable (1) " << mi; mi->setThreadingStatus(MessageItem::NonThreadable); // Locate the parent group for this item attachMessageToGroupHeader(mi); // we're done with this message (also for Pass2) } else { // Threads belong to the most recent message in the thread. This means // that we have to wait until Pass2 or Pass3 to assign a group. mUnassignedMessageListForPass2.append(mi); } } } if (needsImmediateReAttach && !mi->isViewable()) { // The item gathered previously viewable children. They must be immediately // re-shown. So this item must currently be attached to the view. // This is a temporary measure: it will be probably still moved. MessageItem *topmost = mi->topmostMessage(); Q_ASSERT(topmost->threadingStatus() == MessageItem::ParentMissing); attachMessageToGroupHeader(topmost); } } } else { // else no threading requested: we don't even need Pass2 // set not threadable status (even if it might be not true, but in this mode we don't care) //qCDebug(MESSAGELIST_LOG) << "Setting message status from " << mi->threadingStatus() << " to non threadable (2) " << mi; mi->setThreadingStatus(MessageItem::NonThreadable); // locate the parent group for this item if (mAggregation->grouping() == Aggregation::NoGrouping) { attachMessageToParent(mRootItem, mi); // no groups requested, attach directly to root } else { attachMessageToGroupHeader(mi); } // we're done with this message (also for Pass2) } mi = nullptr; // this item was pushed somewhere, create a new one at next iteration curIndex++; if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } if (mi) { delete mi; } return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Cleanup(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Cleanup(ViewItemJob *job, QElapsedTimer elapsedTimer) { Q_ASSERT(mModelForItemFunctions); // UI must be not disconnected here // In this pass we remove the MessageItem objects that are present in the job // and put their children in the unassigned message list. // Note that this list in fact contains MessageItem objects (we need dynamic_cast<>). QList< ModelInvariantIndex * > *invalidatedMessages = job->invariantIndexList(); // We don't shrink the invalidatedMessages because it's basically an array. // It's faster to traverse an array of N entries than to remove K>0 entries // one by one and to traverse the remaining N-K entries. // The begin index of our work int curIndex = job->currentIndex(); // The end index of our work. int endIndex = job->endIndex(); if (curIndex == job->startIndex()) { Q_ASSERT(mOrphanChildrenHash.isEmpty()); } while (curIndex <= endIndex) { // Get the underlying storage message data... auto dyingMessage = dynamic_cast< MessageItem * >(invalidatedMessages->at(curIndex)); // This MUST NOT be null (otherwise we have a bug somewhere in this file). Q_ASSERT(dyingMessage); // If we were going to pre-select this message but we were interrupted // *before* it was actually made viewable, we just clear the pre-selection pointer // and unique id (abort pre-selection). if (dyingMessage == mLastSelectedMessageInFolder) { mLastSelectedMessageInFolder = nullptr; mPreSelectionMode = PreSelectNone; } // remove the message from any pending user job if (mPersistentSetManager) { mPersistentSetManager->removeMessageItemFromAllSets(dyingMessage); if (mPersistentSetManager->setCount() < 1) { delete mPersistentSetManager; mPersistentSetManager = nullptr; } } // Remove the message from threading cache before we start moving up the // children, so that they don't get mislead by the cache mThreadingCache.expireParent(dyingMessage); if (dyingMessage->parent()) { // Handle saving the current selection: if this item was the current before the step // then zero it out. We have killed it and it's OK for the current item to change. if (dyingMessage == mCurrentItemToRestoreAfterViewItemJobStep) { Q_ASSERT(dyingMessage->isViewable()); // Try to select the item below the removed one as it helps in doing a "readon" of emails: // you read a message, decide to delete it and then go to the next. // Qt tends to select the message above the removed one instead (this is a hardcoded logic in // QItemSelectionModelPrivate::_q_rowsAboutToBeRemoved()). mCurrentItemToRestoreAfterViewItemJobStep = mView->messageItemAfter(dyingMessage, MessageTypeAny, false); if (!mCurrentItemToRestoreAfterViewItemJobStep) { // There is no item below. Try the item above. // We still do it better than qt which tends to find the *thread* above // instead of the item above. mCurrentItemToRestoreAfterViewItemJobStep = mView->messageItemBefore(dyingMessage, MessageTypeAny, false); } Q_ASSERT((!mCurrentItemToRestoreAfterViewItemJobStep) || mCurrentItemToRestoreAfterViewItemJobStep->isViewable()); } if ( dyingMessage->isViewable() && ((dyingMessage)->childItemCount() > 0) // has children && mView->isExpanded(q->index(dyingMessage, 0)) // is actually expanded ) { saveExpandedStateOfSubtree(dyingMessage); } auto oldParent = dyingMessage->parent(); oldParent->takeChildItem(q, dyingMessage); // FIXME: This can generate many message movements.. it would be nicer // to start from messages that are higher in the hierarchy so // we would need to move less stuff above. if (oldParent != mRootItem) { messageDetachedUpdateParentProperties(oldParent, dyingMessage); } // We might have already removed its parent from the view, so it // might already be in the orphan child hash... if (dyingMessage->threadingStatus() == MessageItem::ParentMissing) { mOrphanChildrenHash.remove(dyingMessage); // this can turn to a no-op (dyingMessage not present in fact) } } else { // The dying message had no parent: this should happen only if it's already an orphan Q_ASSERT(dyingMessage->threadingStatus() == MessageItem::ParentMissing); Q_ASSERT(mOrphanChildrenHash.contains(dyingMessage)); Q_ASSERT(dyingMessage != mCurrentItemToRestoreAfterViewItemJobStep); mOrphanChildrenHash.remove(dyingMessage); } if (mAggregation->threading() != Aggregation::NoThreading) { // Threading is requested: remove the message from threading caches. // Remove from the cache of potential parent items mThreadingCacheMessageIdMD5ToMessageItem.remove(dyingMessage->messageIdMD5()); // If we also have a cache for subject/reference-based threading then remove the message from there too if (mAggregation->threading() == Aggregation::PerfectReferencesAndSubject) { removeMessageFromReferencesBasedThreadingCache(dyingMessage); removeMessageFromSubjectBasedThreadingCache(dyingMessage); } else if (mAggregation->threading() == Aggregation::PerfectAndReferences) { removeMessageFromReferencesBasedThreadingCache(dyingMessage); } // If this message wasn't perfectly parented then it might still be in another cache. switch (dyingMessage->threadingStatus()) { case MessageItem::ImperfectParentFound: case MessageItem::ParentMissing: if (!dyingMessage->inReplyToIdMD5().isEmpty()) { mThreadingCacheMessageInReplyToIdMD5ToMessageItem.remove(dyingMessage->inReplyToIdMD5()); } break; default: Q_ASSERT(!mThreadingCacheMessageInReplyToIdMD5ToMessageItem.contains(dyingMessage->inReplyToIdMD5(), dyingMessage)); // make gcc happy break; } } while (auto childItem = dyingMessage->firstChildItem()) { auto childMessage = dynamic_cast< MessageItem * >(childItem); Q_ASSERT(childMessage); dyingMessage->takeChildItem(q, childMessage); if (mAggregation->threading() != Aggregation::NoThreading) { if (childMessage->threadingStatus() == MessageItem::PerfectParentFound) { // If the child message was perfectly parented then now it had // lost its perfect parent. Add to the cache of imperfectly parented. if (!childMessage->inReplyToIdMD5().isEmpty()) { Q_ASSERT(!mThreadingCacheMessageInReplyToIdMD5ToMessageItem.contains(childMessage->inReplyToIdMD5(), childMessage)); mThreadingCacheMessageInReplyToIdMD5ToMessageItem.insert(childMessage->inReplyToIdMD5(), childMessage); } } } // Parent is gone childMessage->setThreadingStatus(MessageItem::ParentMissing); // If the child (or any message in its subtree) is going to be selected, // then we must immediately reattach it to a temporary group in order for the // selection to be preserved across multiple steps. Otherwise we could end // with the child-to-be-selected being non viewable at the end // of the view job step. Attach to a temporary group. if ( // child is going to be re-selected (childMessage == mCurrentItemToRestoreAfterViewItemJobStep) || ( // there is a message that is going to be re-selected mCurrentItemToRestoreAfterViewItemJobStep &&// that message is in the childMessage subtree mCurrentItemToRestoreAfterViewItemJobStep->hasAncestor(childMessage) ) ) { attachMessageToGroupHeader(childMessage); Q_ASSERT(childMessage->isViewable()); } mOrphanChildrenHash.insert(childMessage, childMessage); } if (mNewestItem == dyingMessage) { mNewestItem = nullptr; } if (mOldestItem == dyingMessage) { mOldestItem = nullptr; } delete dyingMessage; curIndex++; // FIXME: Maybe we should check smaller steps here since the // code above can generate large message tree movements // for each single item we sweep in the invalidatedMessages list. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } // We looped over the entire deleted message list. job->setCurrentIndex(endIndex + 1); // A quick last cleaning pass: this is usually very fast so we don't have a real // Pass enumeration for it. We just include it as trailer of Pass1Cleanup to be executed // when job->currentIndex() > job->endIndex(); // We move all the messages from the orphan child hash to the unassigned message // list and get them ready for the standard Pass2. auto it = mOrphanChildrenHash.begin(); auto end = mOrphanChildrenHash.end(); curIndex = 0; while (it != end) { mUnassignedMessageListForPass2.append(*it); it = mOrphanChildrenHash.erase(it); // This is still interruptible curIndex++; // FIXME: We could take "larger" steps here if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (it != mOrphanChildrenHash.end()) { return ViewItemJobInterrupted; } } } } return ViewItemJobCompleted; } -ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Update(ViewItemJob *job, const QElapsedTimer &elapsedTimer) +ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJobPass1Update(ViewItemJob *job, QElapsedTimer elapsedTimer) { Q_ASSERT(mModelForItemFunctions); // UI must be not disconnected here // In this pass we simply update the MessageItem objects that are present in the job. // Note that this list in fact contains MessageItem objects (we need dynamic_cast<>). auto messagesThatNeedUpdate = job->invariantIndexList(); // We don't shrink the messagesThatNeedUpdate because it's basically an array. // It's faster to traverse an array of N entries than to remove K>0 entries // one by one and to traverse the remaining N-K entries. // The begin index of our work int curIndex = job->currentIndex(); // The end index of our work. int endIndex = job->endIndex(); while (curIndex <= endIndex) { // Get the underlying storage message data... auto message = dynamic_cast(messagesThatNeedUpdate->at(curIndex)); // This MUST NOT be null (otherwise we have a bug somewhere in this file). Q_ASSERT(message); int row = mInvariantRowMapper->modelInvariantIndexToModelIndexRow(message); if (row < 0) { // Must have been invalidated (so it's basically about to be deleted) Q_ASSERT(!message->isValid()); // Skip it here. curIndex++; continue; } time_t prevDate = message->date(); time_t prevMaxDate = message->maxDate(); bool toDoStatus = message->status().isToAct(); bool prevUnreadStatus = !message->status().isRead(); bool prevImportantStatus = message->status().isImportant(); // The subject/reference based threading cache is sorted by date: we must remove // the item and re-insert it since updateMessageItemData() may change the date too. if (mAggregation->threading() == Aggregation::PerfectReferencesAndSubject) { removeMessageFromReferencesBasedThreadingCache(message); removeMessageFromSubjectBasedThreadingCache(message); } else if (mAggregation->threading() == Aggregation::PerfectAndReferences) { removeMessageFromReferencesBasedThreadingCache(message); } // Do update mStorageModel->updateMessageItemData(message, row); QModelIndex idx = q->index(message, 0); Q_EMIT q->dataChanged(idx, idx); // Reinsert the item to the cache, if needed if (mAggregation->threading() == Aggregation::PerfectReferencesAndSubject) { addMessageToReferencesBasedThreadingCache(message); addMessageToSubjectBasedThreadingCache(message); } else if (mAggregation->threading() == Aggregation::PerfectAndReferences) { addMessageToReferencesBasedThreadingCache(message); } int propertyChangeMask = 0; if (prevDate != message->date()) { propertyChangeMask |= DateChanged; } if (prevMaxDate != message->maxDate()) { propertyChangeMask |= MaxDateChanged; } if (toDoStatus != message->status().isToAct()) { propertyChangeMask |= ActionItemStatusChanged; } if (prevUnreadStatus != (!message->status().isRead())) { propertyChangeMask |= UnreadStatusChanged; } if (prevImportantStatus != (!message->status().isImportant())) { propertyChangeMask |= ImportantStatusChanged; } if (propertyChangeMask) { // Some message data has changed // now we need to handle the changes that might cause re-grouping/re-sorting // and propagate them to the parents. Item *pParent = message->parent(); if (pParent && (pParent != mRootItem)) { // The following function will return true if itemParent may be affected by the change. // If the itemParent isn't affected, we stop climbing. if (handleItemPropertyChanges(propertyChangeMask, pParent, message)) { Q_ASSERT(message->parent()); // handleItemPropertyChanges() must never leave an item detached // Note that actually message->parent() may be different than pParent since // handleItemPropertyChanges() may have re-grouped it. // Time to propagate up. propagateItemPropertiesToParent(message); } } // else there is no parent so the item isn't attached to the view: re-grouping/re-sorting not needed. } // else message data didn't change an there is nothing interesting to do // (re-)apply the filter, if needed if (mFilter && message->isViewable()) { // In all the other cases we (re-)apply the filter to the topmost subtree that this message is in. Item *pTopMostNonRoot = message->topmostNonRoot(); Q_ASSERT(pTopMostNonRoot); Q_ASSERT(pTopMostNonRoot != mRootItem); Q_ASSERT(pTopMostNonRoot->parent() == mRootItem); // FIXME: The call below works, but it's expensive when we are updating // a lot of items with filtering enabled. This is because the updated // items are likely to be in the same subtree which we then filter multiple times. // A point for us is that when filtering there shouldn't be really many // items in the view so the user isn't going to update a lot of them at once... // Well... anyway, the alternative would be to write yet another // specialized routine that would update only the "message" item // above and climb up eventually hiding parents (without descending the sibling subtrees again). // If people complain about performance in this particular case I'll consider that solution. applyFilterToSubtree(pTopMostNonRoot, QModelIndex()); } // otherwise there is no filter or the item isn't viewable: very likely // left detached while propagating property changes. Will filter it // on reattach. // Done updating this message curIndex++; // FIXME: Maybe we should check smaller steps here since the // code above can generate large message tree movements // for each single item we sweep in the messagesThatNeedUpdate list. if ((curIndex % mViewItemJobStepMessageCheckCount) == 0) { if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { if (curIndex <= endIndex) { job->setCurrentIndex(curIndex); return ViewItemJobInterrupted; } } } } return ViewItemJobCompleted; } ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternalForJob(ViewItemJob *job, const QElapsedTimer &elapsedTimer) { // This function does a timed chunk of work for a single Fill View job. // It attempts to process messages until a timeout forces it to return to the caller. // A macro would improve readability here but since this is a good point // to place debugger breakpoints then we need it explicitly. // A (template) helper would need to pass many parameters and would not be inlined... if (job->currentPass() == ViewItemJob::Pass1Fill) { // We're in Pass1Fill of the job. switch (viewItemJobStepInternalForJobPass1Fill(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; break; case ViewItemJobCompleted: // pass 1 has been completed // # TODO: Refactor this, make it virtual or whatever, but switch == bad, code duplication etc job->setCurrentPass(ViewItemJob::Pass2); job->setStartIndex(0); job->setEndIndex(mUnassignedMessageListForPass2.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } else if (job->currentPass() == ViewItemJob::Pass1Cleanup) { // We're in Pass1Cleanup of the job. switch (viewItemJobStepInternalForJobPass1Cleanup(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; break; case ViewItemJobCompleted: // pass 1 has been completed job->setCurrentPass(ViewItemJob::Pass2); job->setStartIndex(0); job->setEndIndex(mUnassignedMessageListForPass2.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } else if (job->currentPass() == ViewItemJob::Pass1Update) { // We're in Pass1Update of the job. switch (viewItemJobStepInternalForJobPass1Update(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; break; case ViewItemJobCompleted: // pass 1 has been completed // Since Pass2, Pass3 and Pass4 are empty for an Update operation // we simply skip them. (TODO: Triple-verify this assertion...). job->setCurrentPass(ViewItemJob::Pass5); job->setStartIndex(0); job->setEndIndex(mGroupHeadersThatNeedUpdate.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } // Pass1Fill/Pass1Cleanup/Pass1Update has been already completed. if (job->currentPass() == ViewItemJob::Pass2) { // We're in Pass2 of the job. switch (viewItemJobStepInternalForJobPass2(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; break; case ViewItemJobCompleted: // pass 2 has been completed job->setCurrentPass(ViewItemJob::Pass3); job->setStartIndex(0); job->setEndIndex(mUnassignedMessageListForPass3.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } if (job->currentPass() == ViewItemJob::Pass3) { // We're in Pass3 of the job. switch (viewItemJobStepInternalForJobPass3(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; case ViewItemJobCompleted: // pass 3 has been completed job->setCurrentPass(ViewItemJob::Pass4); job->setStartIndex(0); job->setEndIndex(mUnassignedMessageListForPass4.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } if (job->currentPass() == ViewItemJob::Pass4) { // We're in Pass4 of the job. switch (viewItemJobStepInternalForJobPass4(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: propagate status to caller return ViewItemJobInterrupted; case ViewItemJobCompleted: // pass 4 has been completed job->setCurrentPass(ViewItemJob::Pass5); job->setStartIndex(0); job->setEndIndex(mGroupHeadersThatNeedUpdate.count() - 1); // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if (elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) { return ViewItemJobInterrupted; } // else proceed with the next pass break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } // Pass4 has been already completed. Proceed to Pass5. return viewItemJobStepInternalForJobPass5(job, elapsedTimer); } #ifdef KDEPIM_FOLDEROPEN_PROFILE // Namespace to collect all the vars and functions for KDEPIM_FOLDEROPEN_PROFILE namespace Stats { // Number of existing jobs/passes static const int numberOfPasses = ViewItemJob::LastIndex; // The pass in the last call of viewItemJobStepInternal(), used to detect when // a new pass starts static int lastPass = -1; // Total number of messages in the folder static int totalMessages; // Per-Job data static int numElements[numberOfPasses]; static int totalTime[numberOfPasses]; static int chunks[numberOfPasses]; // Time, in msecs for some special operations static int expandingTreeTime; static int layoutChangeTime; // Descriptions of the job, for nicer debug output static const char *jobDescription[numberOfPasses] = { "Creating items from messages and simple threading", "Removing messages", "Updating messages", "Additional Threading", "Subject-Based threading", "Grouping", "Group resorting + cleanup" }; // Timer to track time between start of first job and end of last job static QTime firstStartTime; // Timer to track time the current job takes static QTime currentJobStartTime; // Zeros the stats, to be called when the first job starts static void resetStats() { totalMessages = 0; layoutChangeTime = 0; expandingTreeTime = 0; lastPass = -1; for (int i = 0; i < numberOfPasses; ++i) { numElements[i] = 0; totalTime[i] = 0; chunks[i] = 0; } } } // namespace Stats void ModelPrivate::printStatistics() { using namespace Stats; int totalTotalTime = 0; int completeTime = firstStartTime.elapsed(); for (int i = 0; i < numberOfPasses; ++i) { totalTotalTime += totalTime[i]; } float msgPerSecond = totalMessages / (totalTotalTime / 1000.0f); float msgPerSecondComplete = totalMessages / (completeTime / 1000.0f); int messagesWithSameSubjectAvg = 0; int messagesWithSameSubjectMax = 0; for (const auto messages : qAsConst(mThreadingCacheMessageSubjectMD5ToMessageItem)) { if (messages->size() > messagesWithSameSubjectMax) { messagesWithSameSubjectMax = messages->size(); } messagesWithSameSubjectAvg += messages->size(); } messagesWithSameSubjectAvg = messagesWithSameSubjectAvg / (float)mThreadingCacheMessageSubjectMD5ToMessageItem.size(); int totalThreads = 0; if (!mGroupHeaderItemHash.isEmpty()) { foreach (const GroupHeaderItem *groupHeader, mGroupHeaderItemHash) { totalThreads += groupHeader->childItemCount(); } } else { totalThreads = mRootItem->childItemCount(); } qCDebug(MESSAGELIST_LOG) << "Finished filling the view with" << totalMessages << "messages"; qCDebug(MESSAGELIST_LOG) << "That took" << totalTotalTime << "msecs inside the model and" << completeTime << "in total."; qCDebug(MESSAGELIST_LOG) << (totalTotalTime / (float)completeTime) * 100.0f << "percent of the time was spent in the model."; qCDebug(MESSAGELIST_LOG) << "Time for layoutChanged(), in msecs:" << layoutChangeTime << "(" << (layoutChangeTime / (float)totalTotalTime) * 100.0f << "percent )"; qCDebug(MESSAGELIST_LOG) << "Time to expand tree, in msecs:" << expandingTreeTime << "(" << (expandingTreeTime / (float)totalTotalTime) * 100.0f << "percent )"; qCDebug(MESSAGELIST_LOG) << "Number of messages per second in the model:" << msgPerSecond; qCDebug(MESSAGELIST_LOG) << "Number of messages per second in total:" << msgPerSecondComplete; qCDebug(MESSAGELIST_LOG) << "Number of threads:" << totalThreads; qCDebug(MESSAGELIST_LOG) << "Number of groups:" << mGroupHeaderItemHash.size(); qCDebug(MESSAGELIST_LOG) << "Messages per thread:" << totalMessages / (float)totalThreads; qCDebug(MESSAGELIST_LOG) << "Threads per group:" << totalThreads / (float)mGroupHeaderItemHash.size(); qCDebug(MESSAGELIST_LOG) << "Messages with the same subject:" << "Max:" << messagesWithSameSubjectMax << "Avg:" << messagesWithSameSubjectAvg; qCDebug(MESSAGELIST_LOG); qCDebug(MESSAGELIST_LOG) << "Now follows a breakdown of the jobs."; qCDebug(MESSAGELIST_LOG); for (int i = 0; i < numberOfPasses; ++i) { if (totalTime[i] == 0) { continue; } float elementsPerSecond = numElements[i] / (totalTime[i] / 1000.0f); float percent = totalTime[i] / (float)totalTotalTime * 100.0f; qCDebug(MESSAGELIST_LOG) << "----------------------------------------------"; qCDebug(MESSAGELIST_LOG) << "Job" << i + 1 << "(" << jobDescription[i] << ")"; qCDebug(MESSAGELIST_LOG) << "Share of complete time:" << percent << "percent"; qCDebug(MESSAGELIST_LOG) << "Time in msecs:" << totalTime[i]; qCDebug(MESSAGELIST_LOG) << "Number of elements:" << numElements[i]; // TODO: map of element string qCDebug(MESSAGELIST_LOG) << "Elements per second:" << elementsPerSecond; qCDebug(MESSAGELIST_LOG) << "Number of chunks:" << chunks[i]; qCDebug(MESSAGELIST_LOG); } qCDebug(MESSAGELIST_LOG) << "=========================================================="; resetStats(); } #endif ModelPrivate::ViewItemJobResult ModelPrivate::viewItemJobStepInternal() { // This function does a timed chunk of work in our View Fill operation. // It attempts to do processing until it either runs out of jobs // to be done or a timeout forces it to interrupt and jump back to the caller. QElapsedTimer elapsedTimer; elapsedTimer.start(); while (!mViewItemJobs.isEmpty()) { // Have a job to do. ViewItemJob *job = mViewItemJobs.first(); #ifdef KDEPIM_FOLDEROPEN_PROFILE // Here we check if an old job has just completed or if we are at the start of the // first job. We then initialize job data stuff and timers based on this. const int currentPass = job->currentPass(); const bool firstChunk = currentPass != Stats::lastPass; if (currentPass != Stats::lastPass && Stats::lastPass != -1) { Stats::totalTime[Stats::lastPass] = Stats::currentJobStartTime.elapsed(); } const bool firstJob = job->currentPass() == ViewItemJob::Pass1Fill && firstChunk; const int elements = job->endIndex() - job->startIndex(); if (firstJob) { Stats::resetStats(); Stats::totalMessages = elements; Stats::firstStartTime.restart(); } if (firstChunk) { Stats::numElements[currentPass] = elements; Stats::currentJobStartTime.restart(); } Stats::chunks[currentPass]++; Stats::lastPass = currentPass; #endif mViewItemJobStepIdleInterval = job->idleInterval(); mViewItemJobStepChunkTimeout = job->chunkTimeout(); mViewItemJobStepMessageCheckCount = job->messageCheckCount(); if (job->disconnectUI()) { mModelForItemFunctions = nullptr; // disconnect the UI for this job Q_ASSERT(mLoading); // this must be true in the first job // FIXME: Should assert yet more that this is the very first job for this StorageModel // Asserting only mLoading is not enough as we could be using a two-jobs loading strategy // or this could be a job enqueued before the first job has completed. } else { // With a connected UI we need to avoid the view to update the scrollbars at EVERY insertion or expansion. // QTreeViewPrivate::updateScrollBars() is very expensive as it loops through ALL the items in the view every time. // We can't disable the function directly as it's hidden in the private data object of QTreeView // but we can disable the parent QTreeView::updateGeometries() instead. // We will trigger it "manually" at the end of the step. mView->ignoreUpdateGeometries(true); // Ok.. I know that this seems unbelieveable but disabling updates actually // causes a (significant) performance loss in most cases. This is probably because QTreeView // uses delayed layouts when updates are disabled which should be delayed but in // fact are "forced" by next item insertions. The delayed layout algorithm, then // is probably slower than the non-delayed one. // Disabling the paintEvent() doesn't seem to work either. //mView->setUpdatesEnabled( false ); } switch (viewItemJobStepInternalForJob(job, elapsedTimer)) { case ViewItemJobInterrupted: // current job interrupted by timeout: will propagate status to caller // but before this, give some feedback to the user // FIXME: This is now inaccurate, think of something else switch (job->currentPass()) { case ViewItemJob::Pass1Fill: case ViewItemJob::Pass1Cleanup: case ViewItemJob::Pass1Update: Q_EMIT q->statusMessage(i18np("Processed 1 Message of %2", "Processed %1 Messages of %2", job->currentIndex() - job->startIndex(), job->endIndex() - job->startIndex() + 1)); break; case ViewItemJob::Pass2: Q_EMIT q->statusMessage(i18np("Threaded 1 Message of %2", "Threaded %1 Messages of %2", job->currentIndex() - job->startIndex(), job->endIndex() - job->startIndex() + 1)); break; case ViewItemJob::Pass3: Q_EMIT q->statusMessage(i18np("Threaded 1 Message of %2", "Threaded %1 Messages of %2", job->currentIndex() - job->startIndex(), job->endIndex() - job->startIndex() + 1)); break; case ViewItemJob::Pass4: Q_EMIT q->statusMessage(i18np("Grouped 1 Thread of %2", "Grouped %1 Threads of %2", job->currentIndex() - job->startIndex(), job->endIndex() - job->startIndex() + 1)); break; case ViewItemJob::Pass5: Q_EMIT q->statusMessage(i18np("Updated 1 Group of %2", "Updated %1 Groups of %2", job->currentIndex() - job->startIndex(), job->endIndex() - job->startIndex() + 1)); break; default: break; } if (!job->disconnectUI()) { mView->ignoreUpdateGeometries(false); // explicit call to updateGeometries() here mView->updateGeometries(); } return ViewItemJobInterrupted; break; case ViewItemJobCompleted: // If this job worked with a disconnected UI, Q_EMIT layoutChanged() // to reconnect it. We go back to normal operation now. if (job->disconnectUI()) { mModelForItemFunctions = q; // This call would destroy the expanded state of items. // This is why when mModelForItemFunctions was 0 we didn't actually expand them // but we just set a "ExpandNeeded" mark... #ifdef KDEPIM_FOLDEROPEN_PROFILE QTime layoutChangedTimer; layoutChangedTimer.start(); #endif mView->modelAboutToEmitLayoutChanged(); Q_EMIT q->layoutChanged(); mView->modelEmittedLayoutChanged(); #ifdef KDEPIM_FOLDEROPEN_PROFILE Stats::layoutChangeTime = layoutChangedTimer.elapsed(); QTime expandingTime; expandingTime.start(); #endif // expand all the items that need it in a single sweep // FIXME: This takes quite a lot of time, it could be made an interruptible job auto rootChildItems = mRootItem->childItems(); if (rootChildItems) { for (const auto it : qAsConst(*rootChildItems)) { if (it->initialExpandStatus() == Item::ExpandNeeded) { syncExpandedStateOfSubtree(it); } } } #ifdef KDEPIM_FOLDEROPEN_PROFILE Stats::expandingTreeTime = expandingTime.elapsed(); #endif } else { mView->ignoreUpdateGeometries(false); // explicit call to updateGeometries() here mView->updateGeometries(); } // this job has been completed delete mViewItemJobs.takeFirst(); #ifdef KDEPIM_FOLDEROPEN_PROFILE // Last job finished! Stats::totalTime[currentPass] = Stats::currentJobStartTime.elapsed(); printStatistics(); #endif // take care of small jobs which never timeout by themselves because // of a small number of messages. At the end of each job check // the time used and if we're timeoutting and there is another job // then interrupt. if ((elapsedTimer.elapsed() > mViewItemJobStepChunkTimeout) || (elapsedTimer.elapsed() < 0)) { if (!mViewItemJobs.isEmpty()) { return ViewItemJobInterrupted; } // else it's completed in fact } // else proceed with the next job break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } } // no more jobs Q_EMIT q->statusMessage(i18nc("@info:status Finished view fill", "Ready")); return ViewItemJobCompleted; } void ModelPrivate::viewItemJobStep() { // A single step in the View Fill operation. // This function wraps viewItemJobStepInternal() which does the step job // and either completes it or stops because of a timeout. // If the job is stopped then we start a zero-msecs timer to call us // back and resume the job. Otherwise we're just done. mViewItemJobStepStartTime = ::time(nullptr); if (mFillStepTimer.isActive()) { mFillStepTimer.stop(); } if (!mStorageModel) { return; // nothing more to do } // Save the current item in the view as our process may // cause items to be reparented (and QTreeView will forget the current item in the meantime). // This machinery is also needed when we're about to remove items from the view in // a cleanup job: we'll be trying to set as current the item after the one removed. QModelIndex currentIndexBeforeStep = mView->currentIndex(); Item *currentItemBeforeStep = currentIndexBeforeStep.isValid() ? static_cast< Item * >(currentIndexBeforeStep.internalPointer()) : nullptr; // mCurrentItemToRestoreAfterViewItemJobStep will be zeroed out if it's killed mCurrentItemToRestoreAfterViewItemJobStep = currentItemBeforeStep; // Save the current item position in the viewport as QTreeView fails to keep // the current item in the sample place when items are added or removed... QRect rectBeforeViewItemJobStep; const bool lockView = mView->isScrollingLocked(); // This is generally SLOW AS HELL... (so we avoid it if we lock the view and thus don't need it) if (mCurrentItemToRestoreAfterViewItemJobStep && (!lockView)) { rectBeforeViewItemJobStep = mView->visualRect(currentIndexBeforeStep); } // FIXME: If the current item is NOT in the view, preserve the position // of the top visible item. This will make the view move yet less. // Insulate the View from (very likely spurious) "currentChanged()" signals. mView->ignoreCurrentChanges(true); // And go to real work. switch (viewItemJobStepInternal()) { case ViewItemJobInterrupted: // Operation timed out, need to resume in a while if (!mInLengthyJobBatch) { mInLengthyJobBatch = true; } mFillStepTimer.start(mViewItemJobStepIdleInterval); // this is a single shot timer connected to viewItemJobStep() // and go dealing with current/selection out of the switch. break; case ViewItemJobCompleted: // done :) Q_ASSERT(mModelForItemFunctions); // UI must be no (longer) disconnected in this state // Ask the view to remove the eventual busy indications if (mInLengthyJobBatch) { mInLengthyJobBatch = false; } if (mLoading) { mLoading = false; mView->modelFinishedLoading(); } // Apply pre-selection, if any if (mPreSelectionMode != PreSelectNone) { mView->ignoreCurrentChanges(false); bool bSelectionDone = false; switch (mPreSelectionMode) { case PreSelectLastSelected: // fall down break; case PreSelectFirstUnreadCentered: bSelectionDone = mView->selectFirstMessageItem(MessageTypeUnreadOnly, true); // center break; case PreSelectOldestCentered: mView->setCurrentMessageItem(mOldestItem, true /* center */); bSelectionDone = true; break; case PreSelectNewestCentered: mView->setCurrentMessageItem(mNewestItem, true /* center */); bSelectionDone = true; break; case PreSelectNone: // deal with selection below break; default: qCWarning(MESSAGELIST_LOG) << "ERROR: Unrecognized pre-selection mode " << static_cast(mPreSelectionMode); break; } if ((!bSelectionDone) && (mPreSelectionMode != PreSelectNone)) { // fallback to last selected, if possible if (mLastSelectedMessageInFolder) { // we found it in the loading process: select and jump out mView->setCurrentMessageItem(mLastSelectedMessageInFolder); bSelectionDone = true; } } if (bSelectionDone) { mLastSelectedMessageInFolder = nullptr; mPreSelectionMode = PreSelectNone; return; // already taken care of current / selection } } // deal with current/selection out of the switch break; default: // This is *really* a BUG qCWarning(MESSAGELIST_LOG) << "ERROR: returned an invalid result"; Q_ASSERT(false); break; } // Everything else here deals with the selection // If UI is disconnected then we don't have anything else to do here if (!mModelForItemFunctions) { mView->ignoreCurrentChanges(false); return; } // Restore current/selection and/or scrollbar position if (mCurrentItemToRestoreAfterViewItemJobStep) { bool stillIgnoringCurrentChanges = true; // If the assert below fails then the previously current item got detached // and didn't get reattached in the step: this should never happen. Q_ASSERT(mCurrentItemToRestoreAfterViewItemJobStep->isViewable()); // Check if the current item changed QModelIndex currentIndexAfterStep = mView->currentIndex(); Item *currentAfterStep = currentIndexAfterStep.isValid() ? static_cast< Item * >(currentIndexAfterStep.internalPointer()) : nullptr; if (mCurrentItemToRestoreAfterViewItemJobStep != currentAfterStep) { // QTreeView lost the current item... if (mCurrentItemToRestoreAfterViewItemJobStep != currentItemBeforeStep) { // Some view job code expects us to actually *change* the current item. // This is done by the cleanup step which removes items and tries // to set as current the item *after* the removed one, if possible. // We need the view to handle the change though. stillIgnoringCurrentChanges = false; mView->ignoreCurrentChanges(false); } else { // we just have to restore the old current item. The code // outside shouldn't have noticed that we lost it (e.g. the message viewer // still should have the old message opened). So we don't need to // actually notify the view of the restored setting. } // Restore it qCDebug(MESSAGELIST_LOG) << "Gonna restore current here" << mCurrentItemToRestoreAfterViewItemJobStep->subject(); mView->setCurrentIndex(q->index(mCurrentItemToRestoreAfterViewItemJobStep, 0)); } else { // The item we're expected to set as current is already current if (mCurrentItemToRestoreAfterViewItemJobStep != currentItemBeforeStep) { // But we have changed it in the job step. // This means that: we have deleted the current item and chosen a // new candidate as current but Qt also has chosen it as candidate // and already made it current. The problem is that (as of Qt 4.4) // it probably didn't select it. if (!mView->selectionModel()->hasSelection()) { stillIgnoringCurrentChanges = false; mView->ignoreCurrentChanges(false); qCDebug(MESSAGELIST_LOG) << "Gonna restore selection here" << mCurrentItemToRestoreAfterViewItemJobStep->subject(); QItemSelection selection; selection.append(QItemSelectionRange(q->index(mCurrentItemToRestoreAfterViewItemJobStep, 0))); mView->selectionModel()->select(selection, QItemSelectionModel::Select | QItemSelectionModel::Rows); } } } // FIXME: If it was selected before the change, then re-select it (it may happen that it's not) if (!lockView) { // we prefer to keep the currently selected item steady in the view QRect rectAfterViewItemJobStep = mView->visualRect(q->index(mCurrentItemToRestoreAfterViewItemJobStep, 0)); if (rectBeforeViewItemJobStep.y() != rectAfterViewItemJobStep.y()) { // QTreeView lost its position... mView->verticalScrollBar()->setValue(mView->verticalScrollBar()->value() + rectAfterViewItemJobStep.y() - rectBeforeViewItemJobStep.y()); } } // and kill the insulation, if not yet done if (stillIgnoringCurrentChanges) { mView->ignoreCurrentChanges(false); } return; } // Either there was no current item before, or it was lost in a cleanup step and another candidate for // current item couldn't be found (possibly empty view) mView->ignoreCurrentChanges(false); if (currentItemBeforeStep) { // lost in a cleanup.. // tell the view that we have a new current, this time with no insulation mView->slotSelectionChanged(QItemSelection(), QItemSelection()); } } void ModelPrivate::slotStorageModelRowsInserted(const QModelIndex &parent, int from, int to) { if (parent.isValid()) { return; // ugh... should never happen } Q_ASSERT(from <= to); int count = (to - from) + 1; mInvariantRowMapper->modelRowsInserted(from, count); // look if no current job is in the middle int jobCount = mViewItemJobs.count(); for (int idx = 0; idx < jobCount; idx++) { ViewItemJob *job = mViewItemJobs.at(idx); if (job->currentPass() != ViewItemJob::Pass1Fill) { // The job is a cleanup or in a later pass: the storage has been already accessed // and the messages created... no need to care anymore: the invariant row mapper will do the job. continue; } if (job->currentIndex() > job->endIndex()) { // The job finished the Pass1Fill but still waits for the pass indicator to be // changed. This is unlikely but still may happen if the job has been interrupted // and then a call to slotStorageModelRowsRemoved() caused it to be forcibly completed. continue; } // // The following cases are possible: // // from to // | | -> shift up job // from to // | | -> shift up job // from to // | | -> shift up job // from to // | | -> split job // from to // | | -> split job // from to // | | -> job unaffected // // // FOLDER // |-------------------------|---------|--------------| // 0 currentIndex endIndex count // +-- job --+ // if (from > job->endIndex()) { // The change is completely above the job, the job is not affected continue; } if (from > job->currentIndex()) { // and from <= job->endIndex() // The change starts in the middle of the job in a way that it must be split in two. // The first part is unaffected by the shift and ranges from job->currentIndex() to from - 1. // The second part ranges from "from" to job->endIndex() that are now shifted up by count steps. // First add a new job for the second part. auto newJob = new ViewItemJob(from + count, job->endIndex() + count, job->chunkTimeout(), job->idleInterval(), job->messageCheckCount()); Q_ASSERT(newJob->currentIndex() <= newJob->endIndex()); idx++; // we can skip this job in the loop, it's already ok jobCount++; // and our range increases by one. mViewItemJobs.insert(idx, newJob); // Then limit the original job to the first part job->setEndIndex(from - 1); Q_ASSERT(job->currentIndex() <= job->endIndex()); continue; } // The change starts below (or exactly on the beginning of) the job. // The job must be shifted up. job->setCurrentIndex(job->currentIndex() + count); job->setEndIndex(job->endIndex() + count); Q_ASSERT(job->currentIndex() <= job->endIndex()); } bool newJobNeeded = true; // Try to attach to an existing fill job, if any. // To enforce consistency we can attach only if the Fill job // is the last one in the list (might be eventually *also* the first, // and even being already processed but we must make sure that there // aren't jobs _after_ it). if (jobCount > 0) { ViewItemJob *job = mViewItemJobs.at(jobCount - 1); if (job->currentPass() == ViewItemJob::Pass1Fill) { if ( // The job ends just before the added rows (from == (job->endIndex() + 1)) &&// The job didn't reach the end of Pass1Fill yet (job->currentIndex() <= job->endIndex()) ) { // We can still attach this :) job->setEndIndex(to); Q_ASSERT(job->currentIndex() <= job->endIndex()); newJobNeeded = false; } } } if (newJobNeeded) { // FIXME: Should take timing options from aggregation here ? ViewItemJob *job = new ViewItemJob(from, to, 100, 50, 10); mViewItemJobs.append(job); } if (!mFillStepTimer.isActive()) { mFillStepTimer.start(mViewItemJobStepIdleInterval); } } void ModelPrivate::slotStorageModelRowsRemoved(const QModelIndex &parent, int from, int to) { // This is called when the underlying StorageModel emits the rowsRemoved signal. if (parent.isValid()) { return; // ugh... should never happen } // look if no current job is in the middle Q_ASSERT(from <= to); const int count = (to - from) + 1; int jobCount = mViewItemJobs.count(); if (mRootItem && from == 0 && count == mRootItem->childItemCount() && jobCount == 0) { clear(); return; } for (int idx = 0; idx < jobCount; idx++) { ViewItemJob *job = mViewItemJobs.at(idx); if (job->currentPass() != ViewItemJob::Pass1Fill) { // The job is a cleanup or in a later pass: the storage has been already accessed // and the messages created... no need to care: we will invalidate the messages in a while. continue; } if (job->currentIndex() > job->endIndex()) { // The job finished the Pass1Fill but still waits for the pass indicator to be // changed. This is unlikely but still may happen if the job has been interrupted // and then a call to slotStorageModelRowsRemoved() caused it to be forcibly completed. continue; } // // The following cases are possible: // // from to // | | -> shift down job // from to // | | -> shift down and crop job // from to // | | -> kill job // from to // | | -> split job, crop and shift // from to // | | -> crop job // from to // | | -> job unaffected // // // FOLDER // |-------------------------|---------|--------------| // 0 currentIndex endIndex count // +-- job --+ // if (from > job->endIndex()) { // The change is completely above the job, the job is not affected continue; } if (from > job->currentIndex()) { // and from <= job->endIndex() // The change starts in the middle of the job and ends in the middle or after the job. // The first part is unaffected by the shift and ranges from job->currentIndex() to from - 1 // We use the existing job for this. job->setEndIndex(from - 1); // stop before the first removed row Q_ASSERT(job->currentIndex() <= job->endIndex()); if (to < job->endIndex()) { // The change ends inside the job and a part of it can be completed. // We create a new job for the shifted remaining part. It would actually // range from to + 1 up to job->endIndex(), but we need to shift it down by count. // since count = ( to - from ) + 1 so from = to + 1 - count auto newJob = new ViewItemJob(from, job->endIndex() - count, job->chunkTimeout(), job->idleInterval(), job->messageCheckCount()); Q_ASSERT(newJob->currentIndex() < newJob->endIndex()); idx++; // we can skip this job in the loop, it's already ok jobCount++; // and our range increases by one. mViewItemJobs.insert(idx, newJob); } // else the change includes completely the end of the job and no other part of it can be completed. continue; } // The change starts below (or exactly on the beginning of) the job. ( from <= job->currentIndex() ) if (to >= job->endIndex()) { // The change completely covers the job: kill it // We don't delete the job since we want the other passes to be completed // This is because the Pass1Fill may have already filled mUnassignedMessageListForPass2 // and may have set mOldestItem and mNewestItem. We *COULD* clear the unassigned // message list with clearUnassignedMessageLists() but mOldestItem and mNewestItem // could be still dangling pointers. So we just move the current index of the job // after the end (so storage model scan terminates) and let it complete spontaneously. job->setCurrentIndex(job->endIndex() + 1); continue; } if (to >= job->currentIndex()) { // The change partially covers the job. Only a part of it can be completed // and it must be shifted down. It would actually // range from to + 1 up to job->endIndex(), but we need to shift it down by count. // since count = ( to - from ) + 1 so from = to + 1 - count job->setCurrentIndex(from); job->setEndIndex(job->endIndex() - count); Q_ASSERT(job->currentIndex() <= job->endIndex()); continue; } // The change is completely below the job: it must be shifted down. job->setCurrentIndex(job->currentIndex() - count); job->setEndIndex(job->endIndex() - count); } // This will invalidate the ModelInvariantIndex-es that have been removed and return // them all in a nice list that we can feed to a view removal job. auto invalidatedIndexes = mInvariantRowMapper->modelRowsRemoved(from, count); if (invalidatedIndexes) { // Try to attach to an existing cleanup job, if any. // To enforce consistency we can attach only if the Cleanup job // is the last one in the list (might be eventually *also* the first, // and even being already processed but we must make sure that there // aren't jobs _after_ it). if (jobCount > 0) { ViewItemJob *job = mViewItemJobs.at(jobCount - 1); if (job->currentPass() == ViewItemJob::Pass1Cleanup) { if ((job->currentIndex() <= job->endIndex()) && job->invariantIndexList()) { //qCDebug(MESSAGELIST_LOG) << "Appending " << invalidatedIndexes->count() << " invalidated indexes to existing cleanup job"; // We can still attach this :) *(job->invariantIndexList()) += *invalidatedIndexes; job->setEndIndex(job->endIndex() + invalidatedIndexes->count()); delete invalidatedIndexes; invalidatedIndexes = nullptr; } } } if (invalidatedIndexes) { // Didn't append to any existing cleanup job.. create a new one //qCDebug(MESSAGELIST_LOG) << "Creating new cleanup job for " << invalidatedIndexes->count() << " invalidated indexes"; // FIXME: Should take timing options from aggregation here ? auto job = new ViewItemJob(ViewItemJob::Pass1Cleanup, invalidatedIndexes, 100, 50, 10); mViewItemJobs.append(job); } if (!mFillStepTimer.isActive()) { mFillStepTimer.start(mViewItemJobStepIdleInterval); } } } void ModelPrivate::slotStorageModelLayoutChanged() { qCDebug(MESSAGELIST_LOG) << "Storage model layout changed"; // need to reset everything... q->setStorageModel(mStorageModel); qCDebug(MESSAGELIST_LOG) << "Storage model layout changed done"; } void ModelPrivate::slotStorageModelDataChanged(const QModelIndex &fromIndex, const QModelIndex &toIndex) { Q_ASSERT(mStorageModel); // must exist (and be the sender of the signal connected to this slot) int from = fromIndex.row(); int to = toIndex.row(); Q_ASSERT(from <= to); int count = (to - from) + 1; int jobCount = mViewItemJobs.count(); // This will find out the ModelInvariantIndex-es that need an update and will return // them all in a nice list that we can feed to a view removal job. auto indexesThatNeedUpdate = mInvariantRowMapper->modelIndexRowRangeToModelInvariantIndexList(from, count); if (indexesThatNeedUpdate) { // Try to attach to an existing update job, if any. // To enforce consistency we can attach only if the Update job // is the last one in the list (might be eventually *also* the first, // and even being already processed but we must make sure that there // aren't jobs _after_ it). if (jobCount > 0) { ViewItemJob *job = mViewItemJobs.at(jobCount - 1); if (job->currentPass() == ViewItemJob::Pass1Update) { if ((job->currentIndex() <= job->endIndex()) && job->invariantIndexList()) { // We can still attach this :) *(job->invariantIndexList()) += *indexesThatNeedUpdate; job->setEndIndex(job->endIndex() + indexesThatNeedUpdate->count()); delete indexesThatNeedUpdate; indexesThatNeedUpdate = nullptr; } } } if (indexesThatNeedUpdate) { // Didn't append to any existing update job.. create a new one // FIXME: Should take timing options from aggregation here ? auto job = new ViewItemJob(ViewItemJob::Pass1Update, indexesThatNeedUpdate, 100, 50, 10); mViewItemJobs.append(job); } if (!mFillStepTimer.isActive()) { mFillStepTimer.start(mViewItemJobStepIdleInterval); } } } void ModelPrivate::slotStorageModelHeaderDataChanged(Qt::Orientation, int, int) { if (mStorageModelContainsOutboundMessages != mStorageModel->containsOutboundMessages()) { mStorageModelContainsOutboundMessages = mStorageModel->containsOutboundMessages(); Q_EMIT q->headerDataChanged(Qt::Horizontal, 0, q->columnCount()); } } Qt::ItemFlags Model::flags(const QModelIndex &index) const { if (!index.isValid()) { return Qt::NoItemFlags; } Q_ASSERT(d->mModelForItemFunctions); // UI must be connected if a valid index was queried Item *it = static_cast< Item * >(index.internalPointer()); Q_ASSERT(it); if (it->type() == Item::GroupHeader) { return Qt::ItemIsEnabled; } Q_ASSERT(it->type() == Item::Message); if (!static_cast< MessageItem * >(it)->isValid()) { return Qt::NoItemFlags; // not enabled, not selectable } if (static_cast< MessageItem * >(it)->aboutToBeRemoved()) { return Qt::NoItemFlags; // not enabled, not selectable } if (static_cast< MessageItem * >(it)->status().isDeleted()) { return Qt::NoItemFlags; // not enabled, not selectable } return Qt::ItemIsEnabled | Qt::ItemIsSelectable; } QMimeData *MessageList::Core::Model::mimeData(const QModelIndexList &indexes) const { QVector< MessageItem * > msgs; for (const QModelIndex &idx : indexes) { if (idx.isValid()) { Item *item = static_cast(idx.internalPointer()); if (item->type() == MessageList::Core::Item::Message) { msgs << static_cast(idx.internalPointer()); } } } return storageModel()->mimeData(msgs); } Item *Model::rootItem() const { return d->mRootItem; } bool Model::isLoading() const { return d->mLoading; } MessageItem *Model::messageItemByStorageRow(int row) const { if (!d->mStorageModel) { return nullptr; } auto idx = d->mInvariantRowMapper->modelIndexRowToModelInvariantIndex(row); if (!idx) { return nullptr; } return static_cast< MessageItem * >(idx); } MessageItemSetReference Model::createPersistentSet(const QVector &items) { if (!d->mPersistentSetManager) { d->mPersistentSetManager = new MessageItemSetManager(); } MessageItemSetReference ref = d->mPersistentSetManager->createSet(); for (const auto mi : items) { d->mPersistentSetManager->addMessageItem(ref, mi); } return ref; } QList Model::persistentSetCurrentMessageItemList(MessageItemSetReference ref) { if (d->mPersistentSetManager) { return d->mPersistentSetManager->messageItems(ref); } return QList< MessageItem * >(); } void Model::deletePersistentSet(MessageItemSetReference ref) { if (!d->mPersistentSetManager) { return; } d->mPersistentSetManager->removeSet(ref); if (d->mPersistentSetManager->setCount() < 1) { delete d->mPersistentSetManager; d->mPersistentSetManager = nullptr; } } #include "moc_model.cpp" diff --git a/messagelist/src/core/model_p.h b/messagelist/src/core/model_p.h index 0e40b020..ed5c4878 100644 --- a/messagelist/src/core/model_p.h +++ b/messagelist/src/core/model_p.h @@ -1,463 +1,463 @@ /****************************************************************************** * * Copyright 2008 Szymon Tomasz Stefanek * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * *******************************************************************************/ #ifndef MESSAGELIST_CORE_MODEL_P_H #define MESSAGELIST_CORE_MODEL_P_H #include "model.h" #include "threadingcache.h" #include #include class QElapsedTimer; namespace MessageList { namespace Core { class ViewItemJob; class ModelInvariantRowMapper; class MessageItemSetManager; class ModelPrivate { public: explicit ModelPrivate(Model *owner) : q(owner) { } void fillView(); /** * This is called by MessageList::Manager once in a while. * It is a good place to check if the date has changed and * trigger a view reload. */ void checkIfDateChanged(); void viewItemJobStep(); /** * Attempt to find the threading parent for the specified message item. * Sets the message threading status to the appropriate value. * * This function performs In-Reply-To and References threading. */ MessageItem *findMessageParent(MessageItem *mi); /** * Attempt to find the threading parent for the specified message item. * Sets the message threading status to the appropriate value. * * This function performs Subject based threading. */ MessageItem *guessMessageParent(MessageItem *mi); enum AttachOptions { SkipCacheUpdate = 0, StoreInCache = 1 }; void attachMessageToParent(Item *pParent, MessageItem *mi, AttachOptions attachOptions = StoreInCache); void messageDetachedUpdateParentProperties(Item *oldParent, MessageItem *mi); void attachMessageToGroupHeader(MessageItem *mi); void attachGroup(GroupHeaderItem *ghi); enum ViewItemJobResult { ViewItemJobCompleted, ViewItemJobInterrupted }; ViewItemJobResult viewItemJobStepInternal(); ViewItemJobResult viewItemJobStepInternalForJob(ViewItemJob *job, const QElapsedTimer &elapsedTimer); // FIXME: Those look like they should be made virtual in some job class! -> Refactor - ViewItemJobResult viewItemJobStepInternalForJobPass1Fill(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass1Cleanup(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass1Update(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass2(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass3(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass4(ViewItemJob *job, const QElapsedTimer &elapsedTimer); - ViewItemJobResult viewItemJobStepInternalForJobPass5(ViewItemJob *job, const QElapsedTimer &elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass1Fill(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass1Cleanup(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass1Update(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass2(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass3(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass4(ViewItemJob *job, QElapsedTimer elapsedTimer); + ViewItemJobResult viewItemJobStepInternalForJobPass5(ViewItemJob *job, QElapsedTimer elapsedTimer); void clearJobList(); void clearUnassignedMessageLists(); void clearOrphanChildrenHash(); void clearThreadingCacheReferencesIdMD5ToMessageItem(); void clearThreadingCacheMessageSubjectMD5ToMessageItem(); void addMessageToReferencesBasedThreadingCache(MessageItem *mi); void removeMessageFromReferencesBasedThreadingCache(MessageItem *mi); void addMessageToSubjectBasedThreadingCache(MessageItem *mi); void removeMessageFromSubjectBasedThreadingCache(MessageItem *mi); void clear(); /** * Sync the expanded state of the subtree with the specified root. * This will cause the items that are marked with Item::ExpandNeeded to be * expanded also in the view. For optimization purposes the specified root * is assumed to be marked as Item::ExpandNeeded so be sure to check it * before calling this function. */ void syncExpandedStateOfSubtree(Item *root); /** * Save the expanded state of the subtree with the specified root. * The state will be saved in the initialExpandStatus() variable. * For optimization purposes the specified root is assumed to be expanded * and viewable. */ void saveExpandedStateOfSubtree(Item *root); #ifdef KDEPIM_FOLDEROPEN_PROFILE // This prints out all the stats we collected void printStatistics(); #endif enum PropertyChanges { DateChanged = 1, MaxDateChanged = (1 << 1), ActionItemStatusChanged = (1 << 2), UnreadStatusChanged = (1 << 3), ImportantStatusChanged = (1 << 4), AttachmentStatusChanged = (1 << 5) }; /** * Handle the specified property changes in item. Depending on the item * position inside the parent and the types of item and parent the item * might need re-grouping or re-sorting. This function takes care of that. * It is meant to be called from somewhere inside viewItemJobStepInternal() * as it postpones group updates to Pass5. * * parent and item must not be null. propertyChangeMask should not be zero. * * Return true if parent might be affected by the item property changes * and false otherwise. */ bool handleItemPropertyChanges(int propertyChangeMask, Item *parent, Item *item); /** * This one checks if the parent of item requires an update due to the * properties of item (that might have been changed or the item might * have been simply added to the parent). The properties * are propagated up to the root item. As optimization we ASSUME that * the item->parent() exists (is non 0) and is NOT the root item. * Be sure to check it before calling this function (it will assert in debug mode anyway). * ... ah... and don't be afraid: this is NOT (directly) recursive :) */ void propagateItemPropertiesToParent(Item *item); /** * Recursively applies the current filter to the tree originating at the specified item. * The item is hidden if the filter doesn't match (the item or any children of it) * and this function returns false. * If the filter matches somewhere in the subtree then the item isn't hidden * and this function returns true. * * Assumes that the specified item is viewable. */ bool applyFilterToSubtree(Item *item, const QModelIndex &parentIndex); // Slots connected to the underlying StorageModel. void slotStorageModelRowsInserted(const QModelIndex &parent, int from, int to); void slotStorageModelRowsRemoved(const QModelIndex &parent, int from, int to); void slotStorageModelDataChanged(const QModelIndex &fromIndex, const QModelIndex &toIndex); void slotStorageModelHeaderDataChanged(Qt::Orientation orientation, int first, int last); void slotStorageModelLayoutChanged(); void slotApplyFilter(); Model *const q; /** counter to avoid infinite recursions in the setStorageModel() function */ int mRecursionCounterForReset; /** * The currently set storage model: shallow pointer. */ StorageModel *mStorageModel; /** * The currently set aggregation mode: shallow pointer set by Widget */ const Aggregation *mAggregation; /** * The currently used theme: shallow pointer */ const Theme *mTheme; /** * The currently used sort order. Pointer not owned by us, but by the Widget. */ const SortOrder *mSortOrder; /** * The filter to apply on messages. Shallow. Never 0. */ const Filter *mFilter; /** * The timer involved in breaking the "fill" operation in steps */ QTimer mFillStepTimer; /** * Group Key (usually the label) -> GroupHeaderItem, used to quickly find groups, pointers are shallow copies */ QHash< QString, GroupHeaderItem * > mGroupHeaderItemHash; /** * Threading cache. * MessageIdMD5 -> MessageItem, pointers are shallow copies */ QHash< QByteArray, MessageItem * > mThreadingCacheMessageIdMD5ToMessageItem; /** * Threading cache. * MessageInReplyToIdMD5 -> MessageItem, pointers are shallow copies */ QMultiHash< QByteArray, MessageItem * > mThreadingCacheMessageInReplyToIdMD5ToMessageItem; /** * Threading cache. * ReferencesIdMD5 -> MessageItem, pointers are shallow copies */ QHash< QByteArray, QList< MessageItem * > * > mThreadingCacheMessageReferencesIdMD5ToMessageItem; /** * Threading cache. * SubjectMD5 -> MessageItem, pointers are shallow copies */ QHash< QByteArray, QList< MessageItem * > * > mThreadingCacheMessageSubjectMD5ToMessageItem; /** * List of group headers that either need to be re-sorted or must be removed because empty */ QHash< GroupHeaderItem *, GroupHeaderItem * > mGroupHeadersThatNeedUpdate; /** * List of unassigned messages, used to handle threading in two passes, pointers are owned! */ QList< MessageItem * > mUnassignedMessageListForPass2; /** * List of unassigned messages, used to handle threading in two passes, pointers are owned! */ QList< MessageItem * > mUnassignedMessageListForPass3; /** * List of unassigned messages, used to handle threading in two passes, pointers are owned! */ QList< MessageItem * > mUnassignedMessageListForPass4; /** * Hash of orphan children used in Pass1Cleanup. */ QHash< MessageItem *, MessageItem * > mOrphanChildrenHash; /** * Pending fill view jobs, pointers are owned */ QList< ViewItemJob * > mViewItemJobs; /** * The today's date. Set when the StorageModel is set and thus grouping is performed. * This is used to put the today's messages in the "Today" group, for instance. */ QDate mTodayDate; /** * Owned invisible root item, useful to implement algorithms that not need * to handle the special case of parentless items. This is never 0. */ Item *mRootItem; /** * The view we're attached to. Shallow pointer (the View owns us). */ View *mView; /** * The time at the current ViewItemJob step started. Used to compute the time we * spent inside this step and eventually jump out on timeout. */ time_t mViewItemJobStepStartTime; /** * The timeout for a single ViewItemJob step */ int mViewItemJobStepChunkTimeout; /** * The idle time between two ViewItemJob steps */ int mViewItemJobStepIdleInterval; /** * The number of messages we process at once in a ViewItemJob step without * checking the timeouts above. */ int mViewItemJobStepMessageCheckCount; /** * Our mighty ModelInvariantRowMapper: used to workaround an * issue related to the Model/View architecture. * * \sa ModelInvariantRowMapper */ ModelInvariantRowMapper *mInvariantRowMapper; /** * The label for the "Today" group item, cached, so we don't translate it multiple times. */ QString mCachedTodayLabel; /** * The label for the "Yesterday" group item, cached, so we don't translate it multiple times. */ QString mCachedYesterdayLabel; /** * The label for the "Unknown" group item, cached, so we don't translate it multiple times. */ QString mCachedUnknownLabel; /** * The label for the "Last Week" group item, cached, so we don't translate it multiple times. */ QString mCachedLastWeekLabel; /** * The label for the "Two Weeks Ago" group item, cached, so we don't translate it multiple times. */ QString mCachedTwoWeeksAgoLabel; /** * The label for the "Three Weeks Ago" group item, cached, so we don't translate it multiple times. */ QString mCachedThreeWeeksAgoLabel; /** * The label for the "Four Weeks Ago" group item, cached, so we don't translate it multiple times. */ QString mCachedFourWeeksAgoLabel; /** * The label for the "Five Weeks Ago" group item, cached, so we don't translate it multiple times. */ QString mCachedFiveWeeksAgoLabel; /** * Cached bits that we use for fast status checks */ qint32 mCachedWatchedOrIgnoredStatusBits; /** * The labels for week days names group items, cached, so we don't query QLocale multiple times. */ QMap mCachedDayNameLabel; /* * The labels for month names group items, cached, so we don't query QLocale multiple times. */ QMap mCachedMonthNameLabel; /** * Flag signaling a possibly long job batch. This is checked by other * classes and used to display some kind of "please wait" feedback to the user. */ bool mInLengthyJobBatch; /** * We need to save the current item before each job step. This is because * our job may cause items to be reparented (thus removed and readded with the current Qt API) * and QTreeView will loose the current setting. We also use this to force the current * to a specific item after a cleanup job. */ Item *mCurrentItemToRestoreAfterViewItemJobStep; /** * Set to true in the first large loading job. * Reset to false when the job finishes. * * Please note that this is NOT set for later jobs: only for the first (possibly huge) one. */ bool mLoading; /** * Pre-selection is the action of automatically selecting a message just after the folder * has finished loading. We may want to select the message that was selected the last * time this folder has been open, or we may want to select the first unread message. * We also may want to do no pre-selection at all (for example, when the user * starts navigating the view before the pre-selection could actually be made * and pre-selecting would confuse him). This member holds the option. * * See also setStorageModel() and abortMessagePreSelection() */ PreSelectionMode mPreSelectionMode; // Oldest and newest item while loading the model // Not valid afterwards anymore. Used for pre-selection of the newest/oldest message MessageItem *mOldestItem; MessageItem *mNewestItem; /** * The id of the preselected ;essage is "translated" to a message pointer when it's fetched * from the storage. This message is then selected when it becomes viewable * (so at the end of the job). 0 if we have no message to select. * * See also setStorageModel() and abortMessagePreSelection() */ MessageItem *mLastSelectedMessageInFolder; /** * The "persistent message item sets" are (guess what?) sets of messages * that can be referenced globally via a persistent id. The MessageItemSetManager * and this class keep the persistent sets coherent: messages that are deleted * are automatically removed from all the sets. * * Users of this class typically create persistent sets when they start * an asynchronous job and they query them back on the way or when the job is terminated. * * So mPersistentSetManager is in fact the manager for the outstanding "user" jobs. * 0 if no jobs are pending (so there are no persistent sets at the moment). */ MessageItemSetManager *mPersistentSetManager; /** * This pointer is passed to the Item functions that insert children. * When we work with disconnected UI this pointer becomes 0. */ Model *mModelForItemFunctions; /** * The cached result of StorageModel::containsOutboundMessages(). * We access this property at each incoming message and StorageModel::containsOutboundMessages() is * virtual (so it's always an indirect function call). Caching makes sense. */ bool mStorageModelContainsOutboundMessages; /** * Vector of signal-slot connections between StorageModel and us */ QVector mStorageModelConnections; /** * Caches child - parent relation based on Akonadi ID and persists the cache * in a file for each Collection. This allows for very fast reconstruction of * threading. */ ThreadingCache mThreadingCache; }; } // namespace Core } // namespace MessageList #endif //!__MESSAGELIST_CORE_MODEL_P_H diff --git a/messageviewer/src/dkim-verify/autotests/dkimauthenticationstatusinfotest.cpp b/messageviewer/src/dkim-verify/autotests/dkimauthenticationstatusinfotest.cpp index 24dfa936..e57ac8c3 100644 --- a/messageviewer/src/dkim-verify/autotests/dkimauthenticationstatusinfotest.cpp +++ b/messageviewer/src/dkim-verify/autotests/dkimauthenticationstatusinfotest.cpp @@ -1,262 +1,262 @@ /* Copyright (C) 2018-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimauthenticationstatusinfotest.h" #include "dkim-verify/dkimauthenticationstatusinfo.h" #include QTEST_GUILESS_MAIN(DKIMAuthenticationStatusInfoTest) DKIMAuthenticationStatusInfoTest::DKIMAuthenticationStatusInfoTest(QObject *parent) : QObject(parent) { } void DKIMAuthenticationStatusInfoTest::shouldHaveDefaultValue() { MessageViewer::DKIMAuthenticationStatusInfo info; QVERIFY(info.authservId().isEmpty()); QCOMPARE(info.authVersion(), -1); QVERIFY(info.reasonSpec().isEmpty()); QVERIFY(info.listAuthStatusInfo().isEmpty()); } void DKIMAuthenticationStatusInfoTest::shouldParseKey() { QFETCH(QString, key); QFETCH(MessageViewer::DKIMAuthenticationStatusInfo, result); QFETCH(bool, relaxingParsing); QFETCH(bool, success); MessageViewer::DKIMAuthenticationStatusInfo info; const bool val = info.parseAuthenticationStatus(key, relaxingParsing); QCOMPARE(val, success); const bool compareResult = result == info; if (!compareResult) { qDebug() << "parse info: " << info; qDebug() << "expected: " << result; } QVERIFY(compareResult); } void DKIMAuthenticationStatusInfoTest::shouldParseKey_data() { QTest::addColumn("key"); QTest::addColumn("result"); QTest::addColumn("relaxingParsing"); QTest::addColumn("success"); QTest::addRow("empty") << QString() << MessageViewer::DKIMAuthenticationStatusInfo() << false << false; { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("in68.mail.ovh.net")); QVector lst; MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property; property.method = QStringLiteral("dkim"); property.result = QStringLiteral("pass"); property.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("d"); - prop.value = QLatin1String("kde.org"); + prop.type = QStringLiteral("d"); + prop.value = QStringLiteral("kde.org"); property.header.append(prop); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@kde.org"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@kde.org"); property.header.append(prop); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("b"); - prop.value = QLatin1String("\"GMG2ucPx\""); + prop.type = QStringLiteral("b"); + prop.value = QStringLiteral("\"GMG2ucPx\""); property.header.append(prop); } lst.append(property); MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property2; property2.method = QStringLiteral("dkim"); property2.result = QStringLiteral("pass"); property2.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("d"); - prop.value = QLatin1String("kde.org"); + prop.type = QStringLiteral("d"); + prop.value = QStringLiteral("kde.org"); property2.header.append(prop); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@kde.org"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@kde.org"); property2.header.append(prop); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("b"); - prop.value = QLatin1String("\"I3t3p7Up\""); + prop.type = QStringLiteral("b"); + prop.value = QStringLiteral("\"I3t3p7Up\""); property2.header.append(prop); } lst.append(property2); MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property3; property3.method = QStringLiteral("dkim-atps"); property3.result = QStringLiteral("neutral"); property3.methodVersion = 1; lst.append(property3); info.setListAuthStatusInfo(lst); QTest::addRow("test1") << QStringLiteral("in68.mail.ovh.net; dkim=pass (2048-bit key; unprotected) header.d=kde.org header.i=@kde.org header.b=\"GMG2ucPx\"; dkim=pass (2048-bit key; unprotected) header.d=kde.org header.i=@kde.org header.b=\"I3t3p7Up\"; dkim-atps=neutral") << info << false << true; } { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("example.org")); QTest::addRow("none") << QStringLiteral("example.org 1; none;") << info << false << false; } { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("example.org")); QTest::addRow("none2") << QStringLiteral("example.org 1; none") << info << false << false; } { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("example.com")); QVector lst; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property; property.method = QStringLiteral("dkim"); property.result = QStringLiteral("pass"); property.reason = QStringLiteral("good signature"); property.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@mail-router.example.net"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@mail-router.example.net"); property.header.append(prop); } lst.append(property); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property2; property2.method = QStringLiteral("dkim"); property2.result = QStringLiteral("fail"); property2.reason = QStringLiteral("bad signature"); property2.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@newyork.example.com"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@newyork.example.com"); property2.header.append(prop); } lst.append(property2); } info.setListAuthStatusInfo(lst); QTest::addRow("reason") << QStringLiteral("example.com; dkim=pass reason=\"good signature\" header.i=@mail-router.example.net; dkim=fail reason=\"bad signature\" header.i=@newyork.example.com;") << info << false << true; } //It will failed. Fix it { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("example.com")); QVector lst; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property; property.method = QStringLiteral("dkim"); property.result = QStringLiteral("pass"); property.reason = QStringLiteral("good signature"); property.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@mail-router.example.net"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@mail-router.example.net"); property.header.append(prop); } lst.append(property); } { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property2; property2.method = QStringLiteral("dkim"); property2.result = QStringLiteral("fail"); property2.reason = QStringLiteral("bad signature"); property2.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("i"); - prop.value = QLatin1String("@newyork.example.com"); + prop.type = QStringLiteral("i"); + prop.value = QStringLiteral("@newyork.example.com"); property2.header.append(prop); } lst.append(property2); } info.setListAuthStatusInfo(lst); QTest::addRow("reason2") << QStringLiteral("example.com; dkim=pass reason=\"good signature\" header.i=@mail-router.example.net; dkim=fail reason=\"bad signature\" header.i=@newyork.example.com") << info << true << true; } { MessageViewer::DKIMAuthenticationStatusInfo info; info.setAuthVersion(1); info.setAuthservId(QStringLiteral("letterbox.kde.org")); QVector lst; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo property; property.method = QStringLiteral("dmarc"); property.result = QStringLiteral("pass"); property.methodVersion = 1; { MessageViewer::DKIMAuthenticationStatusInfo::AuthStatusInfo::Property prop; - prop.type = QLatin1String("from"); - prop.value = QLatin1String("gmail.com"); + prop.type = QStringLiteral("from"); + prop.value = QStringLiteral("gmail.com"); property.header.append(prop); } lst.append(property); } info.setListAuthStatusInfo(lst); QTest::addRow("gmails") << QStringLiteral("letterbox.kde.org; dmarc=pass (p=none dis=none) header.from=gmail.com\r\n") << info << true << true; } } diff --git a/messageviewer/src/dkim-verify/dkimauthenticationstatusinfoutil.cpp b/messageviewer/src/dkim-verify/dkimauthenticationstatusinfoutil.cpp index 1f496df3..0ef2c2d7 100644 --- a/messageviewer/src/dkim-verify/dkimauthenticationstatusinfoutil.cpp +++ b/messageviewer/src/dkim-verify/dkimauthenticationstatusinfoutil.cpp @@ -1,209 +1,209 @@ /* Copyright (C) 2019-2020 Laurent Montel Code based on ARHParser.jsm from dkim_verifier (Copyright (c) Philippe Lieser) (This software is licensed under the terms of the MIT License.) This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimauthenticationstatusinfoutil.h" /* // domain-name as specified in Section 3.5 of RFC 6376 [DKIM]. let domain_name_p = "(?:" + sub_domain_p + "(?:\\." + sub_domain_p + ")+)"; */ QString MessageViewer::DKIMAuthenticationStatusInfoUtil::wsp_p() { // WSP as specified in Appendix B.1 of RFC 5234 return QStringLiteral("[ \t]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::vchar_p() { // VCHAR as specified in Appendix B.1 of RFC 5234 return QStringLiteral("[!-~]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::letDig_p() { // Let-dig as specified in Section 4.1.2 of RFC 5321 [SMTP]. return QStringLiteral("[A-Za-z0-9]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::ldhStr_p() { // Ldh-str as specified in Section 4.1.2 of RFC 5321 [SMTP]. return QStringLiteral("(?:[A-Za-z0-9-]*%1)").arg(DKIMAuthenticationStatusInfoUtil::letDig_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::keyword_p() { // "Keyword" as specified in Section 4.1.2 of RFC 5321 [SMTP]. return DKIMAuthenticationStatusInfoUtil::ldhStr_p(); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::subDomain_p() { // sub-domain as specified in Section 4.1.2 of RFC 5321 [SMTP]. - return QStringLiteral("(?:%1%2?)").arg(DKIMAuthenticationStatusInfoUtil::letDig_p()).arg(DKIMAuthenticationStatusInfoUtil::ldhStr_p()); + return QStringLiteral("(?:%1%2?)").arg(DKIMAuthenticationStatusInfoUtil::letDig_p(), DKIMAuthenticationStatusInfoUtil::ldhStr_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::obsFws_p() { // obs-FWS as specified in Section 4.2 of RFC 5322 return QStringLiteral("(?:%1+(?:\r\n%1+)*)").arg(DKIMAuthenticationStatusInfoUtil::wsp_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::quotedPair_p() { // quoted-pair as specified in Section 3.2.1 of RFC 5322 // Note: obs-qp is not included, so this pattern matches less then specified! - return QStringLiteral("(?:\\\\(?:%1|%2))").arg(vchar_p()).arg(wsp_p()); + return QStringLiteral("(?:\\\\(?:%1|%2))").arg(vchar_p(), wsp_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::fws_p() { // FWS as specified in Section 3.2.2 of RFC 5322 - return QStringLiteral("(?:(?:(?:%1*\r\n)?%1+)|%2)").arg(wsp_p()).arg(obsFws_p()); + return QStringLiteral("(?:(?:(?:%1*\r\n)?%1+)|%2)").arg(wsp_p(), obsFws_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::fws_op() { return QStringLiteral("%1?").arg(fws_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::ctext_p() { // ctext as specified in Section 3.2.2 of RFC 5322 return QStringLiteral("[!-'*-[\\]-~]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::ccontent_p() { // ccontent as specified in Section 3.2.2 of RFC 5322 // Note: comment is not included, so this pattern matches less then specified! - return QStringLiteral("(?:%1|%2)").arg(ctext_p()).arg(quotedPair_p()); + return QStringLiteral("(?:%1|%2)").arg(ctext_p(), quotedPair_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::comment_p() { // comment as specified in Section 3.2.2 of RFC 5322 - return QStringLiteral("\\((?:%1%2)*%1\\)").arg(fws_op()).arg(ccontent_p()); + return QStringLiteral("\\((?:%1%2)*%1\\)").arg(fws_op(), ccontent_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::cfws_p() { // CFWS as specified in Section 3.2.2 of RFC 5322 [MAIL] - return QStringLiteral("(?:(?:(?:%1%2)+%1)|%3)").arg(fws_op()).arg(comment_p()).arg(fws_p()); + return QStringLiteral("(?:(?:(?:%1%2)+%1)|%3)").arg(fws_op(), comment_p(), fws_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::cfws_op() { return QStringLiteral("%1?").arg(cfws_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::atext() { // atext as specified in Section 3.2.3 of RFC 5322 return QStringLiteral("[!#-'*-+/-9=?A-Z^-~-]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::dotAtomText_p() { // dot-atom-text as specified in Section 3.2.3 of RFC 5322 return QStringLiteral("(?:%1+(?:\\.%1+)*)").arg(atext()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::dotAtom_p() { // dot-atom as specified in Section 3.2.3 of RFC 5322 // dot-atom = [CFWS] dot-atom-text [CFWS] - return QStringLiteral("(?:%1%2%1)").arg(cfws_op()).arg(dotAtomText_p()); + return QStringLiteral("(?:%1%2%1)").arg(cfws_op(), dotAtomText_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::qtext_p() { // qtext as specified in Section 3.2.4 of RFC 5322 // Note: obs-qtext is not included, so this pattern matches less then specified! return QStringLiteral("[!#-[\\]-~]"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::qcontent_p() { // qcontent as specified in Section 3.2.4 of RFC 5322 return QStringLiteral("(?:%1|%2)").arg(qtext_p()).arg(quotedPair_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::quotedString_p() { // quoted-string as specified in Section 3.2.4 of RFC 5322 - return QStringLiteral("(?:%1\"(?:%2%3)*%2\"%1)").arg(cfws_op()).arg(fws_op()).arg(qcontent_p()); + return QStringLiteral("(?:%1\"(?:%2%3)*%2\"%1)").arg(cfws_op(), fws_op(), qcontent_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::quotedString_cp() { - return QStringLiteral("(?:%1\"((?:%2%3)*)%2\"%1)").arg(cfws_op()).arg(fws_op()).arg(qcontent_p()); + return QStringLiteral("(?:%1\"((?:%2%3)*)%2\"%1)").arg(cfws_op(), fws_op(), qcontent_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::localPart_p() { // local-part as specified in Section 3.4.1 of RFC 5322 // Note: obs-local-part is not included, so this pattern matches less then specified! - return QStringLiteral("(?:%1|%2))").arg(dotAtom_p()).arg(quotedString_p()); + return QStringLiteral("(?:%1|%2))").arg(dotAtom_p(), quotedString_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::token_p() { // token as specified in Section 5.1 of RFC 2045. return QStringLiteral("[^ \\x00-\\x1F\\x7F()<>@,;:\\\\\"/[\\]?=]+"); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::value_p() { // "value" as specified in Section 5.1 of RFC 2045. - return QStringLiteral("(?:%1|%2)").arg(token_p()).arg(quotedString_p()); + return QStringLiteral("(?:%1|%2)").arg(token_p(), quotedString_p()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::value_cp() { - return QStringLiteral("(?:(%1)|%2)").arg(token_p()).arg(quotedString_cp()); + return QStringLiteral("(?:(%1)|%2)").arg(token_p(), quotedString_cp()); } QString MessageViewer::DKIMAuthenticationStatusInfoUtil::domainName_p() { // domain-name as specified in Section 3.5 of RFC 6376 [DKIM]. return QStringLiteral("(?:%1(?:\\.%1)+)").arg(subDomain_p()); } // Tries to matches a pattern to the beginning of str. // Adds CFWS_op to the beginning of pattern. // pattern must be followed by string end, ";" or CFWS_p. // If match is found, removes it from str. QString MessageViewer::DKIMAuthenticationStatusInfoUtil::regexMatchO(const QString ®ularExpressionStr) { const QString regexp = (QLatin1Char('^') + DKIMAuthenticationStatusInfoUtil::cfws_op() + QStringLiteral("(?:") + regularExpressionStr + QLatin1Char(')') + QStringLiteral("(?:(?:") + DKIMAuthenticationStatusInfoUtil::cfws_op() + QStringLiteral("\r\n$)|(?=;)|(?=") + DKIMAuthenticationStatusInfoUtil::cfws_p() + QStringLiteral("))")); return regexp; } diff --git a/messageviewer/src/dkim-verify/dkimchecksignaturejob.cpp b/messageviewer/src/dkim-verify/dkimchecksignaturejob.cpp index d21dc965..b4467729 100644 --- a/messageviewer/src/dkim-verify/dkimchecksignaturejob.cpp +++ b/messageviewer/src/dkim-verify/dkimchecksignaturejob.cpp @@ -1,723 +1,723 @@ /* Copyright (C) 2018-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimchecksignaturejob.h" #include "dkimdownloadkeyjob.h" #include "dkimmanagerkey.h" #include "dkiminfo.h" #include "dkimutil.h" #include "dkimkeyrecord.h" #include "messageviewer_dkimcheckerdebug.h" #include #include #include #include #include //see https://tools.ietf.org/html/rfc6376 //#define DEBUG_SIGNATURE_DKIM 1 using namespace MessageViewer; DKIMCheckSignatureJob::DKIMCheckSignatureJob(QObject *parent) : QObject(parent) { } DKIMCheckSignatureJob::~DKIMCheckSignatureJob() { } MessageViewer::DKIMCheckSignatureJob::CheckSignatureResult DKIMCheckSignatureJob::createCheckResult() { MessageViewer::DKIMCheckSignatureJob::CheckSignatureResult result; result.error = mError; result.warning = mWarning; result.status = mStatus; result.sdid = mDkimInfo.domain(); result.auid = mDkimInfo.agentOrUserIdentifier(); result.fromEmail = mFromEmail; result.listSignatureAuthenticationResult = mCheckSignatureAuthenticationResult; return result; } QString DKIMCheckSignatureJob::bodyCanonizationResult() const { return mBodyCanonizationResult; } QString DKIMCheckSignatureJob::headerCanonizationResult() const { return mHeaderCanonizationResult; } void DKIMCheckSignatureJob::start() { if (!mMessage) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Item has not a message"; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } if (auto hrd = mMessage->headerByType("DKIM-Signature")) { mDkimValue = hrd->asUnicodeString(); } //Store mFromEmail before looking at mDkimValue value. Otherwise we can return a from empty if (auto hrd = mMessage->from(false)) { mFromEmail = KEmailAddress::extractEmailAddress(hrd->asUnicodeString()); } if (mDkimValue.isEmpty()) { mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::EmailNotSigned; Q_EMIT result(createCheckResult()); deleteLater(); return; } qCDebug(MESSAGEVIEWER_DKIMCHECKER_LOG) << "mFromEmail " << mFromEmail; if (!mDkimInfo.parseDKIM(mDkimValue)) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Impossible to parse header" << mDkimValue; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } const MessageViewer::DKIMCheckSignatureJob::DKIMStatus status = checkSignature(mDkimInfo); if (status != MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Valid) { mStatus = status; Q_EMIT result(createCheckResult()); deleteLater(); return; } //ComputeBodyHash now. switch (mDkimInfo.bodyCanonization()) { case MessageViewer::DKIMInfo::CanonicalizationType::Unknown: mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InvalidBodyCanonicalization; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; case MessageViewer::DKIMInfo::CanonicalizationType::Simple: mBodyCanonizationResult = bodyCanonizationSimple(); break; case MessageViewer::DKIMInfo::CanonicalizationType::Relaxed: mBodyCanonizationResult = bodyCanonizationRelaxed(); break; } //qDebug() << " bodyCanonizationResult "<< mBodyCanonizationResult << " algorithm " << mDkimInfo.hashingAlgorithm() << mDkimInfo.bodyHash(); if (mDkimInfo.bodyLengthCount() != -1) { //Verify it. if (mDkimInfo.bodyLengthCount() > mBodyCanonizationResult.length()) { // length tag exceeds body size qCDebug(MESSAGEVIEWER_DKIMCHECKER_LOG) << " mDkimInfo.bodyLengthCount() " << mDkimInfo.bodyLengthCount() << " mBodyCanonizationResult.length() " << mBodyCanonizationResult.length(); mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::SignatureTooLarge; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } else if (mDkimInfo.bodyLengthCount() < mBodyCanonizationResult.length()) { mWarning = MessageViewer::DKIMCheckSignatureJob::DKIMWarning::SignatureTooSmall; } // truncated body to the length specified in the "l=" tag mBodyCanonizationResult = mBodyCanonizationResult.left(mDkimInfo.bodyLengthCount()); } if (mBodyCanonizationResult.startsWith(QLatin1String("\r\n"))) { //Remove it from start mBodyCanonizationResult = mBodyCanonizationResult.right(mBodyCanonizationResult.length() -2); } #ifdef DEBUG_SIGNATURE_DKIM QFile caFile(QStringLiteral("/tmp/bodycanon-kmail.txt")); caFile.open(QIODevice::WriteOnly | QIODevice::Text); QTextStream outStream(&caFile); outStream << mBodyCanonizationResult; caFile.close(); #endif QByteArray resultHash; switch (mDkimInfo.hashingAlgorithm()) { case DKIMInfo::HashingAlgorithmType::Sha1: resultHash = MessageViewer::DKIMUtil::generateHash(mBodyCanonizationResult.toLatin1(), QCryptographicHash::Sha1); break; case DKIMInfo::HashingAlgorithmType::Sha256: resultHash = MessageViewer::DKIMUtil::generateHash(mBodyCanonizationResult.toLatin1(), QCryptographicHash::Sha256); break; case DKIMInfo::HashingAlgorithmType::Any: case DKIMInfo::HashingAlgorithmType::Unknown: mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InsupportedHashAlgorithm; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } // compare body hash qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "resultHash " << resultHash << "mDkimInfo.bodyHash()" << mDkimInfo.bodyHash(); if (resultHash != mDkimInfo.bodyHash().toLatin1()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << " Corrupted body hash"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::CorruptedBodyHash; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } if (mDkimInfo.headerCanonization() == MessageViewer::DKIMInfo::CanonicalizationType::Unknown) { mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InvalidHeaderCanonicalization; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } //Parse message header if (!mHeaderParser.wasAlreadyParsed()) { mHeaderParser.setHead(mMessage->head()); mHeaderParser.parse(); } computeHeaderCanonization(true); if (mPolicy.saveKey() == MessageViewer::MessageViewerSettings::EnumSaveKey::Save) { const QString keyValue = MessageViewer::DKIMManagerKey::self()->keyValue(mDkimInfo.selector(), mDkimInfo.domain()); //qDebug() << " mDkimInfo.selector() " << mDkimInfo.selector() << "mDkimInfo.domain() " << mDkimInfo.domain() << keyValue; if (keyValue.isEmpty()) { downloadKey(mDkimInfo); } else { parseDKIMKeyRecord(keyValue, mDkimInfo.domain(), mDkimInfo.selector(), false); } } else { downloadKey(mDkimInfo); } } void DKIMCheckSignatureJob::computeHeaderCanonization(bool removeQuoteOnContentType) { //Compute Hash Header switch (mDkimInfo.headerCanonization()) { case MessageViewer::DKIMInfo::CanonicalizationType::Unknown: return; case MessageViewer::DKIMInfo::CanonicalizationType::Simple: mHeaderCanonizationResult = headerCanonizationSimple(); break; case MessageViewer::DKIMInfo::CanonicalizationType::Relaxed: mHeaderCanonizationResult = headerCanonizationRelaxed(removeQuoteOnContentType); break; } // In hash step 2, the Signer/Verifier MUST pass the following to the // hash algorithm in the indicated order. // 1. The header fields specified by the "h=" tag, in the order // specified in that tag, and canonicalized using the header // canonicalization algorithm specified in the "c=" tag. Each // header field MUST be terminated with a single CRLF. // 2. The DKIM-Signature header field that exists (verifying) or will // be inserted (signing) in the message, with the value of the "b=" // tag (including all surrounding whitespace) deleted (i.e., treated // as the empty string), canonicalized using the header // canonicalization algorithm specified in the "c=" tag, and without // a trailing CRLF. // add DKIM-Signature header to the hash input // with the value of the "b=" tag (including all surrounding whitespace) deleted //Add dkim-signature as lowercase QString dkimValue = mDkimValue; dkimValue = dkimValue.left(dkimValue.indexOf(QLatin1String("b=")) + 2); switch (mDkimInfo.headerCanonization()) { case MessageViewer::DKIMInfo::CanonicalizationType::Unknown: return; case MessageViewer::DKIMInfo::CanonicalizationType::Simple: - mHeaderCanonizationResult += QLatin1String("\r\n") + MessageViewer::DKIMUtil::headerCanonizationSimple(QLatin1String("dkim-signature"), dkimValue); + mHeaderCanonizationResult += QLatin1String("\r\n") + MessageViewer::DKIMUtil::headerCanonizationSimple(QStringLiteral("dkim-signature"), dkimValue); break; case MessageViewer::DKIMInfo::CanonicalizationType::Relaxed: - mHeaderCanonizationResult += QLatin1String("\r\n") + MessageViewer::DKIMUtil::headerCanonizationRelaxed(QLatin1String("dkim-signature"), dkimValue, removeQuoteOnContentType); + mHeaderCanonizationResult += QLatin1String("\r\n") + MessageViewer::DKIMUtil::headerCanonizationRelaxed(QStringLiteral("dkim-signature"), dkimValue, removeQuoteOnContentType); break; } #ifdef DEBUG_SIGNATURE_DKIM QFile headerFile(QStringLiteral("/tmp/headercanon-kmail-%1.txt").arg(removeQuoteOnContentType ? QLatin1String("removequote") : QLatin1String("withquote"))); headerFile.open(QIODevice::WriteOnly | QIODevice::Text); QTextStream outHeaderStream(&headerFile); outHeaderStream << mHeaderCanonizationResult; headerFile.close(); #endif } void DKIMCheckSignatureJob::setHeaderParser(const DKIMHeaderParser &headerParser) { mHeaderParser = headerParser; } void DKIMCheckSignatureJob::setCheckSignatureAuthenticationResult(const QVector &lst) { mCheckSignatureAuthenticationResult = lst; } QString DKIMCheckSignatureJob::bodyCanonizationSimple() const { /* * canonicalize the body using the simple algorithm * specified in Section 3.4.3 of RFC 6376 */ // The "simple" body canonicalization algorithm ignores all empty lines // at the end of the message body. An empty line is a line of zero // length after removal of the line terminator. If there is no body or // no trailing CRLF on the message body, a CRLF is added. It makes no // other changes to the message body. In more formal terms, the // "simple" body canonicalization algorithm converts "*CRLF" at the end // of the body to a single "CRLF". // Note that a completely empty or missing body is canonicalized as a // single "CRLF"; that is, the canonicalized length will be 2 octets. return MessageViewer::DKIMUtil::bodyCanonizationSimple(QString::fromLatin1(mMessage->encodedBody())); } QString DKIMCheckSignatureJob::bodyCanonizationRelaxed() const { /* * canonicalize the body using the relaxed algorithm * specified in Section 3.4.4 of RFC 6376 */ /* a. Reduce whitespace: * Ignore all whitespace at the end of lines. Implementations MUST NOT remove the CRLF at the end of the line. * Reduce all sequences of WSP within a line to a single SP character. b. Ignore all empty lines at the end of the message body. "Empty line" is defined in Section 3.4.3. If the body is non-empty but does not end with a CRLF, a CRLF is added. (For email, this is only possible when using extensions to SMTP or non-SMTP transport mechanisms.) */ const QString returnValue = MessageViewer::DKIMUtil::bodyCanonizationRelaxed(QString::fromLatin1(mMessage->encodedBody())); return returnValue; } QString DKIMCheckSignatureJob::headerCanonizationSimple() const { QString headers; DKIMHeaderParser parser = mHeaderParser; for (const QString &header : mDkimInfo.listSignedHeader()) { const QString str = parser.headerType(header.toLower()); if (!str.isEmpty()) { if (!headers.isEmpty()) { headers += QLatin1String("\r\n"); } headers += MessageViewer::DKIMUtil::headerCanonizationSimple(header, str); } } return headers; } QString DKIMCheckSignatureJob::headerCanonizationRelaxed(bool removeQuoteOnContentType) const { // The "relaxed" header canonicalization algorithm MUST apply the // following steps in order: // o Convert all header field names (not the header field values) to // lowercase. For example, convert "SUBJect: AbC" to "subject: AbC". // o Unfold all header field continuation lines as described in // [RFC5322]; in particular, lines with terminators embedded in // continued header field values (that is, CRLF sequences followed by // WSP) MUST be interpreted without the CRLF. Implementations MUST // NOT remove the CRLF at the end of the header field value. // o Convert all sequences of one or more WSP characters to a single SP // character. WSP characters here include those before and after a // line folding boundary. // o Delete all WSP characters at the end of each unfolded header field // value. // o Delete any WSP characters remaining before and after the colon // separating the header field name from the header field value. The // colon separator MUST be retained. QString headers; DKIMHeaderParser parser = mHeaderParser; for (const QString &header : mDkimInfo.listSignedHeader()) { const QString str = parser.headerType(header.toLower()); if (!str.isEmpty()) { if (!headers.isEmpty()) { headers += QLatin1String("\r\n"); } headers += MessageViewer::DKIMUtil::headerCanonizationRelaxed(header, str, removeQuoteOnContentType); } } return headers; } void DKIMCheckSignatureJob::downloadKey(const DKIMInfo &info) { DKIMDownloadKeyJob *job = new DKIMDownloadKeyJob(this); job->setDomainName(info.domain()); job->setSelectorName(info.selector()); connect(job, &DKIMDownloadKeyJob::error, this, [this](const QString &errorString) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Impossible to start downloadkey: error returned: " << errorString; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::ImpossibleToDownloadKey; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); }); connect(job, &DKIMDownloadKeyJob::success, this, &DKIMCheckSignatureJob::slotDownloadKeyDone); if (!job->start()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Impossible to start downloadkey"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::ImpossibleToDownloadKey; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); } } void DKIMCheckSignatureJob::slotDownloadKeyDone(const QList &lst, const QString &domain, const QString &selector) { QByteArray ba; if (lst.count() != 1) { for (const QByteArray &b : lst) { ba += b; } //qCDebug(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Key result has more that 1 element" << lst; } else { ba = lst.at(0); } parseDKIMKeyRecord(QString::fromLocal8Bit(ba), domain, selector, true); } void DKIMCheckSignatureJob::parseDKIMKeyRecord(const QString &str, const QString &domain, const QString &selector, bool storeKeyValue) { qCDebug(MESSAGEVIEWER_DKIMCHECKER_LOG) << "void DKIMCheckSignatureJob::parseDKIMKeyRecord(const QString &str, const QString &domain, const QString &selector, bool storeKeyValue) key:" << str; if (!mDkimKeyRecord.parseKey(str)) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Impossible to parse key record " << str; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } if (mDkimKeyRecord.keyType() != QLatin1String("rsa")) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "mDkimKeyRecord key type is unknown " << mDkimKeyRecord.keyType() << " str " << str; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } // if s flag is set in DKIM key record // AUID must be from the same domain as SDID (and not a subdomain) if (mDkimKeyRecord.flags().contains(QLatin1String("s"))) { // s Any DKIM-Signature header fields using the "i=" tag MUST have // the same domain value on the right-hand side of the "@" in the // "i=" tag and the value of the "d=" tag. That is, the "i=" // domain MUST NOT be a subdomain of "d=". Use of this flag is // RECOMMENDED unless subdomaining is required. if (mDkimInfo.iDomain() != mDkimInfo.domain()) { mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::DomainI; Q_EMIT result(createCheckResult()); deleteLater(); return; } } // check that the testing flag is not set if (mDkimKeyRecord.flags().contains(QLatin1String("y"))) { if (!mPolicy.verifySignatureWhenOnlyTest()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Testing mode!"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::TestKeyMode; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } } if (mDkimKeyRecord.publicKey().isEmpty()) { // empty value means that this public key has been revoked qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "mDkimKeyRecord public key is empty. It was revoked "; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::PublicKeyWasRevoked; mStatus = MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; Q_EMIT result(createCheckResult()); deleteLater(); return; } if (storeKeyValue) { Q_EMIT storeKey(str, domain, selector); } verifyRSASignature(); } void DKIMCheckSignatureJob::verifyRSASignature() { QCA::ConvertResult conversionResult; //qDebug() << "mDkimKeyRecord.publicKey() " < currentDate) { mWarning = DKIMCheckSignatureJob::DKIMWarning::SignatureCreatedInFuture; } if (info.signature().isEmpty()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Signature doesn't exist"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::MissingSignature; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if (!info.listSignedHeader().contains(QLatin1String("from"), Qt::CaseInsensitive)) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "From is not include in headers list"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::MissingFrom; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if (info.domain().isEmpty()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Domain is not defined."; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::DomainNotExist; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if (info.query() != QLatin1String("dns/txt")) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Query is incorrect: " << info.query(); mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InvalidQueryMethod; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if ((info.hashingAlgorithm() == MessageViewer::DKIMInfo::HashingAlgorithmType::Any) || (info.hashingAlgorithm() == MessageViewer::DKIMInfo::HashingAlgorithmType::Unknown)) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "body header algorithm is empty"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InvalidBodyHashAlgorithm; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if (info.signingAlgorithm().isEmpty()) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "signature algorithm is empty"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::InvalidSignAlgorithm; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } if (info.hashingAlgorithm() == DKIMInfo::HashingAlgorithmType::Sha1) { if (mPolicy.rsaSha1Policy() == MessageViewer::MessageViewerSettings::EnumPolicyRsaSha1::Nothing) { //nothing } else if (mPolicy.rsaSha1Policy() == MessageViewer::MessageViewerSettings::EnumPolicyRsaSha1::Warning) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "hash algorithm is not secure sha1 : Error"; mWarning = MessageViewer::DKIMCheckSignatureJob::DKIMWarning::HashAlgorithmUnsafe; } else if (mPolicy.rsaSha1Policy() == MessageViewer::MessageViewerSettings::EnumPolicyRsaSha1::Error) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "hash algorithm is not secure sha1: Error"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::HashAlgorithmUnsafeSha1; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } } //qDebug() << "info.agentOrUserIdentifier() " << info.agentOrUserIdentifier() << " info.iDomain() " << info.iDomain(); if (!info.agentOrUserIdentifier().endsWith(info.iDomain())) { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "AUID is not in a subdomain of SDID"; mError = MessageViewer::DKIMCheckSignatureJob::DKIMError::IDomainError; return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Invalid; } //Add more test //TODO check if info is valid return MessageViewer::DKIMCheckSignatureJob::DKIMStatus::Valid; } DKIMCheckSignatureJob::DKIMError DKIMCheckSignatureJob::error() const { return mError; } DKIMCheckSignatureJob::DKIMStatus DKIMCheckSignatureJob::status() const { return mStatus; } void DKIMCheckSignatureJob::setStatus(DKIMCheckSignatureJob::DKIMStatus status) { mStatus = status; } QString DKIMCheckSignatureJob::dkimValue() const { return mDkimValue; } bool DKIMCheckSignatureJob::CheckSignatureResult::isValid() const { return status != DKIMCheckSignatureJob::DKIMStatus::Unknown; } bool DKIMCheckSignatureJob::CheckSignatureResult::operator==(const DKIMCheckSignatureJob::CheckSignatureResult &other) const { return error == other.error && warning == other.warning && status == other.status && fromEmail == other.fromEmail && auid == other.auid && sdid == other.sdid && listSignatureAuthenticationResult == other.listSignatureAuthenticationResult; } bool DKIMCheckSignatureJob::CheckSignatureResult::operator!=(const DKIMCheckSignatureJob::CheckSignatureResult &other) const { return !CheckSignatureResult::operator==(other); } QDebug operator <<(QDebug d, const DKIMCheckSignatureJob::CheckSignatureResult &t) { d << " error " << t.error; d << " warning " << t.warning; d << " status " << t.status; d << " signedBy " << t.sdid; d << " fromEmail " << t.fromEmail; d << " auid " << t.auid; d << " authenticationResult " << t.listSignatureAuthenticationResult; return d; } QDebug operator <<(QDebug d, const DKIMCheckSignatureJob::DKIMCheckSignatureAuthenticationResult &t) { d << " method " << t.method; d << " errorStr " << t.errorStr; d << " status " << t.status; d << " sdid " << t.sdid; d << " auid " << t.auid; return d; } bool DKIMCheckSignatureJob::DKIMCheckSignatureAuthenticationResult::operator==(const DKIMCheckSignatureJob::DKIMCheckSignatureAuthenticationResult &other) const { return errorStr == other.errorStr && method == other.method && status == other.status && sdid == other.sdid && auid == other.auid; } bool DKIMCheckSignatureJob::DKIMCheckSignatureAuthenticationResult::isValid() const { //TODO improve it return (method != AuthenticationMethod::Unknown); } diff --git a/messageviewer/src/dkim-verify/dkimmanagerkey.cpp b/messageviewer/src/dkim-verify/dkimmanagerkey.cpp index cf6e2524..dc1f8fb9 100644 --- a/messageviewer/src/dkim-verify/dkimmanagerkey.cpp +++ b/messageviewer/src/dkim-verify/dkimmanagerkey.cpp @@ -1,125 +1,125 @@ /* Copyright (C) 2018-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimmanagerkey.h" #include "dkimutil.h" #include #include #include #include #include using namespace MessageViewer; DKIMManagerKey::DKIMManagerKey(QObject *parent) : QObject(parent) { mQcaInitializer = new QCA::Initializer(QCA::Practical, 64); loadKeys(); } DKIMManagerKey::~DKIMManagerKey() { delete mQcaInitializer; saveKeys(); } DKIMManagerKey *DKIMManagerKey::self() { static DKIMManagerKey s_self; return &s_self; } QString DKIMManagerKey::keyValue(const QString &selector, const QString &domain) { for (const KeyInfo &keyInfo : qAsConst(mKeys)) { if (keyInfo.selector == selector && keyInfo.domain == domain) { return keyInfo.keyValue; } } return {}; } void DKIMManagerKey::addKey(const KeyInfo &key) { if (!mKeys.contains(key)) { mKeys.append(key); } } void DKIMManagerKey::removeKey(const QString &key) { for (const KeyInfo &keyInfo : qAsConst(mKeys)) { if (keyInfo.keyValue == key) { mKeys.removeAll(keyInfo); break; } } } QVector DKIMManagerKey::keys() const { return mKeys; } void DKIMManagerKey::loadKeys() { const KSharedConfig::Ptr &config = KSharedConfig::openConfig(MessageViewer::DKIMUtil::defaultConfigFileName(), KConfig::NoGlobals); const QStringList keyGroups = config->groupList().filter(QRegularExpression(QStringLiteral("DKIM Key Record #\\d+"))); mKeys.clear(); for (const QString &groupName : keyGroups) { KConfigGroup group = config->group(groupName); - const QString selector = group.readEntry(QLatin1String("Selector"), QString()); - const QString domain = group.readEntry(QLatin1String("Domain"), QString()); - const QString key = group.readEntry(QLatin1String("Key"), QString()); + const QString selector = group.readEntry(QStringLiteral("Selector"), QString()); + const QString domain = group.readEntry(QStringLiteral("Domain"), QString()); + const QString key = group.readEntry(QStringLiteral("Key"), QString()); mKeys.append(KeyInfo{key, selector, domain}); } } void DKIMManagerKey::saveKeys() { const KSharedConfig::Ptr &config = KSharedConfig::openConfig(MessageViewer::DKIMUtil::defaultConfigFileName(), KConfig::NoGlobals); const QStringList filterGroups = config->groupList().filter(QRegularExpression(QStringLiteral("DKIM Key Record #\\d+"))); for (const QString &group : filterGroups) { config->deleteGroup(group); } for (int i = 0, total = mKeys.count(); i < total; ++i) { const QString groupName = QStringLiteral("DKIM Key Record #%1").arg(i); KConfigGroup group = config->group(groupName); const KeyInfo &info = mKeys.at(i); group.writeEntry(QStringLiteral("Selector"), info.selector); group.writeEntry(QStringLiteral("Domain"), info.domain); group.writeEntry(QStringLiteral("Key"), info.keyValue); } } void DKIMManagerKey::saveKeys(const QVector &lst) { mKeys = lst; saveKeys(); } bool KeyInfo::operator ==(const KeyInfo &other) const { return keyValue == other.keyValue && selector == other.selector && domain == other.domain; } diff --git a/messageviewer/src/dkim-verify/dkimmanageruleswidget.cpp b/messageviewer/src/dkim-verify/dkimmanageruleswidget.cpp index 28c0cd6b..1b6ec098 100644 --- a/messageviewer/src/dkim-verify/dkimmanageruleswidget.cpp +++ b/messageviewer/src/dkim-verify/dkimmanageruleswidget.cpp @@ -1,208 +1,208 @@ /* Copyright (C) 2019-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimmanagerulescombobox.h" #include "dkimmanageruleswidget.h" #include "dkimruledialog.h" #include "messageviewer_dkimcheckerdebug.h" #include #include #include #include #include #include #include #include #include using namespace MessageViewer; DKIMManageRulesWidgetItem::DKIMManageRulesWidgetItem(QTreeWidget *parent) : QTreeWidgetItem(parent) { mRuleTypeCombobox = new DKIMManageRulesComboBox; treeWidget()->setItemWidget(this, ColumnType::RuleType, mRuleTypeCombobox); } DKIMManageRulesWidgetItem::~DKIMManageRulesWidgetItem() { } MessageViewer::DKIMRule DKIMManageRulesWidgetItem::rule() const { return mRule; } void DKIMManageRulesWidgetItem::setRule(const MessageViewer::DKIMRule &rule) { if (mRule != rule) { mRule = rule; updateInfo(); } } void DKIMManageRulesWidgetItem::updateInfo() { setCheckState(ColumnType::Enabled, mRule.enabled() ? Qt::Checked : Qt::Unchecked); setText(ColumnType::Domain, mRule.domain()); setText(ColumnType::ListId, mRule.listId()); setText(ColumnType::From, mRule.from()); setText(ColumnType::SDid, mRule.signedDomainIdentifier().join(QLatin1Char(' '))); setText(ColumnType::Priority, QString::number(mRule.priority())); mRuleTypeCombobox->setRuleType(mRule.ruleType()); } DKIMManageRulesWidget::DKIMManageRulesWidget(QWidget *parent) : QWidget(parent) { QVBoxLayout *mainLayout = new QVBoxLayout(this); mainLayout->setObjectName(QStringLiteral("mainLayout")); mainLayout->setContentsMargins(0, 0, 0, 0); mTreeWidget = new QTreeWidget(this); mTreeWidget->setObjectName(QStringLiteral("treewidget")); mTreeWidget->setRootIsDecorated(false); mTreeWidget->setHeaderLabels({i18n("Active"), i18n("Domain"), i18n("List-ID"), i18n("From"), i18n("SDID"), i18n("Rule type"), i18n("Priority")}); mTreeWidget->setContextMenuPolicy(Qt::CustomContextMenu); mTreeWidget->setAlternatingRowColors(true); KTreeWidgetSearchLine *searchLineEdit = new KTreeWidgetSearchLine(this, mTreeWidget); searchLineEdit->setObjectName(QStringLiteral("searchlineedit")); searchLineEdit->setClearButtonEnabled(true); mainLayout->addWidget(searchLineEdit); mainLayout->addWidget(mTreeWidget); - connect(mTreeWidget, &QTreeWidget::customContextMenuRequested, this, &DKIMManageRulesWidget::customContextMenuRequested); + connect(mTreeWidget, &QTreeWidget::customContextMenuRequested, this, &DKIMManageRulesWidget::slotCustomContextMenuRequested); connect(mTreeWidget, &QTreeWidget::itemDoubleClicked, this, [this](QTreeWidgetItem *item) { if (item) { DKIMManageRulesWidgetItem *rulesItem = dynamic_cast(item); modifyRule(rulesItem); } }); } DKIMManageRulesWidget::~DKIMManageRulesWidget() { } void DKIMManageRulesWidget::loadSettings() { const QVector rules = MessageViewer::DKIMManagerRules::self()->rules(); for (const MessageViewer::DKIMRule &rule : rules) { DKIMManageRulesWidgetItem *item = new DKIMManageRulesWidgetItem(mTreeWidget); item->setRule(rule); } } void DKIMManageRulesWidget::saveSettings() { QVector rules; for (int i = 0, total = mTreeWidget->topLevelItemCount(); i < total; ++i) { QTreeWidgetItem *item = mTreeWidget->topLevelItem(i); DKIMManageRulesWidgetItem *ruleItem = static_cast(item); rules.append(ruleItem->rule()); } MessageViewer::DKIMManagerRules::self()->saveRules(rules); } QByteArray DKIMManageRulesWidget::saveHeaders() const { return mTreeWidget->header()->saveState(); } void DKIMManageRulesWidget::restoreHeaders(const QByteArray &header) { mTreeWidget->header()->restoreState(header); } void DKIMManageRulesWidget::addRule() { QPointer dlg = new DKIMRuleDialog(this); if (dlg->exec()) { const MessageViewer::DKIMRule rule = dlg->rule(); if (rule.isValid()) { DKIMManageRulesWidgetItem *item = new DKIMManageRulesWidgetItem(mTreeWidget); item->setRule(rule); } else { qCDebug(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Rule is not valid"; } } delete dlg; } void DKIMManageRulesWidget::duplicateRule(DKIMManageRulesWidgetItem *rulesItem) { QPointer dlg = new DKIMRuleDialog(this); dlg->loadRule(rulesItem->rule()); if (dlg->exec()) { const MessageViewer::DKIMRule rule = dlg->rule(); if (rule.isValid()) { DKIMManageRulesWidgetItem *item = new DKIMManageRulesWidgetItem(mTreeWidget); item->setRule(rule); } } delete dlg; } void DKIMManageRulesWidget::modifyRule(DKIMManageRulesWidgetItem *rulesItem) { QPointer dlg = new DKIMRuleDialog(this); dlg->loadRule(rulesItem->rule()); if (dlg->exec()) { const MessageViewer::DKIMRule rule = dlg->rule(); if (rule.isValid()) { rulesItem->setRule(rule); } } delete dlg; } -void DKIMManageRulesWidget::customContextMenuRequested(const QPoint &pos) +void DKIMManageRulesWidget::slotCustomContextMenuRequested(const QPoint &pos) { Q_UNUSED(pos); QTreeWidgetItem *item = mTreeWidget->currentItem(); QMenu menu(this); menu.addAction(QIcon::fromTheme(QStringLiteral("list-add")), i18n("Add..."), this, [this]() { addRule(); }); DKIMManageRulesWidgetItem *rulesItem = dynamic_cast(item); if (rulesItem) { menu.addAction(QIcon::fromTheme(QStringLiteral("document-edit")), i18n("Modify..."), this, [this, rulesItem]() { modifyRule(rulesItem); }); menu.addSeparator(); menu.addAction(QIcon::fromTheme(QStringLiteral("edit-duplicate")), i18n("Duplicate Rule"), this, [this, rulesItem]() { duplicateRule(rulesItem); }); menu.addSeparator(); menu.addAction(QIcon::fromTheme(QStringLiteral("edit-delete")), i18n("Remove Rule"), this, [this, item]() { if (KMessageBox::Yes == KMessageBox::warningYesNo(this, i18n("Do you want to delete this rule?"), i18n("Delete Rule"))) { delete item; } }); } if (mTreeWidget->topLevelItemCount() > 0) { menu.addSeparator(); menu.addAction(i18n("Delete All"), this, [this]() { if (KMessageBox::Yes == KMessageBox::warningYesNo(this, i18n("Do you want to delete all the rules?"), i18n("Delete Rules"))) { mTreeWidget->clear(); } }); } menu.exec(QCursor::pos()); } diff --git a/messageviewer/src/dkim-verify/dkimmanageruleswidget.h b/messageviewer/src/dkim-verify/dkimmanageruleswidget.h index c0305d58..adce963c 100644 --- a/messageviewer/src/dkim-verify/dkimmanageruleswidget.h +++ b/messageviewer/src/dkim-verify/dkimmanageruleswidget.h @@ -1,81 +1,81 @@ /* Copyright (C) 2019-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef DKIMMANAGERULESWIDGET_H #define DKIMMANAGERULESWIDGET_H #include "messageviewer_export.h" #include #include #include class QTreeWidget; namespace MessageViewer { class DKIMManageRulesComboBox; /** * @brief The DKIMManageRulesWidgetItem class * @author Laurent Montel */ class MESSAGEVIEWER_EXPORT DKIMManageRulesWidgetItem : public QTreeWidgetItem { public: enum ColumnType { Enabled = 0, Domain = 1, ListId = 2, From = 3, SDid = 4, RuleType = 5, Priority = 6, }; explicit DKIMManageRulesWidgetItem(QTreeWidget *parent = nullptr); ~DKIMManageRulesWidgetItem(); Q_REQUIRED_RESULT MessageViewer::DKIMRule rule() const; void setRule(const MessageViewer::DKIMRule &rule); private: void updateInfo(); MessageViewer::DKIMRule mRule; DKIMManageRulesComboBox *mRuleTypeCombobox = nullptr; }; /** * @brief The DKIMManageRulesWidget class * @author Laurent Montel */ class MESSAGEVIEWER_EXPORT DKIMManageRulesWidget : public QWidget { Q_OBJECT public: explicit DKIMManageRulesWidget(QWidget *parent = nullptr); ~DKIMManageRulesWidget(); void loadSettings(); void saveSettings(); Q_REQUIRED_RESULT QByteArray saveHeaders() const; void restoreHeaders(const QByteArray &header); void addRule(); private: void modifyRule(DKIMManageRulesWidgetItem *rulesItem); - void customContextMenuRequested(const QPoint &); + void slotCustomContextMenuRequested(const QPoint &); void duplicateRule(DKIMManageRulesWidgetItem *rulesItem); QTreeWidget *mTreeWidget = nullptr; }; } #endif // DKIMMANAGERULESWIDGET_H diff --git a/messageviewer/src/dkim-verify/dkimutil.cpp b/messageviewer/src/dkim-verify/dkimutil.cpp index 7e72e724..4f4468bd 100644 --- a/messageviewer/src/dkim-verify/dkimutil.cpp +++ b/messageviewer/src/dkim-verify/dkimutil.cpp @@ -1,204 +1,204 @@ /* Copyright (C) 2018-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "dkimutil.h" #include "messageviewer_dkimcheckerdebug.h" #include #include QString MessageViewer::DKIMUtil::bodyCanonizationRelaxed(QString body) { /* * canonicalize the body using the relaxed algorithm * specified in Section 3.4.4 of RFC 6376 */ /* a. Reduce whitespace: * Ignore all whitespace at the end of lines. Implementations MUST NOT remove the CRLF at the end of the line. * Reduce all sequences of WSP within a line to a single SP character. b. Ignore all empty lines at the end of the message body. "Empty line" is defined in Section 3.4.3. If the body is non-empty but does not end with a CRLF, a CRLF is added. (For email, this is only possible when using extensions to SMTP or non-SMTP transport mechanisms.) */ - body.replace(QLatin1String("\n"), QLatin1String("\r\n")); - body.replace(QRegularExpression(QLatin1String("[ \t]+\r\n")), QLatin1String("\r\n")); - body.replace(QRegularExpression(QLatin1String("[ \t]+")), QStringLiteral(" ")); - body.replace(QRegularExpression(QLatin1String("((\r\n)+?)$")), QLatin1String("\r\n")); + body.replace(QStringLiteral("\n"), QStringLiteral("\r\n")); + body.replace(QRegularExpression(QStringLiteral("[ \t]+\r\n")), QStringLiteral("\r\n")); + body.replace(QRegularExpression(QStringLiteral("[ \t]+")), QStringLiteral(" ")); + body.replace(QRegularExpression(QStringLiteral("((\r\n)+?)$")), QStringLiteral("\r\n")); if (body == QLatin1String("\r\n")) { body.clear(); } return body; } QString MessageViewer::DKIMUtil::bodyCanonizationSimple(QString body) { // The "simple" body canonicalization algorithm ignores all empty lines // at the end of the message body. An empty line is a line of zero // length after removal of the line terminator. If there is no body or // no trailing CRLF on the message body, a CRLF is added. It makes no // other changes to the message body. In more formal terms, the // "simple" body canonicalization algorithm converts "*CRLF" at the end // of the body to a single "CRLF". // Note that a completely empty or missing body is canonicalized as a // single "CRLF"; that is, the canonicalized length will be 2 octets. - body.replace(QLatin1String("\n"), QLatin1String("\r\n")); - body.replace(QRegularExpression(QLatin1String("((\r\n)+)?$")), QLatin1String("\r\n")); + body.replace(QStringLiteral("\n"), QStringLiteral("\r\n")); + body.replace(QRegularExpression(QStringLiteral("((\r\n)+)?$")), QStringLiteral("\r\n")); if (body.endsWith(QLatin1String("\r\n"))) { //Remove it from start body.chop(2); } if (body.isEmpty()) { - body = QLatin1String("\r\n"); + body = QStringLiteral("\r\n"); } return body; } QByteArray MessageViewer::DKIMUtil::generateHash(const QByteArray &body, QCryptographicHash::Algorithm algo) { return QCryptographicHash::hash(body, algo).toBase64(); } QString MessageViewer::DKIMUtil::headerCanonizationSimple(const QString &headerName, const QString &headerValue) { //TODO verify it lower it ? return headerName + QLatin1Char(':') + headerValue; } QString MessageViewer::DKIMUtil::headerCanonizationRelaxed(const QString &headerName, const QString &headerValue, bool removeQuoteOnContentType) { // The "relaxed" header canonicalization algorithm MUST apply the // following steps in order: // o Convert all header field names (not the header field values) to // lowercase. For example, convert "SUBJect: AbC" to "subject: AbC". // o Unfold all header field continuation lines as described in // [RFC5322]; in particular, lines with terminators embedded in // continued header field values (that is, CRLF sequences followed by // WSP) MUST be interpreted without the CRLF. Implementations MUST // NOT remove the CRLF at the end of the header field value. // o Convert all sequences of one or more WSP characters to a single SP // character. WSP characters here include those before and after a // line folding boundary. // o Delete all WSP characters at the end of each unfolded header field // value. // o Delete any WSP characters remaining before and after the colon // separating the header field name from the header field value. The // colon separator MUST be retained. QString newHeaderName = headerName.toLower(); QString newHeaderValue = headerValue; newHeaderValue.replace(QRegularExpression(QStringLiteral("\r\n[ \t]+")), QStringLiteral(" ")); newHeaderValue.replace(QRegularExpression(QStringLiteral("[ \t]+")), QStringLiteral(" ")); newHeaderValue.replace(QRegularExpression(QStringLiteral("[ \t]+\r\n")), QStringLiteral("\r\n")); //Perhaps remove tab after headername and before value name //newHeaderValue.replace(QRegularExpression(QStringLiteral("[ \t]*:[ \t]")), QStringLiteral(":")); if (newHeaderName == QLatin1String("content-type") && removeQuoteOnContentType) { //Remove quote in charset if (newHeaderValue.contains(QLatin1String("charset=\""))) { newHeaderValue.remove(QLatin1Char('"')); } } //Remove extra space. newHeaderValue = newHeaderValue.trimmed(); return newHeaderName + QLatin1Char(':') + newHeaderValue; } QString MessageViewer::DKIMUtil::cleanString(QString str) { //Move as static ? // WSP help pattern as specified in Section 2.8 of RFC 6376 const QString pattWSP = QStringLiteral("[ \t]"); // FWS help pattern as specified in Section 2.8 of RFC 6376 const QString pattFWS = QStringLiteral("(?:") + pattWSP + QStringLiteral("*(?:\r\n)?") + pattWSP + QStringLiteral("+)"); str.replace(QRegularExpression(pattFWS), QString()); return str; } QString MessageViewer::DKIMUtil::emailDomain(const QString &emailDomain) { return emailDomain.right(emailDomain.length() - emailDomain.indexOf(QLatin1Char('@')) - 1); } QString MessageViewer::DKIMUtil::emailSubDomain(const QString &emailDomain) { int dotNumber = 0; for (int i = emailDomain.length() - 1; i >= 0; --i) { if (emailDomain.at(i) == QLatin1Char('.')) { dotNumber++; if (dotNumber == 2) { return emailDomain.right(emailDomain.length() - i - 1); } } } return emailDomain; } QString MessageViewer::DKIMUtil::defaultConfigFileName() { return QStringLiteral("dkimsettingsrc"); } QString MessageViewer::DKIMUtil::convertAuthenticationMethodEnumToString(MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod method) { QString methodStr; switch (method) { case MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Unknown: qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Undefined type"; break; case MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dkim: methodStr = QStringLiteral("dkim"); break; case MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Spf: methodStr = QStringLiteral("spf"); break; case MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dmarc: methodStr = QStringLiteral("dmarc"); break; case MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dkimatps: methodStr = QStringLiteral("dkim-atps"); break; } return methodStr; } MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod MessageViewer::DKIMUtil::convertAuthenticationMethodStringToEnum(const QString &str) { if (str == QLatin1String("dkim")) { return MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dkim; } else if (str == QLatin1String("spf")) { return MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Spf; } else if (str == QLatin1String("dmarc")) { return MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dmarc; } else if (str == QLatin1String("dkim-atps")) { return MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Dkimatps; } else { qCWarning(MESSAGEVIEWER_DKIMCHECKER_LOG) << "Undefined type " << str; return MessageViewer::DKIMCheckSignatureJob::AuthenticationMethod::Unknown; } } diff --git a/messageviewer/src/header/kxface.cpp b/messageviewer/src/header/kxface.cpp index 0b5b7708..d0e2681a 100644 --- a/messageviewer/src/header/kxface.cpp +++ b/messageviewer/src/header/kxface.cpp @@ -1,734 +1,734 @@ /* This file is part of libkdepim. Original compface: Copyright (c) James Ashton - Sydney University - June 1990. //krazy:exclude=copyright Additions for KDE: Copyright (c) 2004 Jakob Schröter This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "kxface.h" #include #include #include #include #include #include #define GEN(g) F[h] ^= G.g[k]; break #define BITSPERDIG 4 #define DIGITS (PIXELS / BITSPERDIG) #define DIGSPERWORD 4 #define WORDSPERLINE (WIDTH / DIGSPERWORD / BITSPERDIG) /* compressed output uses the full range of printable characters. * in ascii these are in a contiguous block so we just need to know * the first and last. The total number of printables is needed too */ #define FIRSTPRINT '!' #define LASTPRINT '~' #define NUMPRINTS (LASTPRINT - FIRSTPRINT + 1) /* output line length for compressed data */ static const int MAXLINELEN = 78; /* Portable, very large unsigned integer arithmetic is needed. * Implementation uses arrays of WORDs. COMPs must have at least * twice as many bits as WORDs to handle intermediate results */ #define COMP unsigned long #define WORDCARRY (1 << BITSPERWORD) #define WORDMASK (WORDCARRY - 1) #define ERR_OK 0 /* successful completion */ #define ERR_EXCESS 1 /* completed OK but some input was ignored */ #define ERR_INSUFF -1 /* insufficient input. Bad face format? */ #define ERR_INTERNAL -2 /* Arithmetic overflow or buffer overflow */ #define BLACK 0 #define GREY 1 #define WHITE 2 static const int MAX_XFACE_LENGTH = 2048; using namespace MessageViewer; KXFace::KXFace() { NumProbs = 0; } KXFace::~KXFace() { } QString KXFace::fromImage(const QImage &image) { if (image.isNull()) { return QString(); } QImage scaledImg = image.scaled(48, 48, Qt::IgnoreAspectRatio, Qt::SmoothTransformation); QByteArray ba; QBuffer buffer(&ba, this); buffer.open(QIODevice::WriteOnly); scaledImg.save(&buffer, "XBM"); QString xbm(QString::fromLatin1(ba)); xbm.remove(0, xbm.indexOf(QLatin1String("{")) + 1); xbm.truncate(xbm.indexOf(QLatin1String("}"))); xbm.remove(QLatin1Char(' ')); xbm.remove(QLatin1Char(',')); xbm.remove(QStringLiteral("0x")); xbm.remove(QLatin1Char('\n')); xbm.truncate(576); QString tmp = QLatin1String(xbm.toLatin1()); int len = tmp.length(); for (int i = 0; i < len; ++i) { switch (tmp[i].toLatin1()) { case '1': tmp[i] = '8'; break; case '2': tmp[i] = '4'; break; case '3': tmp[i] = 'c'; break; case '4': tmp[i] = '2'; break; case '5': tmp[i] = 'a'; break; case '7': tmp[i] = 'e'; break; case '8': tmp[i] = '1'; break; case 'A': case 'a': tmp[i] = '5'; break; case 'B': case 'b': tmp[i] = 'd'; break; case 'C': case 'c': tmp[i] = '3'; break; case 'D': case 'd': tmp[i] = 'b'; break; case 'E': case 'e': tmp[i] = '7'; break; } if (i % 2) { QChar t = tmp[i]; tmp[i] = tmp[i - 1]; tmp[i - 1] = t; } } - tmp.replace(QRegularExpression(QLatin1String("(\\w{12})")), QStringLiteral("\\1\n")); - tmp.replace(QRegularExpression(QLatin1String("(\\w{4})")), QStringLiteral("0x\\1,")); + tmp.replace(QRegularExpression(QStringLiteral("(\\w{12})")), QStringLiteral("\\1\n")); + tmp.replace(QRegularExpression(QStringLiteral("(\\w{4})")), QStringLiteral("0x\\1,")); len = tmp.length(); char *fbuf = (char *)malloc(len + 1); strncpy(fbuf, tmp.toLatin1().constData(), len); fbuf[len] = '\0'; if (!(status = setjmp(comp_env))) { ReadFace(fbuf); GenFace(); CompAll(fbuf); } QString ret(QString::fromLatin1(fbuf)); free(fbuf); return ret; } QImage KXFace::toImage(const QString &xface) { if (xface.length() > MAX_XFACE_LENGTH) { return QImage(); } char *fbuf = (char *)malloc(MAX_XFACE_LENGTH); memset(fbuf, '\0', MAX_XFACE_LENGTH); strncpy(fbuf, xface.toLatin1().constData(), xface.length()); QByteArray img; if (!(status = setjmp(comp_env))) { UnCompAll(fbuf); /* compress otherwise */ UnGenFace(); img = WriteFace(); } free(fbuf); QImage p; p.loadFromData(img, "XBM"); return p; } //============================================================================ // more or less original compface 1.4 source void KXFace::RevPush(const Prob *p) { if (NumProbs >= PIXELS * 2 - 1) { longjmp(comp_env, ERR_INTERNAL); } ProbBuf[NumProbs++] = (Prob *)p; } void KXFace::BigPush(Prob *p) { static unsigned char tmp; BigDiv(p->p_range, &tmp); BigMul(0); BigAdd(tmp + p->p_offset); } int KXFace::BigPop(const Prob *p) { static unsigned char tmp; int i; BigDiv(0, &tmp); i = 0; while ((tmp < p->p_offset) || (tmp >= p->p_range + p->p_offset)) { p++; ++i; } BigMul(p->p_range); BigAdd(tmp - p->p_offset); return i; } /* Divide B by a storing the result in B and the remainder in the word * pointer to by r */ void KXFace::BigDiv(unsigned char a, unsigned char *r) { int i; unsigned char *w; COMP c, d; a &= WORDMASK; if ((a == 1) || (B.b_words == 0)) { *r = 0; return; } if (a == 0) { /* treat this as a == WORDCARRY */ /* and just shift everything right a WORD (unsigned char)*/ i = --B.b_words; w = B.b_word; *r = *w; while (i--) { *w = *(w + 1); w++; } *w = 0; return; } w = B.b_word + (i = B.b_words); c = 0; while (i--) { c <<= BITSPERWORD; c += (COMP)*--w; d = c / (COMP)a; c = c % (COMP)a; *w = (unsigned char)(d & WORDMASK); } *r = c; if (B.b_word[B.b_words - 1] == 0) { B.b_words--; } } /* Multiply a by B storing the result in B */ void KXFace::BigMul(unsigned char a) { int i; unsigned char *w; COMP c; a &= WORDMASK; if ((a == 1) || (B.b_words == 0)) { return; } if (a == 0) { /* treat this as a == WORDCARRY */ /* and just shift everything left a WORD (unsigned char) */ if ((i = B.b_words++) >= MAXWORDS - 1) { longjmp(comp_env, ERR_INTERNAL); } w = B.b_word + i; while (i--) { *w = *(w - 1); w--; } *w = 0; return; } i = B.b_words; w = B.b_word; c = 0; while (i--) { c += (COMP)*w * (COMP)a; *(w++) = (unsigned char)(c & WORDMASK); c >>= BITSPERWORD; } if (c) { if (B.b_words++ >= MAXWORDS) { longjmp(comp_env, ERR_INTERNAL); } *w = (COMP)(c & WORDMASK); } } /* Add to a to B storing the result in B */ void KXFace::BigAdd(unsigned char a) { int i; unsigned char *w; COMP c; a &= WORDMASK; if (a == 0) { return; } i = 0; w = B.b_word; c = a; while ((i < B.b_words) && c) { c += (COMP)*w; *w++ = (unsigned char)(c & WORDMASK); c >>= BITSPERWORD; ++i; } if ((i == B.b_words) && c) { if (B.b_words++ >= MAXWORDS) { longjmp(comp_env, ERR_INTERNAL); } *w = (COMP)(c & WORDMASK); } } void KXFace::BigClear() { B.b_words = 0; } QByteArray KXFace::WriteFace() { char *s; int i, j, bits, digits, words; //int digsperword = DIGSPERWORD; //int wordsperline = WORDSPERLINE; QByteArray t( "#define noname_width 48\n#define noname_height 48\nstatic char noname_bits[] = {\n "); j = t.length() - 1; s = F; bits = digits = words = i = 0; t.resize(MAX_XFACE_LENGTH); int digsperword = 2; int wordsperline = 15; while (s < F + PIXELS) { if ((bits == 0) && (digits == 0)) { t[j++] = '0'; t[j++] = 'x'; } if (*(s++)) { i = (i >> 1) | 0x8; } else { i >>= 1; } if (++bits == BITSPERDIG) { j++; t[j - ((digits & 1) * 2)] = *(i + HexDigits); bits = i = 0; if (++digits == digsperword) { if (s >= F + PIXELS) { break; } t[j++] = ','; digits = 0; if (++words == wordsperline) { t[j++] = '\n'; t[j++] = ' '; words = 0; } } } } t.resize(j + 1); t += "};\n"; return t; } void KXFace::UnCompAll(char *fbuf) { char *p; BigClear(); BigRead(fbuf); p = F; while (p < F + PIXELS) { *(p++) = 0; } UnCompress(F, 16, 16, 0); UnCompress(F + 16, 16, 16, 0); UnCompress(F + 32, 16, 16, 0); UnCompress(F + WIDTH * 16, 16, 16, 0); UnCompress(F + WIDTH * 16 + 16, 16, 16, 0); UnCompress(F + WIDTH * 16 + 32, 16, 16, 0); UnCompress(F + WIDTH * 32, 16, 16, 0); UnCompress(F + WIDTH * 32 + 16, 16, 16, 0); UnCompress(F + WIDTH * 32 + 32, 16, 16, 0); } void KXFace::UnCompress(char *f, int wid, int hei, int lev) { switch (BigPop(&levels[lev][0])) { case WHITE: return; case BLACK: PopGreys(f, wid, hei); return; default: wid /= 2; hei /= 2; lev++; UnCompress(f, wid, hei, lev); UnCompress(f + wid, wid, hei, lev); UnCompress(f + hei * WIDTH, wid, hei, lev); UnCompress(f + wid + hei * WIDTH, wid, hei, lev); return; } } void KXFace::BigWrite(char *fbuf) { static unsigned char tmp; static char buf[DIGITS]; char *s; int i; s = buf; while (B.b_words > 0) { BigDiv(NUMPRINTS, &tmp); *(s++) = tmp + FIRSTPRINT; } i = 7; // leave room for the field name on the first line *(fbuf++) = ' '; while (s-- > buf) { if (i == 0) { *(fbuf++) = ' '; } *(fbuf++) = *s; if (++i >= MAXLINELEN) { *(fbuf++) = '\n'; i = 0; } } if (i > 0) { *(fbuf++) = '\n'; } *(fbuf++) = '\0'; } void KXFace::BigRead(char *fbuf) { int c; while (*fbuf != '\0') { c = *(fbuf++); if ((c < FIRSTPRINT) || (c > LASTPRINT)) { continue; } BigMul(NUMPRINTS); BigAdd((unsigned char)(c - FIRSTPRINT)); } } void KXFace::ReadFace(char *fbuf) { int c, i; char *s, *t; t = s = fbuf; for (i = strlen(s); i > 0; --i) { c = (int)*(s++); if ((c >= '0') && (c <= '9')) { if (t >= fbuf + DIGITS) { status = ERR_EXCESS; break; } *(t++) = c - '0'; } else if ((c >= 'A') && (c <= 'F')) { if (t >= fbuf + DIGITS) { status = ERR_EXCESS; break; } *(t++) = c - 'A' + 10; } else if ((c >= 'a') && (c <= 'f')) { if (t >= fbuf + DIGITS) { status = ERR_EXCESS; break; } *(t++) = c - 'a' + 10; } else if (((c == 'x') || (c == 'X')) && (t > fbuf) && (*(t - 1) == 0)) { t--; } } if (t < fbuf + DIGITS) { longjmp(comp_env, ERR_INSUFF); } s = fbuf; t = F; c = 1 << (BITSPERDIG - 1); while (t < F + PIXELS) { *(t++) = (*s & c) ? 1 : 0; if ((c >>= 1) == 0) { s++; c = 1 << (BITSPERDIG - 1); } } } void KXFace::GenFace() { static char newp[PIXELS]; char *f1; char *f2; int i; f1 = newp; f2 = F; i = PIXELS; while (i-- > 0) { *(f1++) = *(f2++); } Gen(newp); } void KXFace::UnGenFace() { Gen(F); } // static void KXFace::Gen(char *f) { int m, l, k, j, i, h; for (j = 0; j < HEIGHT; ++j) { for (i = 0; i < WIDTH; ++i) { h = i + j * WIDTH; k = 0; for (l = i - 2; l <= i + 2; ++l) { for (m = j - 2; m <= j; ++m) { if ((l >= i) && (m == j)) { continue; } if ((l > 0) && (l <= WIDTH) && (m > 0)) { k = *(f + l + m * WIDTH) ? k * 2 + 1 : k * 2; } } } switch (i) { case 1: switch (j) { case 1: GEN(g_22); case 2: GEN(g_21); default: GEN(g_20); } break; case 2: switch (j) { case 1: GEN(g_12); case 2: GEN(g_11); default: GEN(g_10); } break; case WIDTH - 1: switch (j) { case 1: GEN(g_42); case 2: GEN(g_41); default: GEN(g_40); } break; /* i runs from 0 to WIDTH-1, so case can never occur. I leave the code in because it appears exactly like this in the original compface code. case WIDTH : switch (j) { case 1 : GEN(g_32); case 2 : GEN(g_31); default : GEN(g_30); } break; */ default: switch (j) { case 1: GEN(g_02); case 2: GEN(g_01); default: GEN(g_00); } break; } } } } void KXFace::PopGreys(char *f, int wid, int hei) { if (wid > 3) { wid /= 2; hei /= 2; PopGreys(f, wid, hei); PopGreys(f + wid, wid, hei); PopGreys(f + WIDTH * hei, wid, hei); PopGreys(f + WIDTH * hei + wid, wid, hei); } else { wid = BigPop(freqs); if (wid & 1) { *f = 1; } if (wid & 2) { *(f + 1) = 1; } if (wid & 4) { *(f + WIDTH) = 1; } if (wid & 8) { *(f + WIDTH + 1) = 1; } } } void KXFace::CompAll(char *fbuf) { Compress(F, 16, 16, 0); Compress(F + 16, 16, 16, 0); Compress(F + 32, 16, 16, 0); Compress(F + WIDTH * 16, 16, 16, 0); Compress(F + WIDTH * 16 + 16, 16, 16, 0); Compress(F + WIDTH * 16 + 32, 16, 16, 0); Compress(F + WIDTH * 32, 16, 16, 0); Compress(F + WIDTH * 32 + 16, 16, 16, 0); Compress(F + WIDTH * 32 + 32, 16, 16, 0); BigClear(); while (NumProbs > 0) { BigPush(ProbBuf[--NumProbs]); } BigWrite(fbuf); } void KXFace::Compress(char *f, int wid, int hei, int lev) { if (AllWhite(f, wid, hei)) { RevPush(&levels[lev][WHITE]); return; } if (AllBlack(f, wid, hei)) { RevPush(&levels[lev][BLACK]); PushGreys(f, wid, hei); return; } RevPush(&levels[lev][GREY]); wid /= 2; hei /= 2; lev++; Compress(f, wid, hei, lev); Compress(f + wid, wid, hei, lev); Compress(f + hei * WIDTH, wid, hei, lev); Compress(f + wid + hei * WIDTH, wid, hei, lev); } int KXFace::AllWhite(char *f, int wid, int hei) { return (*f == 0) && Same(f, wid, hei); } int KXFace::AllBlack(char *f, int wid, int hei) { if (wid > 3) { wid /= 2; hei /= 2; return AllBlack(f, wid, hei) && AllBlack(f + wid, wid, hei) && AllBlack(f + WIDTH * hei, wid, hei) && AllBlack(f + WIDTH * hei + wid, wid, hei); } else { return *f || *(f + 1) || *(f + WIDTH) || *(f + WIDTH + 1); } } int KXFace::Same(char *f, int wid, int hei) { char val, *row; int x; val = *f; while (hei--) { row = f; x = wid; while (x--) { if (*(row++) != val) { return 0; } } f += WIDTH; } return 1; } void KXFace::PushGreys(char *f, int wid, int hei) { if (wid > 3) { wid /= 2; hei /= 2; PushGreys(f, wid, hei); PushGreys(f + wid, wid, hei); PushGreys(f + WIDTH * hei, wid, hei); PushGreys(f + WIDTH * hei + wid, wid, hei); } else { RevPush(freqs + *f + 2 * *(f + 1) + 4 * *(f + WIDTH) +8 * *(f + WIDTH + 1)); } } diff --git a/mimetreeparser/src/bodypartformatterfactory.cpp b/mimetreeparser/src/bodypartformatterfactory.cpp index 39cae6d5..ed1f2c37 100644 --- a/mimetreeparser/src/bodypartformatterfactory.cpp +++ b/mimetreeparser/src/bodypartformatterfactory.cpp @@ -1,184 +1,184 @@ /* bodypartformatterfactory.cpp This file is part of KMail, the KDE mail client. Copyright (c) 2004 Marc Mutz , Ingo Kloecker KMail is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. KMail is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA In addition, as a special exception, the copyright holders give permission to link the code of this program with any edition of the Qt library by Trolltech AS, Norway (or with modified versions of Qt that use the same license as Qt), and distribute linked combinations including the two. You must obey the GNU General Public License in all respects for all of the code used other than Qt. If you modify this file, you may extend this exception to your version of the file, but you are not obligated to do so. If you do not wish to do so, delete this exception statement from your version. */ #include "bodypartformatterfactory.h" #include "bodypartformatterfactory_p.h" #include "interfaces/bodypartformatter.h" #include "mimetreeparser_debug.h" #include #include #include #include using namespace MimeTreeParser; BodyPartFormatterFactoryPrivate::BodyPartFormatterFactoryPrivate(BodyPartFormatterFactory *factory) : q(factory) { } void BodyPartFormatterFactoryPrivate::setup() { if (registry.empty()) { messageviewer_create_builtin_bodypart_formatters(); q->loadPlugins(); } assert(!registry.empty()); } void BodyPartFormatterFactoryPrivate::insert(const QString &mimeType, const Interface::BodyPartFormatter *formatter, int priority) { if (mimeType.isEmpty() || !formatter) { return; } QMimeDatabase db; const auto mt = db.mimeTypeForName(mimeType); FormatterInfo info; info.formatter = formatter; info.priority = priority; auto &v = registry[mt.isValid() ? mt.name() : mimeType]; v.push_back(info); - std::stable_sort(v.begin(), v.end(), [](const FormatterInfo &lhs, const FormatterInfo &rhs) { + std::stable_sort(v.begin(), v.end(), [](FormatterInfo lhs, FormatterInfo rhs) { return lhs.priority > rhs.priority; }); } void BodyPartFormatterFactoryPrivate::appendFormattersForType(const QString &mimeType, QVector &formatters) { const auto it = registry.constFind(mimeType); if (it == registry.constEnd()) { return; } for (const auto &f : it.value()) { formatters.push_back(f.formatter); } } BodyPartFormatterFactory::BodyPartFormatterFactory() : d(new BodyPartFormatterFactoryPrivate(this)) { } BodyPartFormatterFactory::~BodyPartFormatterFactory() { delete d; } BodyPartFormatterFactory *BodyPartFormatterFactory::instance() { static BodyPartFormatterFactory s_instance; return &s_instance; } void BodyPartFormatterFactory::insert(const QString &mimeType, const Interface::BodyPartFormatter *formatter, int priority) { d->insert(mimeType.toLower(), formatter, priority); } QVector BodyPartFormatterFactory::formattersForType(const QString &mimeType) const { QVector r; d->setup(); QMimeDatabase db; std::vector processedTypes; processedTypes.push_back(mimeType.toLower()); // add all formatters we have along the mimetype hierarchy for (std::size_t i = 0; i < processedTypes.size(); ++i) { const auto mt = db.mimeTypeForName(processedTypes[i]); if (mt.isValid()) { processedTypes[i] = mt.name(); // resolve alias if necessary } if (processedTypes[i] == QLatin1String("application/octet-stream")) { // we'll deal with that later continue; } d->appendFormattersForType(processedTypes[i], r); const auto parentTypes = mt.parentMimeTypes(); for (const auto &parentType : parentTypes) { if (std::find(processedTypes.begin(), processedTypes.end(), parentType) != processedTypes.end()) { continue; } processedTypes.push_back(parentType); } } // make sure we always have a suitable fallback formatter if (mimeType.startsWith(QLatin1String("multipart/"))) { if (mimeType != QLatin1String("multipart/mixed")) { d->appendFormattersForType(QStringLiteral("multipart/mixed"), r); } } else { d->appendFormattersForType(QStringLiteral("application/octet-stream"), r); } assert(!r.empty()); return r; } void BodyPartFormatterFactory::loadPlugins() { KPluginLoader::forEachPlugin(QStringLiteral("messageviewer/bodypartformatter"), [this](const QString &path) { QPluginLoader loader(path); const auto formatterData = loader.metaData().value(QLatin1String("MetaData")).toObject().value(QLatin1String("formatter")).toArray(); if (formatterData.isEmpty()) { return; } auto plugin = qobject_cast(loader.instance()); if (!plugin) { return; } const MimeTreeParser::Interface::BodyPartFormatter *bfp = nullptr; for (int i = 0; (bfp = plugin->bodyPartFormatter(i)) && i < formatterData.size(); ++i) { const auto metaData = formatterData.at(i).toObject(); const auto mimetype = metaData.value(QLatin1String("mimetype")).toString(); if (mimetype.isEmpty()) { qCWarning(MIMETREEPARSER_LOG) << "BodyPartFormatterFactory: plugin" << path << "returned empty mimetype specification for index" << i; break; } // priority should always be higher than the built-in ones, otherwise what's the point? const auto priority = metaData.value(QLatin1String("priority")).toInt() + 100; qCDebug(MIMETREEPARSER_LOG) << "plugin for " << mimetype << priority; insert(mimetype, bfp, priority); } }); } diff --git a/mimetreeparser/src/temporaryfile/autotests/attachmenttemporaryfilesdirstest.cpp b/mimetreeparser/src/temporaryfile/autotests/attachmenttemporaryfilesdirstest.cpp index a8da80d7..d750dfb0 100644 --- a/mimetreeparser/src/temporaryfile/autotests/attachmenttemporaryfilesdirstest.cpp +++ b/mimetreeparser/src/temporaryfile/autotests/attachmenttemporaryfilesdirstest.cpp @@ -1,154 +1,154 @@ /* Copyright (c) 2014-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "attachmenttemporaryfilesdirstest.h" #include "../attachmenttemporaryfilesdirs.h" #include #include #include using namespace MimeTreeParser; AttachmentTemporaryFilesDirsTest::AttachmentTemporaryFilesDirsTest(QObject *parent) : QObject(parent) { } AttachmentTemporaryFilesDirsTest::~AttachmentTemporaryFilesDirsTest() { } void AttachmentTemporaryFilesDirsTest::shouldHaveDefaultValue() { AttachmentTemporaryFilesDirs attachmentDir; QVERIFY(attachmentDir.temporaryFiles().isEmpty()); QVERIFY(attachmentDir.temporaryDirs().isEmpty()); } void AttachmentTemporaryFilesDirsTest::shouldAddTemporaryFiles() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempFile(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryFiles().count(), 1); attachmentDir.addTempFile(QStringLiteral("foo1")); QCOMPARE(attachmentDir.temporaryFiles().count(), 2); } void AttachmentTemporaryFilesDirsTest::shouldAddTemporaryDirs() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempDir(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryDirs().count(), 1); attachmentDir.addTempDir(QStringLiteral("foo1")); QCOMPARE(attachmentDir.temporaryDirs().count(), 2); } void AttachmentTemporaryFilesDirsTest::shouldNotAddSameFiles() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempFile(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryFiles().count(), 1); attachmentDir.addTempFile(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryFiles().count(), 1); } void AttachmentTemporaryFilesDirsTest::shouldNotAddSameDirs() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempDir(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryDirs().count(), 1); attachmentDir.addTempDir(QStringLiteral("foo")); QCOMPARE(attachmentDir.temporaryDirs().count(), 1); } void AttachmentTemporaryFilesDirsTest::shouldForceRemoveTemporaryDirs() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempDir(QStringLiteral("foo")); attachmentDir.addTempDir(QStringLiteral("foo1")); QCOMPARE(attachmentDir.temporaryDirs().count(), 2); attachmentDir.forceCleanTempFiles(); QCOMPARE(attachmentDir.temporaryDirs().count(), 0); QCOMPARE(attachmentDir.temporaryFiles().count(), 0); } void AttachmentTemporaryFilesDirsTest::shouldForceRemoveTemporaryFiles() { AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempFile(QStringLiteral("foo")); attachmentDir.addTempFile(QStringLiteral("foo2")); QCOMPARE(attachmentDir.temporaryFiles().count(), 2); attachmentDir.forceCleanTempFiles(); QCOMPARE(attachmentDir.temporaryFiles().count(), 0); QCOMPARE(attachmentDir.temporaryDirs().count(), 0); } void AttachmentTemporaryFilesDirsTest::shouldCreateDeleteTemporaryFiles() { QTemporaryDir tmpDir; QVERIFY(tmpDir.isValid()); QFile file(tmpDir.path() + QStringLiteral("/foo")); if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { qDebug() << "Can open file"; return; } tmpDir.setAutoRemove(false); file.close(); QVERIFY(file.exists()); AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempDir(tmpDir.path()); attachmentDir.addTempFile(file.fileName()); QVERIFY(!attachmentDir.temporaryFiles().isEmpty()); - QCOMPARE(attachmentDir.temporaryFiles().first(), file.fileName()); + QCOMPARE(attachmentDir.temporaryFiles().constFirst(), file.fileName()); const QString path = tmpDir.path(); attachmentDir.forceCleanTempFiles(); QCOMPARE(attachmentDir.temporaryFiles().count(), 0); QCOMPARE(attachmentDir.temporaryDirs().count(), 0); QVERIFY(!QDir(path).exists()); } void AttachmentTemporaryFilesDirsTest::shouldRemoveTemporaryFilesAfterTime() { QTemporaryDir tmpDir; QVERIFY(tmpDir.isValid()); QFile file(tmpDir.path() + QStringLiteral("/foo")); if (!file.open(QIODevice::WriteOnly | QIODevice::Text)) { qDebug() << "Can open file"; return; } tmpDir.setAutoRemove(false); file.close(); QVERIFY(file.exists()); AttachmentTemporaryFilesDirs attachmentDir; attachmentDir.addTempDir(tmpDir.path()); attachmentDir.addTempFile(file.fileName()); QVERIFY(!attachmentDir.temporaryFiles().isEmpty()); - QCOMPARE(attachmentDir.temporaryFiles().first(), file.fileName()); + QCOMPARE(attachmentDir.temporaryFiles().constFirst(), file.fileName()); attachmentDir.setDelayRemoveAllInMs(500); QTest::qSleep(1000); attachmentDir.removeTempFiles(); const QString path = tmpDir.path(); attachmentDir.forceCleanTempFiles(); QCOMPARE(attachmentDir.temporaryFiles().count(), 0); QCOMPARE(attachmentDir.temporaryDirs().count(), 0); QVERIFY(!QDir(path).exists()); } QTEST_GUILESS_MAIN(AttachmentTemporaryFilesDirsTest) diff --git a/templateparser/src/templateparserjob.cpp b/templateparser/src/templateparserjob.cpp index 1099f4c7..03fcfbba 100644 --- a/templateparser/src/templateparserjob.cpp +++ b/templateparser/src/templateparserjob.cpp @@ -1,1565 +1,1565 @@ /* Copyright (C) 2017-2020 Laurent Montel This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public License for more details. You should have received a copy of the GNU Library General Public License along with this library; see the file COPYING.LIB. If not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include "templateparserjob.h" #include "templateparserjob_p.h" #include "templateparserextracthtmlinfo.h" #include "globalsettings_templateparser.h" #include "customtemplates_kfg.h" #include "templatesconfiguration_kfg.h" #include "templatesconfiguration.h" #include "templatesutil.h" #include "templatesutil_p.h" #include #include #include #include #include #include #include #include #include #include #include #include #include "templateparser_debug.h" #include #include #include #include #include #include namespace { Q_DECL_CONSTEXPR inline int pipeTimeout() { return 15 * 1000; } static QTextCodec *selectCharset(const QStringList &charsets, const QString &text) { for (const QString &name : charsets) { // We use KCharsets::codecForName() instead of QTextCodec::codecForName() here, because // the former knows us-ascii is latin1. bool ok = true; QTextCodec *codec = nullptr; if (name == QLatin1String("locale")) { codec = QTextCodec::codecForLocale(); } else { codec = KCharsets::charsets()->codecForName(name, ok); } if (!ok || !codec) { qCWarning(TEMPLATEPARSER_LOG) << "Could not get text codec for charset" << name; continue; } if (codec->canEncode(text)) { // Special check for us-ascii (needed because us-ascii is not exactly latin1). if (name == QLatin1String("us-ascii") && !KMime::isUsAscii(text)) { continue; } qCDebug(TEMPLATEPARSER_LOG) << "Chosen charset" << name << codec->name(); return codec; } } qCDebug(TEMPLATEPARSER_LOG) << "No appropriate charset found."; return KCharsets::charsets()->codecForName(QStringLiteral("utf-8")); } } using namespace TemplateParser; TemplateParserJobPrivate::TemplateParserJobPrivate(const KMime::Message::Ptr &amsg, const TemplateParserJob::Mode amode) : mMsg(amsg) , mMode(amode) { mEmptySource = new MimeTreeParser::SimpleObjectTreeSource; mEmptySource->setDecryptMessage(mAllowDecryption); mOtp = new MimeTreeParser::ObjectTreeParser(mEmptySource); mOtp->setAllowAsync(false); } TemplateParserJobPrivate::~TemplateParserJobPrivate() { delete mEmptySource; delete mOtp; } void TemplateParserJobPrivate::setAllowDecryption(const bool allowDecryption) { mAllowDecryption = allowDecryption; mEmptySource->setDecryptMessage(mAllowDecryption); } TemplateParserJob::TemplateParserJob(const KMime::Message::Ptr &amsg, const Mode amode, QObject *parent) : QObject(parent) , d(new TemplateParserJobPrivate(amsg, amode)) { } TemplateParserJob::~TemplateParserJob() = default; void TemplateParserJob::setSelection(const QString &selection) { d->mSelection = selection; } void TemplateParserJob::setAllowDecryption(const bool allowDecryption) { d->setAllowDecryption(allowDecryption); } bool TemplateParserJob::shouldStripSignature() const { // Only strip the signature when replying, it should be preserved when forwarding return (d->mMode == Reply || d->mMode == ReplyAll) && TemplateParserSettings::self()->stripSignature(); } void TemplateParserJob::setIdentityManager(KIdentityManagement::IdentityManager *ident) { d->m_identityManager = ident; } void TemplateParserJob::setCharsets(const QStringList &charsets) { d->mCharsets = charsets; } int TemplateParserJob::parseQuotes(const QString &prefix, const QString &str, QString "e) { int pos = prefix.length(); int len; const int str_len = str.length(); // Also allow the german lower double-quote sign as quote separator, not only // the standard ASCII quote ("). This fixes bug 166728. const QList< QChar > quoteChars = {QLatin1Char('"'), 0x201C}; QChar prev(QChar::Null); pos++; len = pos; while (pos < str_len) { const QChar c = str[pos]; pos++; len++; if (!prev.isNull()) { quote.append(c); prev = QChar::Null; } else { if (c == QLatin1Char('\\')) { prev = c; } else if (quoteChars.contains(c)) { break; } else { quote.append(c); } } } return len; } void TemplateParserJob::process(const KMime::Message::Ptr &aorig_msg, qint64 afolder) { if (aorig_msg == nullptr) { qCDebug(TEMPLATEPARSER_LOG) << "aorig_msg == 0!"; Q_EMIT parsingDone(d->mForceCursorPosition); deleteLater(); return; } d->mOrigMsg = aorig_msg; d->mFolder = afolder; const QString tmpl = findTemplate(); if (tmpl.isEmpty()) { Q_EMIT parsingDone(d->mForceCursorPosition); deleteLater(); return; } processWithTemplate(tmpl); } void TemplateParserJob::process(const QString &tmplName, const KMime::Message::Ptr &aorig_msg, qint64 afolder) { d->mForceCursorPosition = false; d->mOrigMsg = aorig_msg; d->mFolder = afolder; const QString tmpl = findCustomTemplate(tmplName); processWithTemplate(tmpl); } void TemplateParserJob::processWithIdentity(uint uoid, const KMime::Message::Ptr &aorig_msg, qint64 afolder) { d->mIdentity = uoid; process(aorig_msg, afolder); } MimeTreeParser::MessagePart::Ptr toplevelTextNode(MimeTreeParser::MessagePart::Ptr messageTree) { foreach (const auto &mp, messageTree->subParts()) { auto text = mp.dynamicCast(); const auto attach = mp.dynamicCast(); if (text && !attach) { // TextMessagePart can have several subparts cause of PGP inline, we search for the first part with content foreach (const auto &sub, mp->subParts()) { if (!sub->text().trimmed().isEmpty()) { return sub; } } return text; } else if (const auto html = mp.dynamicCast()) { return html; } else if (const auto alternative = mp.dynamicCast()) { return alternative; } else { auto ret = toplevelTextNode(mp); if (ret) { return ret; } } } return MimeTreeParser::MessagePart::Ptr(); } void TemplateParserJob::processWithTemplate(const QString &tmpl) { d->mOtp->parseObjectTree(d->mOrigMsg.data()); const auto mp = toplevelTextNode(d->mOtp->parsedPart()); QString plainText = mp->plaintextContent(); QString htmlElement; if (mp->isHtml()) { htmlElement = d->mOtp->htmlContent(); if (plainText.isEmpty()) { //HTML-only mails plainText = htmlElement; } } else { //plain mails only QString htmlReplace = plainText.toHtmlEscaped(); htmlReplace.replace(QLatin1Char('\n'), QStringLiteral("
")); htmlElement = QStringLiteral("%1\n").arg(htmlReplace); } TemplateParserExtractHtmlInfo *job = new TemplateParserExtractHtmlInfo(this); connect(job, &TemplateParserExtractHtmlInfo::finished, this, &TemplateParserJob::slotExtractInfoDone); job->setHtmlForExtractingTextPlain(plainText); job->setTemplate(tmpl); job->setHtmlForExtractionHeaderAndBody(htmlElement); job->start(); } void TemplateParserJob::slotExtractInfoDone(const TemplateParserExtractHtmlInfoResult &result) { d->mExtractHtmlInfoResult = result; const QString tmpl = d->mExtractHtmlInfoResult.mTemplate; const int tmpl_len = tmpl.length(); QString plainBody, htmlBody; bool dnl = false; auto definedLocale = QLocale(); for (int i = 0; i < tmpl_len; ++i) { QChar c = tmpl[i]; // qCDebug(TEMPLATEPARSER_LOG) << "Next char: " << c; if (c == QLatin1Char('%')) { const QString cmd = tmpl.mid(i + 1); if (cmd.startsWith(QLatin1Char('-'))) { // dnl qCDebug(TEMPLATEPARSER_LOG) << "Command: -"; dnl = true; i += 1; } else if (cmd.startsWith(QLatin1String("REM="))) { // comments qCDebug(TEMPLATEPARSER_LOG) << "Command: REM="; QString q; const int len = parseQuotes(QStringLiteral("REM="), cmd, q); i += len; } else if (cmd.startsWith(QLatin1String("LANGUAGE="))) { QString q; const int len = parseQuotes(QStringLiteral("LANGUAGE="), cmd, q); i += len; if (!q.isEmpty()) { definedLocale = QLocale(q); } } else if (cmd.startsWith(QLatin1String("DICTIONARYLANGUAGE="))) { QString q; const int len = parseQuotes(QStringLiteral("DICTIONARYLANGUAGE="), cmd, q); i += len; if (!q.isEmpty()) { KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-Dictionary"); header->fromUnicodeString(q, "utf-8"); d->mMsg->setHeader(header); } } else if (cmd.startsWith(QLatin1String("INSERT=")) || cmd.startsWith(QLatin1String("PUT="))) { QString q; int len = 0; if (cmd.startsWith(QLatin1String("INSERT="))) { // insert content of specified file as is qCDebug(TEMPLATEPARSER_LOG) << "Command: INSERT="; len = parseQuotes(QStringLiteral("INSERT="), cmd, q); } else { // insert content of specified file as is qCDebug(TEMPLATEPARSER_LOG) << "Command: PUT="; len = parseQuotes(QStringLiteral("PUT="), cmd, q); } i += len; QString path = KShell::tildeExpand(q); QFileInfo finfo(path); if (finfo.isRelative()) { path = QDir::homePath() + QLatin1Char('/') + q; } QFile file(path); if (file.open(QIODevice::ReadOnly)) { const QByteArray content = file.readAll(); const QString str = QString::fromLocal8Bit(content.constData(), content.size()); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (d->mDebug) { KMessageBox::error( nullptr, i18nc("@info", "Cannot insert content from file %1: %2", path, file.errorString())); } } else if (cmd.startsWith(QLatin1String("SYSTEM="))) { // insert content of specified file as is qCDebug(TEMPLATEPARSER_LOG) << "Command: SYSTEM="; QString q; const int len = parseQuotes(QStringLiteral("SYSTEM="), cmd, q); i += len; const QString pipe_cmd = q; const QString str = pipe(pipe_cmd, QString()); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("QUOTEPIPE="))) { // pipe message body through command and insert it as quotation qCDebug(TEMPLATEPARSER_LOG) << "Command: QUOTEPIPE="; QString q; const int len = parseQuotes(QStringLiteral("QUOTEPIPE="), cmd, q); i += len; const QString pipe_cmd = q; if (d->mOrigMsg) { const QString plainStr = pipe(pipe_cmd, plainMessageText(shouldStripSignature(), NoSelectionAllowed)); QString plainQuote = quotedPlainText(plainStr); if (plainQuote.endsWith(QLatin1Char('\n'))) { plainQuote.chop(1); } plainBody.append(plainQuote); const QString htmlStr = pipe(pipe_cmd, htmlMessageText(shouldStripSignature(), NoSelectionAllowed)); const QString htmlQuote = quotedHtmlText(htmlStr); htmlBody.append(htmlQuote); } } else if (cmd.startsWith(QLatin1String("QUOTE"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: QUOTE"; i += strlen("QUOTE"); if (d->mOrigMsg) { QString plainQuote = quotedPlainText(plainMessageText(shouldStripSignature(), SelectionAllowed)); if (plainQuote.endsWith(QLatin1Char('\n'))) { plainQuote.chop(1); } plainBody.append(plainQuote); const QString htmlQuote = quotedHtmlText(htmlMessageText(shouldStripSignature(), SelectionAllowed)); htmlBody.append(htmlQuote); } } else if (cmd.startsWith(QLatin1String("FORCEDPLAIN"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FORCEDPLAIN"; d->mQuotes = ReplyAsPlain; i += strlen("FORCEDPLAIN"); } else if (cmd.startsWith(QLatin1String("FORCEDHTML"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FORCEDHTML"; d->mQuotes = ReplyAsHtml; i += strlen("FORCEDHTML"); } else if (cmd.startsWith(QLatin1String("QHEADERS"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: QHEADERS"; i += strlen("QHEADERS"); if (d->mOrigMsg) { QString plainQuote = quotedPlainText(QString::fromLatin1(MessageCore::StringUtil::headerAsSendableString(d->mOrigMsg))); if (plainQuote.endsWith(QLatin1Char('\n'))) { plainQuote.chop(1); } plainBody.append(plainQuote); const QString htmlQuote = quotedHtmlText(QString::fromLatin1(MessageCore::StringUtil::headerAsSendableString(d->mOrigMsg))); const QString str = plainTextToHtml(htmlQuote); htmlBody.append(str); } } else if (cmd.startsWith(QLatin1String("HEADERS"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: HEADERS"; i += strlen("HEADERS"); if (d->mOrigMsg) { const QString str = QString::fromLatin1(MessageCore::StringUtil::headerAsSendableString(d->mOrigMsg)); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("TEXTPIPE="))) { // pipe message body through command and insert it as is qCDebug(TEMPLATEPARSER_LOG) << "Command: TEXTPIPE="; QString q; const int len = parseQuotes(QStringLiteral("TEXTPIPE="), cmd, q); i += len; const QString pipe_cmd = q; if (d->mOrigMsg) { const QString plainStr = pipe(pipe_cmd, plainMessageText(shouldStripSignature(), NoSelectionAllowed)); plainBody.append(plainStr); const QString htmlStr = pipe(pipe_cmd, htmlMessageText(shouldStripSignature(), NoSelectionAllowed)); htmlBody.append(htmlStr); } } else if (cmd.startsWith(QLatin1String("MSGPIPE="))) { // pipe full message through command and insert result as is qCDebug(TEMPLATEPARSER_LOG) << "Command: MSGPIPE="; QString q; const int len = parseQuotes(QStringLiteral("MSGPIPE="), cmd, q); i += len; if (d->mOrigMsg) { QString pipe_cmd = q; const QString str = pipe(pipe_cmd, QString::fromLatin1(d->mOrigMsg->encodedContent())); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("BODYPIPE="))) { // pipe message body generated so far through command and insert result as is qCDebug(TEMPLATEPARSER_LOG) << "Command: BODYPIPE="; QString q; const int len = parseQuotes(QStringLiteral("BODYPIPE="), cmd, q); i += len; const QString pipe_cmd = q; const QString plainStr = pipe(pipe_cmd, plainBody); plainBody.append(plainStr); const QString htmlStr = pipe(pipe_cmd, htmlBody); const QString body = plainTextToHtml(htmlStr); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("CLEARPIPE="))) { // pipe message body generated so far through command and // insert result as is replacing current body qCDebug(TEMPLATEPARSER_LOG) << "Command: CLEARPIPE="; QString q; const int len = parseQuotes(QStringLiteral("CLEARPIPE="), cmd, q); i += len; const QString pipe_cmd = q; const QString plainStr = pipe(pipe_cmd, plainBody); plainBody = plainStr; const QString htmlStr = pipe(pipe_cmd, htmlBody); htmlBody = htmlStr; KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-CursorPos"); header->fromUnicodeString(QString::number(0), "utf-8"); d->mMsg->setHeader(header); } else if (cmd.startsWith(QLatin1String("TEXT"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TEXT"; i += strlen("TEXT"); if (d->mOrigMsg) { const QString plainStr = plainMessageText(shouldStripSignature(), NoSelectionAllowed); plainBody.append(plainStr); const QString htmlStr = htmlMessageText(shouldStripSignature(), NoSelectionAllowed); htmlBody.append(htmlStr); } } else if (cmd.startsWith(QLatin1String("OTEXTSIZE"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTEXTSIZE"; i += strlen("OTEXTSIZE"); if (d->mOrigMsg) { const QString str = QStringLiteral("%1").arg(d->mOrigMsg->body().length()); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTEXT"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTEXT"; i += strlen("OTEXT"); if (d->mOrigMsg) { const QString plainStr = plainMessageText(shouldStripSignature(), NoSelectionAllowed); plainBody.append(plainStr); const QString htmlStr = htmlMessageText(shouldStripSignature(), NoSelectionAllowed); htmlBody.append(htmlStr); } } else if (cmd.startsWith(QLatin1String("OADDRESSEESADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OADDRESSEESADDR"; i += strlen("OADDRESSEESADDR"); if (d->mOrigMsg) { const QString to = d->mOrigMsg->to()->asUnicodeString(); const QString cc = d->mOrigMsg->cc()->asUnicodeString(); if (!to.isEmpty()) { const QString toLine = i18nc("@item:intext email To", "To:") + QLatin1Char(' ') + to; plainBody.append(toLine); const QString body = plainTextToHtml(toLine); htmlBody.append(body); } if (!to.isEmpty() && !cc.isEmpty()) { plainBody.append(QLatin1Char('\n')); const QString str = plainTextToHtml(QString(QLatin1Char('\n'))); htmlBody.append(str); } if (!cc.isEmpty()) { const QString ccLine = i18nc("@item:intext email CC", "CC:") + QLatin1Char(' ') + cc; plainBody.append(ccLine); const QString str = plainTextToHtml(ccLine); htmlBody.append(str); } } } else if (cmd.startsWith(QLatin1String("CCADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: CCADDR"; i += strlen("CCADDR"); const QString str = d->mMsg->cc()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("CCNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: CCNAME"; i += strlen("CCNAME"); const QString str = d->mMsg->cc()->displayString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("CCFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: CCFNAME"; i += strlen("CCFNAME"); const QString str = d->mMsg->cc()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("CCLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: CCLNAME"; i += strlen("CCLNAME"); const QString str = d->mMsg->cc()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TOADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TOADDR"; i += strlen("TOADDR"); const QString str = d->mMsg->to()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TONAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TONAME"; i += strlen("TONAME"); const QString str = (d->mMsg->to()->displayString()); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TOFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TOFNAME"; i += strlen("TOFNAME"); const QString str = d->mMsg->to()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TOLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TOLNAME"; i += strlen("TOLNAME"); const QString str = d->mMsg->to()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TOLIST"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TOLIST"; i += strlen("TOLIST"); const QString str = d->mMsg->to()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("FROMADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FROMADDR"; i += strlen("FROMADDR"); const QString str = d->mMsg->from()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("FROMNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FROMNAME"; i += strlen("FROMNAME"); const QString str = d->mMsg->from()->displayString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("FROMFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FROMFNAME"; i += strlen("FROMFNAME"); const QString str = d->mMsg->from()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("FROMLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FROMLNAME"; i += strlen("FROMLNAME"); const QString str = d->mMsg->from()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("FULLSUBJECT")) || cmd.startsWith(QLatin1String("FULLSUBJ"))) { if (cmd.startsWith(QLatin1String("FULLSUBJ"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: FULLSUBJ"; i += strlen("FULLSUBJ"); } else { qCDebug(TEMPLATEPARSER_LOG) << "Command: FULLSUBJECT"; i += strlen("FULLSUBJECT"); } const QString str = d->mMsg->subject()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("MSGID"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: MSGID"; i += strlen("MSGID"); const QString str = d->mMsg->messageID()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("OHEADER="))) { // insert specified content of header from original message qCDebug(TEMPLATEPARSER_LOG) << "Command: OHEADER="; QString q; const int len = parseQuotes(QStringLiteral("OHEADER="), cmd, q); i += len; if (d->mOrigMsg) { const QString hdr = q; QString str; if (auto hrdMsgOrigin = d->mOrigMsg->headerByType(hdr.toLocal8Bit().constData())) { str = hrdMsgOrigin->asUnicodeString(); } plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("HEADER="))) { // insert specified content of header from current message qCDebug(TEMPLATEPARSER_LOG) << "Command: HEADER="; QString q; const int len = parseQuotes(QStringLiteral("HEADER="), cmd, q); i += len; const QString hdr = q; QString str; if (auto hrdMsgOrigin = d->mOrigMsg->headerByType(hdr.toLocal8Bit().constData())) { str = hrdMsgOrigin->asUnicodeString(); } plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("HEADER( "))) { // insert specified content of header from current message qCDebug(TEMPLATEPARSER_LOG) << "Command: HEADER("; QRegularExpressionMatch match; - const int res = cmd.indexOf(QRegularExpression(QLatin1String("^HEADER\\((.+)\\)")), 0, &match); + const int res = cmd.indexOf(QRegularExpression(QStringLiteral("^HEADER\\((.+)\\)")), 0, &match); if (res != 0) { // something wrong i += strlen("HEADER( "); } else { i += match.capturedLength(0); //length of HEADER( + ) const QString hdr = match.captured(1).trimmed(); QString str; if (auto hrdMsgOrigin = d->mOrigMsg->headerByType(hdr.toLocal8Bit().constData())) { str = hrdMsgOrigin->asUnicodeString(); } plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OCCADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OCCADDR"; i += strlen("OCCADDR"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->cc()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OCCNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OCCNAME"; i += strlen("OCCNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->cc()->displayString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OCCFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OCCFNAME"; i += strlen("OCCFNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->cc()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OCCLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OCCLNAME"; i += strlen("OCCLNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->cc()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTOADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTOADDR"; i += strlen("OTOADDR"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTONAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTONAME"; i += strlen("OTONAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->displayString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTOFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTOFNAME"; i += strlen("OTOFNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTOLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTOLNAME"; i += strlen("OTOLNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTOLIST"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTOLIST"; i += strlen("OTOLIST"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTO"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTO"; i += strlen("OTO"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->to()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OFROMADDR"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFROMADDR"; i += strlen("OFROMADDR"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->from()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OFROMNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFROMNAME"; i += strlen("OFROMNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->from()->displayString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OFROMFNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFROMFNAME"; i += strlen("OFROMFNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->from()->displayString(); const QString firstNameFromEmail = TemplateParser::Util::getFirstNameFromEmail(str); plainBody.append(firstNameFromEmail); const QString body = plainTextToHtml(firstNameFromEmail); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OFROMLNAME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFROMLNAME"; i += strlen("OFROMLNAME"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->from()->displayString(); plainBody.append(TemplateParser::Util::getLastNameFromEmail(str)); const QString body = plainTextToHtml(TemplateParser::Util::getLastNameFromEmail(str)); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OFULLSUBJECT")) || cmd.startsWith(QLatin1String("OFULLSUBJ"))) { if (cmd.startsWith(QLatin1String("OFULLSUBJECT"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFULLSUBJECT"; i += strlen("OFULLSUBJECT"); } else { qCDebug(TEMPLATEPARSER_LOG) << "Command: OFULLSUBJ"; i += strlen("OFULLSUBJ"); } if (d->mOrigMsg) { const QString str = d->mOrigMsg->subject()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OMSGID"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OMSGID"; i += strlen("OMSGID"); if (d->mOrigMsg) { const QString str = d->mOrigMsg->messageID()->asUnicodeString(); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("DATEEN"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: DATEEN"; i += strlen("DATEEN"); const QDateTime date = QDateTime::currentDateTime(); QLocale locale(QLocale::C); const QString str = locale.toString(date.date(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("DATESHORT"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: DATESHORT"; i += strlen("DATESHORT"); const QDateTime date = QDateTime::currentDateTime(); const QString str = definedLocale.toString(date.date(), QLocale::ShortFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("DATE"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: DATE"; i += strlen("DATE"); const QDateTime date = QDateTime::currentDateTime(); const QString str = definedLocale.toString(date.date(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("DOW"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: DOW"; i += strlen("DOW"); const QDateTime date = QDateTime::currentDateTime(); const QString str = definedLocale.dayName(date.date().dayOfWeek(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TIMELONGEN"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TIMELONGEN"; i += strlen("TIMELONGEN"); const QDateTime date = QDateTime::currentDateTime(); QLocale locale(QLocale::C); const QString str = locale.toString(date.time(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TIMELONG"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TIMELONG"; i += strlen("TIMELONG"); const QDateTime date = QDateTime::currentDateTime(); const QString str = definedLocale.toString(date.time(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("TIME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: TIME"; i += strlen("TIME"); const QDateTime date = QDateTime::currentDateTime(); const QString str = definedLocale.toString(date.time(), QLocale::ShortFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } else if (cmd.startsWith(QLatin1String("ODATEEN"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: ODATEEN"; i += strlen("ODATEEN"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = QLocale::c().toString(date.date(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("ODATESHORT"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: ODATESHORT"; i += strlen("ODATESHORT"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = definedLocale.toString(date.date(), QLocale::ShortFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("ODATE"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: ODATE"; i += strlen("ODATE"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = definedLocale.toString(date.date(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("ODOW"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: ODOW"; i += strlen("ODOW"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = definedLocale.dayName(date.date().dayOfWeek(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTIMELONGEN"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTIMELONGEN"; i += strlen("OTIMELONGEN"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); QLocale locale(QLocale::C); const QString str = locale.toString(date.time(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTIMELONG"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTIMELONG"; i += strlen("OTIMELONG"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = definedLocale.toString(date.time(), QLocale::LongFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("OTIME"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: OTIME"; i += strlen("OTIME"); if (d->mOrigMsg) { const QDateTime date = d->mOrigMsg->date()->dateTime().toLocalTime(); const QString str = definedLocale.toString(date.time(), QLocale::ShortFormat); plainBody.append(str); const QString body = plainTextToHtml(str); htmlBody.append(body); } } else if (cmd.startsWith(QLatin1String("BLANK"))) { // do nothing qCDebug(TEMPLATEPARSER_LOG) << "Command: BLANK"; i += strlen("BLANK"); } else if (cmd.startsWith(QLatin1String("NOP"))) { // do nothing qCDebug(TEMPLATEPARSER_LOG) << "Command: NOP"; i += strlen("NOP"); } else if (cmd.startsWith(QLatin1String("CLEAR"))) { // clear body buffer; not too useful yet qCDebug(TEMPLATEPARSER_LOG) << "Command: CLEAR"; i += strlen("CLEAR"); plainBody.clear(); htmlBody.clear(); KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-CursorPos"); header->fromUnicodeString(QString::number(0), "utf-8"); d->mMsg->setHeader(header); } else if (cmd.startsWith(QLatin1String("DEBUGOFF"))) { // turn off debug qCDebug(TEMPLATEPARSER_LOG) << "Command: DEBUGOFF"; i += strlen("DEBUGOFF"); d->mDebug = false; } else if (cmd.startsWith(QLatin1String("DEBUG"))) { // turn on debug qCDebug(TEMPLATEPARSER_LOG) << "Command: DEBUG"; i += strlen("DEBUG"); d->mDebug = true; } else if (cmd.startsWith(QLatin1String("CURSOR"))) { // turn on debug qCDebug(TEMPLATEPARSER_LOG) << "Command: CURSOR"; int oldI = i; i += strlen("CURSOR"); KMime::Headers::Generic *header = new KMime::Headers::Generic("X-KMail-CursorPos"); header->fromUnicodeString(QString::number(plainBody.length()), "utf-8"); /* if template is: * FOOBAR * %CURSOR * * Make sure there is an empty line for the cursor otherwise it will be placed at the end of FOOBAR */ if (oldI > 0 && tmpl[ oldI - 1 ] == QLatin1Char('\n') && i == tmpl_len - 1) { plainBody.append(QLatin1Char('\n')); } d->mMsg->setHeader(header); d->mForceCursorPosition = true; //FIXME HTML part for header remaining } else if (cmd.startsWith(QLatin1String("SIGNATURE"))) { qCDebug(TEMPLATEPARSER_LOG) << "Command: SIGNATURE"; i += strlen("SIGNATURE"); plainBody.append(getPlainSignature()); htmlBody.append(getHtmlSignature()); } else { // wrong command, do nothing plainBody.append(c); htmlBody.append(c); } } else if (dnl && (c == QLatin1Char('\n') || c == QLatin1Char('\r'))) { // skip if ((tmpl.size() > i + 1) && ((c == QLatin1Char('\n') && tmpl[i + 1] == QLatin1Char('\r')) || (c == QLatin1Char('\r') && tmpl[i + 1] == QLatin1Char('\n')))) { // skip one more i += 1; } dnl = false; } else { plainBody.append(c); if (c == QLatin1Char('\n') || c == QLatin1Char('\r')) { htmlBody.append(QLatin1String("
")); htmlBody.append(c); if (tmpl.size() > i + 1 && ((c == QLatin1Char('\n') && tmpl[i + 1] == QLatin1Char('\r')) || (c == QLatin1Char('\r') && tmpl[i + 1] == QLatin1Char('\n')))) { htmlBody.append(tmpl[i + 1]); plainBody.append(tmpl[i + 1]); i += 1; } } else { htmlBody.append(c); } } } // Clear the HTML body if FORCEDPLAIN has set ReplyAsPlain, OR if, // there is no use of FORCED command but a configure setting has ReplyUsingHtml disabled, // OR the original mail has no HTML part. const KMime::Content *content = d->mOrigMsg->mainBodyPart("text/html"); if (d->mQuotes == ReplyAsPlain || (d->mQuotes != ReplyAsHtml && !TemplateParserSettings::self()->replyUsingHtml()) || (!content || !content->hasContent())) { htmlBody.clear(); } else { makeValidHtml(htmlBody); } if (d->mMode == NewMessage && plainBody.isEmpty() && !d->mExtractHtmlInfoResult.mPlainText.isEmpty()) { plainBody = d->mExtractHtmlInfoResult.mPlainText; } /* if (d->mMode == NewMessage && htmlBody.isEmpty() && !d->mExtractHtmlInfoResult.mHtmlElement.isEmpty()) { htmlBody = d->mExtractHtmlInfoResult.mHtmlElement; } */ addProcessedBodyToMessage(plainBody, htmlBody); Q_EMIT parsingDone(d->mForceCursorPosition); deleteLater(); } QString TemplateParserJob::getPlainSignature() const { const KIdentityManagement::Identity &identity = d->m_identityManager->identityForUoid(d->mIdentity); if (identity.isNull()) { return QString(); } KIdentityManagement::Signature signature = const_cast(identity).signature(); if (signature.type() == KIdentityManagement::Signature::Inlined && signature.isInlinedHtml()) { return signature.toPlainText(); } else { return signature.rawText(); } } // TODO If %SIGNATURE command is on, then override it with signature from // "KMail configure->General->identity->signature". // There should be no two signatures. QString TemplateParserJob::getHtmlSignature() const { const KIdentityManagement::Identity &identity = d->m_identityManager->identityForUoid(d->mIdentity); if (identity.isNull()) { return QString(); } KIdentityManagement::Signature signature = const_cast(identity).signature(); if (!signature.isInlinedHtml()) { signature = signature.rawText().toHtmlEscaped(); return signature.rawText().replace(QLatin1Char('\n'), QStringLiteral("
")); } return signature.rawText(); } void TemplateParserJob::addProcessedBodyToMessage(const QString &plainBody, const QString &htmlBody) const { MessageCore::ImageCollector ic; ic.collectImagesFrom(d->mOrigMsg.data()); // Now, delete the old content and set the new content, which // is either only the new text or the new text with some attachments. const auto parts = d->mMsg->contents(); for (KMime::Content *content : parts) { d->mMsg->removeContent(content, true /*delete*/); } // Set To and CC from the template if (!d->mTo.isEmpty()) { d->mMsg->to()->fromUnicodeString(d->mMsg->to()->asUnicodeString() + QLatin1Char(',') + d->mTo, "utf-8"); } if (!d->mCC.isEmpty()) { d->mMsg->cc()->fromUnicodeString(d->mMsg->cc()->asUnicodeString() + QLatin1Char(',') + d->mCC, "utf-8"); } d->mMsg->contentType()->clear(); // to get rid of old boundary //const QByteArray boundary = KMime::multiPartBoundary(); KMime::Content *const mainTextPart = htmlBody.isEmpty() ? createPlainPartContent(plainBody) : createMultipartAlternativeContent(plainBody, htmlBody); mainTextPart->assemble(); KMime::Content *textPart = mainTextPart; if (!ic.images().empty()) { textPart = createMultipartRelated(ic, mainTextPart); textPart->assemble(); } // If we have some attachments, create a multipart/mixed mail and // add the normal body as well as the attachments KMime::Content *mainPart = textPart; if (d->mMode == Forward) { auto attachments = d->mOrigMsg->attachments(); attachments += d->mOtp->nodeHelper()->attachmentsOfExtraContents(); if (!attachments.isEmpty()) { mainPart = createMultipartMixed(attachments, textPart); mainPart->assemble(); } } d->mMsg->setBody(mainPart->encodedBody()); d->mMsg->setHeader(mainPart->contentType()); d->mMsg->setHeader(mainPart->contentTransferEncoding()); d->mMsg->assemble(); d->mMsg->parse(); } KMime::Content *TemplateParserJob::createMultipartMixed(const QVector &attachments, KMime::Content *textPart) const { KMime::Content *mixedPart = new KMime::Content(d->mMsg.data()); const QByteArray boundary = KMime::multiPartBoundary(); mixedPart->contentType()->setMimeType("multipart/mixed"); mixedPart->contentType()->setBoundary(boundary); mixedPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE7Bit); mixedPart->addContent(textPart); int attachmentNumber = 1; for (KMime::Content *attachment : attachments) { mixedPart->addContent(attachment); // If the content type has no name or filename parameter, add one, since otherwise the name // would be empty in the attachment view of the composer, which looks confusing if (attachment->contentType(false)) { if (!attachment->contentType()->hasParameter(QStringLiteral("name")) && !attachment->contentType()->hasParameter(QStringLiteral("filename"))) { attachment->contentType()->setParameter( QStringLiteral("name"), i18nc("@item:intext", "Attachment %1", attachmentNumber)); } } ++attachmentNumber; } return mixedPart; } KMime::Content *TemplateParserJob::createMultipartRelated(const MessageCore::ImageCollector &ic, KMime::Content *mainTextPart) const { KMime::Content *relatedPart = new KMime::Content(d->mMsg.data()); const QByteArray boundary = KMime::multiPartBoundary(); relatedPart->contentType()->setMimeType("multipart/related"); relatedPart->contentType()->setBoundary(boundary); relatedPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE7Bit); relatedPart->addContent(mainTextPart); for (KMime::Content *image : ic.images()) { qCWarning(TEMPLATEPARSER_LOG) << "Adding" << image->contentID() << "as an embedded image"; relatedPart->addContent(image); } return relatedPart; } KMime::Content *TemplateParserJob::createPlainPartContent(const QString &plainBody) const { KMime::Content *textPart = new KMime::Content(d->mMsg.data()); textPart->contentType()->setMimeType("text/plain"); QTextCodec *charset = selectCharset(d->mCharsets, plainBody); textPart->contentType()->setCharset(charset->name()); textPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE8Bit); textPart->fromUnicodeString(plainBody); return textPart; } KMime::Content *TemplateParserJob::createMultipartAlternativeContent(const QString &plainBody, const QString &htmlBody) const { KMime::Content *multipartAlternative = new KMime::Content(d->mMsg.data()); multipartAlternative->contentType()->setMimeType("multipart/alternative"); const QByteArray boundary = KMime::multiPartBoundary(); multipartAlternative->contentType()->setBoundary(boundary); KMime::Content *textPart = createPlainPartContent(plainBody); multipartAlternative->addContent(textPart); KMime::Content *htmlPart = new KMime::Content(d->mMsg.data()); htmlPart->contentType()->setMimeType("text/html"); QTextCodec *charset = selectCharset(d->mCharsets, htmlBody); htmlPart->contentType()->setCharset(charset->name()); htmlPart->contentTransferEncoding()->setEncoding(KMime::Headers::CE8Bit); htmlPart->fromUnicodeString(htmlBody); multipartAlternative->addContent(htmlPart); return multipartAlternative; } QString TemplateParserJob::findCustomTemplate(const QString &tmplName) { CTemplates t(tmplName); d->mTo = t.to(); d->mCC = t.cC(); const QString content = t.content(); if (!content.isEmpty()) { return content; } else { return findTemplate(); } } QString TemplateParserJob::findTemplate() { // qCDebug(TEMPLATEPARSER_LOG) << "Trying to find template for mode" << mode; QString tmpl; qCDebug(TEMPLATEPARSER_LOG) << "Folder identify found:" << d->mFolder; if (d->mFolder >= 0) { // only if a folder was found QString fid = QString::number(d->mFolder); Templates fconf(fid); if (fconf.useCustomTemplates()) { // does folder use custom templates? switch (d->mMode) { case NewMessage: tmpl = fconf.templateNewMessage(); break; case Reply: tmpl = fconf.templateReply(); break; case ReplyAll: tmpl = fconf.templateReplyAll(); break; case Forward: tmpl = fconf.templateForward(); break; default: qCDebug(TEMPLATEPARSER_LOG) << "Unknown message mode:" << d->mMode; return QString(); } d->mQuoteString = fconf.quoteString(); if (!tmpl.isEmpty()) { return tmpl; // use folder-specific template } } } if (!d->mIdentity) { // find identity message belongs to d->mIdentity = identityUoid(d->mMsg); if (!d->mIdentity && d->mOrigMsg) { d->mIdentity = identityUoid(d->mOrigMsg); } d->mIdentity = d->m_identityManager->identityForUoidOrDefault(d->mIdentity).uoid(); if (!d->mIdentity) { qCDebug(TEMPLATEPARSER_LOG) << "Oops! No identity for message"; } } qCDebug(TEMPLATEPARSER_LOG) << "Identity found:" << d->mIdentity; QString iid; if (d->mIdentity) { iid = TemplatesConfiguration::configIdString(d->mIdentity); // templates ID for that identity } else { iid = QStringLiteral("IDENTITY_NO_IDENTITY"); // templates ID for no identity } Templates iconf(iid); if (iconf.useCustomTemplates()) { // does identity use custom templates? switch (d->mMode) { case NewMessage: tmpl = iconf.templateNewMessage(); break; case Reply: tmpl = iconf.templateReply(); break; case ReplyAll: tmpl = iconf.templateReplyAll(); break; case Forward: tmpl = iconf.templateForward(); break; default: qCDebug(TEMPLATEPARSER_LOG) << "Unknown message mode:" << d->mMode; return QString(); } d->mQuoteString = iconf.quoteString(); if (!tmpl.isEmpty()) { return tmpl; // use identity-specific template } } switch (d->mMode) { // use the global template case NewMessage: tmpl = TemplateParserSettings::self()->templateNewMessage(); break; case Reply: tmpl = TemplateParserSettings::self()->templateReply(); break; case ReplyAll: tmpl = TemplateParserSettings::self()->templateReplyAll(); break; case Forward: tmpl = TemplateParserSettings::self()->templateForward(); break; default: qCDebug(TEMPLATEPARSER_LOG) << "Unknown message mode:" << d->mMode; return QString(); } d->mQuoteString = TemplateParserSettings::self()->quoteString(); return tmpl; } QString TemplateParserJob::pipe(const QString &cmd, const QString &buf) { KProcess process; bool success; process.setOutputChannelMode(KProcess::SeparateChannels); process.setShellCommand(cmd); process.start(); if (process.waitForStarted(pipeTimeout())) { bool finished = false; if (!buf.isEmpty()) { process.write(buf.toLatin1()); } if (buf.isEmpty() || process.waitForBytesWritten(pipeTimeout())) { if (!buf.isEmpty()) { process.closeWriteChannel(); } if (process.waitForFinished(pipeTimeout())) { success = (process.exitStatus() == QProcess::NormalExit); finished = true; } else { finished = false; success = false; } } else { success = false; finished = false; } // The process has started, but did not finish in time. Kill it. if (!finished) { process.kill(); } } else { success = false; } if (!success && d->mDebug) { KMessageBox::error( nullptr, xi18nc("@info", "Pipe command %1 failed.", cmd)); } if (success) { return QTextCodec::codecForLocale()->toUnicode(process.readAllStandardOutput()); } else { return QString(); } } void TemplateParserJob::setWordWrap(bool wrap, int wrapColWidth) { d->mWrap = wrap; d->mColWrap = wrapColWidth; } QString TemplateParserJob::plainMessageText(bool aStripSignature, AllowSelection isSelectionAllowed) const { if (!d->mSelection.isEmpty() && (isSelectionAllowed == SelectionAllowed)) { return d->mSelection; } if (!d->mOrigMsg) { return QString(); } const auto mp = toplevelTextNode(d->mOtp->parsedPart()); QString result = mp->plaintextContent(); if (result.isEmpty()) { result = d->mExtractHtmlInfoResult.mPlainText; } if (aStripSignature) { result = MessageCore::StringUtil::stripSignature(result); } return result; } QString TemplateParserJob::htmlMessageText(bool aStripSignature, AllowSelection isSelectionAllowed) { if (!d->mSelection.isEmpty() && (isSelectionAllowed == SelectionAllowed)) { //TODO implement d->mSelection for HTML return d->mSelection; } d->mHeadElement = d->mExtractHtmlInfoResult.mHeaderElement; const QString bodyElement = d->mExtractHtmlInfoResult.mBodyElement; if (!bodyElement.isEmpty()) { if (aStripSignature) { //FIXME strip signature works partially for HTML mails return MessageCore::StringUtil::stripSignature(bodyElement); } return bodyElement; } if (aStripSignature) { //FIXME strip signature works partially for HTML mails return MessageCore::StringUtil::stripSignature(d->mExtractHtmlInfoResult.mHtmlElement); } return d->mExtractHtmlInfoResult.mHtmlElement; } QString TemplateParserJob::quotedPlainText(const QString &selection) const { QString content = TemplateParser::Util::removeSpaceAtBegin(selection); const QString indentStr = MessageCore::StringUtil::formatQuotePrefix(d->mQuoteString, d->mOrigMsg->from()->displayString()); if (TemplateParserSettings::self()->smartQuote() && d->mWrap) { content = MessageCore::StringUtil::smartQuote(content, d->mColWrap - indentStr.length()); } content.replace(QLatin1Char('\n'), QLatin1Char('\n') + indentStr); content.prepend(indentStr); content += QLatin1Char('\n'); return content; } QString TemplateParserJob::quotedHtmlText(const QString &selection) const { QString content = selection; //TODO 1) look for all the variations of
and remove the blank lines //2) implement vertical bar for quoted HTML mail. //3) After vertical bar is implemented, If a user wants to edit quoted message, // then the
tags below should open and close as when required. //Add blockquote tag, so that quoted message can be differentiated from normal message content = QLatin1String("
") + content + QLatin1String("
"); return content; } uint TemplateParserJob::identityUoid(const KMime::Message::Ptr &msg) const { QString idString; if (auto hrd = msg->headerByType("X-KMail-Identity")) { idString = hrd->asUnicodeString().trimmed(); } bool ok = false; unsigned int id = idString.toUInt(&ok); if (!ok || id == 0) { id = d->m_identityManager->identityForAddress( msg->to()->asUnicodeString() + QLatin1String(", ") + msg->cc()->asUnicodeString()).uoid(); } return id; } bool TemplateParserJob::isHtmlSignature() const { const KIdentityManagement::Identity &identity = d->m_identityManager->identityForUoid(d->mIdentity); if (identity.isNull()) { return false; } const KIdentityManagement::Signature signature = const_cast(identity).signature(); return signature.isInlinedHtml(); } QString TemplateParserJob::plainTextToHtml(const QString &body) { QString str = body; str = str.toHtmlEscaped(); str.replace(QLatin1Char('\n'), QStringLiteral("
\n")); return str; } void TemplateParserJob::makeValidHtml(QString &body) { QRegExp regEx; regEx.setMinimal(true); regEx.setPattern(QStringLiteral("")); if (!body.isEmpty() && !body.contains(regEx)) { regEx.setPattern(QStringLiteral("")); if (!body.contains(regEx)) { body = QLatin1String("") + body + QLatin1String("
"); } regEx.setPattern(QStringLiteral("")); if (!body.contains(regEx)) { body = QLatin1String("") + d->mHeadElement + QLatin1String("") + body; } body = QLatin1String("") + body + QLatin1String(""); } } diff --git a/templateparser/src/templatesutil.cpp b/templateparser/src/templatesutil.cpp index c9f3b085..6569bee3 100644 --- a/templateparser/src/templatesutil.cpp +++ b/templateparser/src/templatesutil.cpp @@ -1,219 +1,219 @@ /* Copyright (c) 2011-2020 Laurent Montel This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "templatesutil.h" #include "templatesutil_p.h" #include #include #include #include using namespace TemplateParser; void TemplateParser::Util::deleteTemplate(const QString &id) { KSharedConfig::Ptr config = KSharedConfig::openConfig(QStringLiteral("templatesconfigurationrc"), KConfig::NoGlobals); const QString key = QStringLiteral("Templates #%1").arg(id); if (config->hasGroup(key)) { KConfigGroup group = config->group(key); group.deleteGroup(); group.sync(); } } QStringList TemplateParser::Util::keywordsWithArgs() { const QStringList keywordsWithArgs = QStringList() << QStringLiteral("%REM=\"\"%-") << QStringLiteral("%INSERT=\"\"") << QStringLiteral("%SYSTEM=\"\"") << QStringLiteral("%QUOTEPIPE=\"\"") << QStringLiteral("%MSGPIPE=\"\"") << QStringLiteral("%BODYPIPE=\"\"") << QStringLiteral("%CLEARPIPE=\"\"") << QStringLiteral("%TEXTPIPE=\"\"") << QStringLiteral("%OHEADER=\"\"") << QStringLiteral("%HEADER=\"\"") << QStringLiteral("%DICTIONARYLANGUAGE=\"\"") << QStringLiteral("%LANGUAGE=\"\""); return keywordsWithArgs; } QStringList TemplateParser::Util::keywords() { const QStringList keywords = QStringList() << QStringLiteral("%QUOTE") << QStringLiteral("%FORCEDPLAIN") << QStringLiteral("%FORCEDHTML") << QStringLiteral("%QHEADERS") << QStringLiteral("%HEADERS") << QStringLiteral("%TEXT") << QStringLiteral("%OTEXTSIZE") << QStringLiteral("%OTEXT") << QStringLiteral("%OADDRESSEESADDR") << QStringLiteral("%CCADDR") << QStringLiteral("%CCNAME") << QStringLiteral("%CCFNAME") << QStringLiteral("%CCLNAME") << QStringLiteral("%TOADDR") << QStringLiteral("%TONAME") << QStringLiteral("%TOFNAME") << QStringLiteral("%TOLNAME") << QStringLiteral("%TOLIST") << QStringLiteral("%FROMADDR") << QStringLiteral("%FROMNAME") << QStringLiteral("%FROMFNAME") << QStringLiteral("%FROMLNAME") << QStringLiteral("%FULLSUBJECT") << QStringLiteral("%MSGID") << QStringLiteral("%HEADER\\( ") << QStringLiteral("%OCCADDR") << QStringLiteral("%OCCNAME") << QStringLiteral("%OCCFNAME") << QStringLiteral("%OCCLNAME") << QStringLiteral("%OTOADDR") << QStringLiteral("%OTONAME") << QStringLiteral("%OTOFNAME") << QStringLiteral("%OTOLNAME") << QStringLiteral("%OTOLIST") << QStringLiteral("%OTO") << QStringLiteral("%OFROMADDR") << QStringLiteral("%OFROMNAME") << QStringLiteral("%OFROMFNAME") << QStringLiteral("%OFROMLNAME") << QStringLiteral("%OFULLSUBJECT") << QStringLiteral("%OFULLSUBJ") << QStringLiteral("%OMSGID") << QStringLiteral("%DATEEN") << QStringLiteral("%DATESHORT") << QStringLiteral("%DATE") << QStringLiteral("%DOW") << QStringLiteral("%TIMELONGEN") << QStringLiteral("%TIMELONG") << QStringLiteral("%TIME") << QStringLiteral("%ODATEEN") << QStringLiteral("%ODATESHORT") << QStringLiteral("%ODATE") << QStringLiteral("%ODOW") << QStringLiteral("%OTIMELONGEN") << QStringLiteral("%OTIMELONG") << QStringLiteral("%OTIME") << QStringLiteral("%BLANK") << QStringLiteral("%NOP") << QStringLiteral("%CLEAR") << QStringLiteral("%DEBUGOFF") << QStringLiteral("%DEBUG") << QStringLiteral("%CURSOR") << QStringLiteral("%SIGNATURE"); return keywords; } QString TemplateParser::Util::getFirstNameFromEmail(const QString &str) { // simple logic: // if there is ',' in name, than format is 'Last, First' // else format is 'First Last' // last resort -- return 'name' from 'name@domain' int sep_pos; QString res; if ((sep_pos = str.indexOf(QLatin1Char('@'))) > 0) { int i; for (i = (sep_pos - 1); i >= 0; --i) { QChar c = str[i]; if (c.isLetterOrNumber()) { res.prepend(c); } else { break; } } } else if ((sep_pos = str.indexOf(QLatin1Char(','))) > 0) { int i; bool begin = false; const int strLength(str.length()); for (i = sep_pos; i < strLength; ++i) { QChar c = str[i]; if (c.isLetterOrNumber()) { begin = true; res.append(c); } else if (begin) { break; } } } else { int i; const int strLength(str.length()); for (i = 0; i < strLength; ++i) { QChar c = str[i]; if (c.isLetterOrNumber()) { res.append(c); } else { break; } } } return res; } QString TemplateParser::Util::getLastNameFromEmail(const QString &str) { // simple logic: // if there is ',' in name, than format is 'Last, First' // else format is 'First Last' int sep_pos; QString res; if ((sep_pos = str.indexOf(QLatin1Char(','))) > 0) { int i; for (i = sep_pos; i >= 0; --i) { QChar c = str[i]; if (c.isLetterOrNumber()) { res.prepend(c); } else { break; } } } else { if ((sep_pos = str.indexOf(QLatin1Char(' '))) > 0) { bool begin = false; const int strLength(str.length()); for (int i = sep_pos; i < strLength; ++i) { QChar c = str[i]; if (c.isLetterOrNumber()) { begin = true; res.append(c); } else if (begin) { break; } } } } return res; } QString TemplateParser::Util::removeSpaceAtBegin(const QString &selection) { QString content = selection; // Remove blank lines at the beginning: - const int firstNonWS = content.indexOf(QRegularExpression(QLatin1String("\\S"))); + const int firstNonWS = content.indexOf(QRegularExpression(QStringLiteral("\\S"))); const int lineStart = content.lastIndexOf(QLatin1Char('\n'), firstNonWS); if (lineStart >= 0) { content.remove(0, lineStart); } return content; }