diff --git a/src/networking/onlinesearch/onlinesearchabstract.cpp b/src/networking/onlinesearch/onlinesearchabstract.cpp index 458dccfc..369b4ff8 100644 --- a/src/networking/onlinesearch/onlinesearchabstract.cpp +++ b/src/networking/onlinesearch/onlinesearchabstract.cpp @@ -1,683 +1,702 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "onlinesearchabstract.h" #include #include #include #include #include #include #ifdef HAVE_QTWIDGETS #include #include #include #endif // HAVE_QTWIDGETS #ifdef HAVE_KF5 #include #include #endif // HAVE_KF5 #include #include #include "internalnetworkaccessmanager.h" +#include "onlinesearchabstract_p.h" #include "logging_networking.h" const QString OnlineSearchAbstract::queryKeyFreeText = QStringLiteral("free"); const QString OnlineSearchAbstract::queryKeyTitle = QStringLiteral("title"); const QString OnlineSearchAbstract::queryKeyAuthor = QStringLiteral("author"); const QString OnlineSearchAbstract::queryKeyYear = QStringLiteral("year"); const int OnlineSearchAbstract::resultNoError = 0; const int OnlineSearchAbstract::resultCancelled = 0; /// may get redefined in the future! const int OnlineSearchAbstract::resultUnspecifiedError = 1; const int OnlineSearchAbstract::resultAuthorizationRequired = 2; const int OnlineSearchAbstract::resultNetworkError = 3; const int OnlineSearchAbstract::resultInvalidArguments = 4; const char *OnlineSearchAbstract::httpUnsafeChars = "%:/=+$?&\0"; #ifdef HAVE_QTWIDGETS -QStringList OnlineSearchQueryFormAbstract::authorLastNames(const Entry &entry) +OnlineSearchAbstract::Form::Private::Private() + : config(KSharedConfig::openConfig(QStringLiteral("kbibtexrc"))) { + /// nothing +} + +QStringList OnlineSearchAbstract::Form::Private::authorLastNames(const Entry &entry) { const Encoder &encoder = Encoder::instance(); const Value v = entry[Entry::ftAuthor]; QStringList result; result.reserve(v.size()); for (const QSharedPointer &vi : v) { QSharedPointer p = vi.dynamicCast(); if (!p.isNull()) result.append(encoder.convertToPlainAscii(p->lastName())); } return result; } -QString OnlineSearchQueryFormAbstract::guessFreeText(const Entry &entry) const +QString OnlineSearchAbstract::Form::Private::guessFreeText(const Entry &entry) { /// If there is a DOI value in this entry, use it as free text static const QStringList doiKeys = {Entry::ftDOI, Entry::ftUrl}; for (const QString &doiKey : doiKeys) if (!entry.value(doiKey).isEmpty()) { const QString text = PlainTextValue::text(entry[doiKey]); const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(text); if (doiRegExpMatch.hasMatch()) return doiRegExpMatch.captured(0); } /// If there is no free text yet (e.g. no DOI number), try to identify an arXiv eprint number static const QStringList arxivKeys = {QStringLiteral("eprint"), Entry::ftNumber}; for (const QString &arxivKey : arxivKeys) if (!entry.value(arxivKey).isEmpty()) { const QString text = PlainTextValue::text(entry[arxivKey]); const QRegularExpressionMatch arXivRegExpMatch = KBibTeX::arXivRegExpWithPrefix.match(text); if (arXivRegExpMatch.hasMatch()) return arXivRegExpMatch.captured(1); } return QString(); } #endif // HAVE_QTWIDGETS OnlineSearchAbstract::OnlineSearchAbstract(QObject *parent) : QObject(parent), m_hasBeenCanceled(false), numSteps(0), curStep(0), m_previousBusyState(false), m_delayedStoppedSearchReturnCode(0) { m_parent = parent; } #ifdef HAVE_QTWIDGETS QIcon OnlineSearchAbstract::icon(QListWidgetItem *listWidgetItem) { static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9_]"), QRegularExpression::CaseInsensitiveOption); const QString cacheDirectory = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/favicons/"); QDir().mkpath(cacheDirectory); const QString fileNameStem = cacheDirectory + QString(favIconUrl()).remove(invalidChars); const QStringList fileNameExtensions {QStringLiteral(".ico"), QStringLiteral(".png"), QString()}; for (const QString &extension : fileNameExtensions) { const QString fileName = fileNameStem + extension; if (QFileInfo::exists(fileName)) return QIcon(fileName); } QNetworkRequest request(favIconUrl()); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); reply->setObjectName(fileNameStem); if (listWidgetItem != nullptr) m_iconReplyToListWidgetItem.insert(reply, listWidgetItem); connect(reply, &QNetworkReply::finished, this, &OnlineSearchAbstract::iconDownloadFinished); return QIcon::fromTheme(QStringLiteral("applications-internet")); } -OnlineSearchQueryFormAbstract *OnlineSearchAbstract::customWidget(QWidget *) { +OnlineSearchAbstract::Form *OnlineSearchAbstract::customWidget(QWidget *) { return nullptr; } void OnlineSearchAbstract::startSearchFromForm() { m_hasBeenCanceled = false; curStep = numSteps = 0; delayedStoppedSearch(resultNoError); } #endif // HAVE_QTWIDGETS QString OnlineSearchAbstract::name() { if (m_name.isEmpty()) { static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9]"), QRegularExpression::CaseInsensitiveOption); m_name = label().remove(invalidChars); } return m_name; } bool OnlineSearchAbstract::busy() const { return numSteps > 0 && curStep < numSteps; } void OnlineSearchAbstract::cancel() { m_hasBeenCanceled = true; curStep = numSteps = 0; refreshBusyProperty(); } QStringList OnlineSearchAbstract::splitRespectingQuotationMarks(const QString &text) { int p1 = 0, p2, max = text.length(); QStringList result; while (p1 < max) { while (text[p1] == ' ') ++p1; p2 = p1; if (text[p2] == '"') { ++p2; while (p2 < max && text[p2] != '"') ++p2; } else while (p2 < max && text[p2] != ' ') ++p2; result << text.mid(p1, p2 - p1 + 1).simplified(); p1 = p2 + 1; } return result; } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply) { QUrl url; return handleErrors(reply, url); } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply, QUrl &newUrl) { /// The URL to be shown or logged shall not contain any API key const QUrl urlToShow = InternalNetworkAccessManager::removeApiKey(reply->url()); newUrl = QUrl(); if (m_hasBeenCanceled) { stopSearch(resultCancelled); return false; } else if (reply->error() != QNetworkReply::NoError) { m_hasBeenCanceled = true; const QString errorString = reply->errorString(); qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "failed (error code" << reply->error() << "," << errorString << "), HTTP code" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() << ":" << reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toByteArray() << ") for URL" << urlToShow.toDisplayString(); const QNetworkRequest &request = reply->request(); /// Dump all HTTP headers that were sent with the original request (except for API keys) const QList rawHeadersSent = request.rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersSent) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "SENT " << rawHeaderName << ":" << request.rawHeader(rawHeaderName); } /// Dump all HTTP headers that were received const QList rawHeadersReceived = reply->rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersReceived) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "RECVD " << rawHeaderName << ":" << reply->rawHeader(rawHeaderName); } #ifdef HAVE_KF5 sendVisualNotification(errorString.isEmpty() ? i18n("Searching '%1' failed for unknown reason.", label()) : i18n("Searching '%1' failed with error message:\n\n%2", label(), errorString), label(), QStringLiteral("kbibtex"), 7 * 1000); #endif // HAVE_KF5 int resultCode = resultUnspecifiedError; if (reply->error() == QNetworkReply::AuthenticationRequiredError || reply->error() == QNetworkReply::ProxyAuthenticationRequiredError) resultCode = resultAuthorizationRequired; else if (reply->error() == QNetworkReply::HostNotFoundError || reply->error() == QNetworkReply::TimeoutError) resultCode = resultNetworkError; stopSearch(resultCode); return false; } /** * Check the reply for various problems that might point to * more severe issues. Remember: those are only indicators * to problems which have to be handled elsewhere (therefore, * returning 'true' is totally ok here). */ if (reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) { newUrl = reply->url().resolved(reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl()); } else if (reply->size() == 0) qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "on url" << urlToShow.toDisplayString() << "returned no data"; return true; } QString OnlineSearchAbstract::htmlAttribute(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return QString(); ///< no closing angle bracket found const QString attributePattern = QString(QStringLiteral(" %1=")).arg(attribute); const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return QString(); ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const int openingQuotationMarkPos = attributePatternPos + attributePatternLen; const QChar quotationMark = htmlCode[openingQuotationMarkPos]; if (quotationMark != QLatin1Char('"') && quotationMark != QLatin1Char('\'')) { /// No valid opening quotation mark found int spacePos = openingQuotationMarkPos; while (spacePos < endPos && !htmlCode[spacePos].isSpace()) ++spacePos; if (spacePos > endPos) return QString(); ///< no closing space found return htmlCode.mid(openingQuotationMarkPos, spacePos - openingQuotationMarkPos); } else { /// Attribute has either single or double quotation marks const int closingQuotationMarkPos = htmlCode.indexOf(quotationMark, openingQuotationMarkPos + 1); if (closingQuotationMarkPos < 0 || closingQuotationMarkPos > endPos) return QString(); ///< closing quotation mark not found within limits return htmlCode.mid(openingQuotationMarkPos + 1, closingQuotationMarkPos - openingQuotationMarkPos - 1); } } bool OnlineSearchAbstract::htmlAttributeIsSelected(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return false; ///< no closing angle bracket found const QString attributePattern = QStringLiteral(" ") + attribute; const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return false; ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const QChar nextAfterAttributePattern = htmlCode[attributePatternPos + attributePatternLen]; if (nextAfterAttributePattern.isSpace() || nextAfterAttributePattern == QLatin1Char('>') || nextAfterAttributePattern == QLatin1Char('/')) /// No value given for attribute (old-style HTML), so assuming it means checked/selected return true; else if (nextAfterAttributePattern == QLatin1Char('=')) { /// Expecting value to attribute, so retrieve it and check for 'selected' or 'checked' const QString attributeValue = htmlAttribute(htmlCode, attributePatternPos, attribute).toLower(); return attributeValue == QStringLiteral("selected") || attributeValue == QStringLiteral("checked"); } /// Reaching this point only if HTML code is invalid return false; } #ifdef HAVE_KF5 /** * Display a passive notification popup using the D-Bus interface. * Copied from KDialog with modifications. */ void OnlineSearchAbstract::sendVisualNotification(const QString &text, const QString &title, const QString &icon, int timeout) { static const QString dbusServiceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusInterfaceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusPath = QStringLiteral("/org/freedesktop/Notifications"); // check if service already exists on plugin instantiation QDBusConnectionInterface *interface = QDBusConnection::sessionBus().interface(); if (interface == nullptr || !interface->isServiceRegistered(dbusServiceName)) { return; } if (timeout <= 0) timeout = 10 * 1000; QDBusMessage m = QDBusMessage::createMethodCall(dbusServiceName, dbusPath, dbusInterfaceName, QStringLiteral("Notify")); const QList args {QStringLiteral("kdialog"), 0U, icon, title, text, QStringList(), QVariantMap(), timeout}; m.setArguments(args); QDBusMessage replyMsg = QDBusConnection::sessionBus().call(m); if (replyMsg.type() == QDBusMessage::ReplyMessage) { if (!replyMsg.arguments().isEmpty()) { return; } // Not displaying any error messages as this is optional for kdialog // and KPassivePopup is a perfectly valid fallback. //else { // qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: received reply with no arguments."; //} } else if (replyMsg.type() == QDBusMessage::ErrorMessage) { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: failed to send D-Bus message"; //qCDebug(LOG_KBIBTEX_NETWORKING) << replyMsg; } else { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Unexpected reply type"; } } #endif // HAVE_KF5 QString OnlineSearchAbstract::encodeURL(QString rawText) { const char *cur = httpUnsafeChars; while (*cur != '\0') { rawText = rawText.replace(QChar(*cur), '%' + QString::number(*cur, 16)); ++cur; } rawText = rawText.replace(QLatin1Char(' '), QLatin1Char('+')); return rawText; } QString OnlineSearchAbstract::decodeURL(QString rawText) { static const QRegularExpression mimeRegExp(QStringLiteral("%([0-9A-Fa-f]{2})")); QRegularExpressionMatch mimeRegExpMatch; while ((mimeRegExpMatch = mimeRegExp.match(rawText)).hasMatch()) { bool ok = false; QChar c(mimeRegExpMatch.captured(1).toInt(&ok, 16)); if (ok) rawText = rawText.replace(mimeRegExpMatch.captured(0), c); } rawText = rawText.replace(QStringLiteral("&"), QStringLiteral("&")).replace(QLatin1Char('+'), QStringLiteral(" ")); return rawText; } QMap OnlineSearchAbstract::formParameters(const QString &htmlText, int startPos) const { /// how to recognize HTML tags static const QString formTagEnd = QStringLiteral(""); static const QString inputTagBegin = QStringLiteral(""); static const QString optionTagBegin = QStringLiteral("