diff --git a/src/data/value.cpp b/src/data/value.cpp index 47123a65..5371aac4 100644 --- a/src/data/value.cpp +++ b/src/data/value.cpp @@ -1,707 +1,705 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see . * ***************************************************************************/ #include "value.h" #include #include #include #include #include #ifdef HAVE_KF5 #include #include #include #else // HAVE_KF5 #define i18n(text) QObject::tr(text) #endif // HAVE_KF5 #include #include "file.h" #include "logging_data.h" quint64 ValueItem::internalIdCounter = 0; uint qHash(const QSharedPointer &valueItem) { return qHash(valueItem->id()); } const QRegularExpression ValueItem::ignoredInSorting(QStringLiteral("[{}\\\\]+")); ValueItem::ValueItem() : internalId(++internalIdCounter) { /// nothing } ValueItem::~ValueItem() { /// nothing } quint64 ValueItem::id() const { return internalId; } bool ValueItem::operator!=(const ValueItem &other) const { return !operator ==(other); } Keyword::Keyword(const Keyword &other) : m_text(other.m_text) { /// nothing } Keyword::Keyword(const QString &text) : m_text(text) { /// nothing } void Keyword::setText(const QString &text) { m_text = text; } QString Keyword::text() const { return m_text; } void Keyword::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { if (replaceMode == ValueItem::AnySubstring) m_text = m_text.replace(before, after); else if (replaceMode == ValueItem::CompleteMatch && m_text == before) m_text = after; } bool Keyword::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { const QString text = QString(m_text).remove(ignoredInSorting); return text.contains(pattern, caseSensitive); } bool Keyword::operator==(const ValueItem &other) const { const Keyword *otherKeyword = dynamic_cast(&other); if (otherKeyword != nullptr) { return otherKeyword->text() == text(); } else return false; } bool Keyword::isKeyword(const ValueItem &other) { return typeid(other) == typeid(Keyword); } Person::Person(const QString &firstName, const QString &lastName, const QString &suffix) : m_firstName(firstName), m_lastName(lastName), m_suffix(suffix) { /// nothing } Person::Person(const Person &other) : m_firstName(other.firstName()), m_lastName(other.lastName()), m_suffix(other.suffix()) { /// nothing } QString Person::firstName() const { return m_firstName; } QString Person::lastName() const { return m_lastName; } QString Person::suffix() const { return m_suffix; } void Person::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { if (replaceMode == ValueItem::AnySubstring) { m_firstName = m_firstName.replace(before, after); m_lastName = m_lastName.replace(before, after); m_suffix = m_suffix.replace(before, after); } else if (replaceMode == ValueItem::CompleteMatch) { if (m_firstName == before) m_firstName = after; if (m_lastName == before) m_lastName = after; if (m_suffix == before) m_suffix = after; } } bool Person::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { const QString firstName = QString(m_firstName).remove(ignoredInSorting); const QString lastName = QString(m_lastName).remove(ignoredInSorting); const QString suffix = QString(m_suffix).remove(ignoredInSorting); return firstName.contains(pattern, caseSensitive) || lastName.contains(pattern, caseSensitive) || suffix.contains(pattern, caseSensitive) || QString(QStringLiteral("%1 %2|%2, %1")).arg(firstName, lastName).contains(pattern, caseSensitive); } bool Person::operator==(const ValueItem &other) const { const Person *otherPerson = dynamic_cast(&other); if (otherPerson != nullptr) { return otherPerson->firstName() == firstName() && otherPerson->lastName() == lastName() && otherPerson->suffix() == suffix(); } else return false; } QString Person::transcribePersonName(const Person *person, const QString &formatting) { return transcribePersonName(formatting, person->firstName(), person->lastName(), person->suffix()); } QString Person::transcribePersonName(const QString &formatting, const QString &firstName, const QString &lastName, const QString &suffix) { QString result = formatting; int p1 = -1, p2 = -1, p3 = -1; while ((p1 = result.indexOf('<')) >= 0 && (p2 = result.indexOf('>', p1 + 1)) >= 0 && (p3 = result.indexOf('%', p1)) >= 0 && p3 < p2) { QString insert; switch (result[p3 + 1].toLatin1()) { case 'f': insert = firstName; break; case 'l': insert = lastName; break; case 's': insert = suffix; break; } if (!insert.isEmpty()) insert = result.mid(p1 + 1, p3 - p1 - 1) + insert + result.mid(p3 + 2, p2 - p3 - 2); result = result.left(p1) + insert + result.mid(p2 + 1); } return result; } bool Person::isPerson(const ValueItem &other) { return typeid(other) == typeid(Person); } QDebug operator<<(QDebug dbg, const Person &person) { dbg.nospace() << "Person " << Person::transcribePersonName(&person, Preferences::defaultPersonNameFormat); return dbg; } MacroKey::MacroKey(const MacroKey &other) : m_text(other.m_text) { /// nothing } MacroKey::MacroKey(const QString &text) : m_text(text) { /// nothing } void MacroKey::setText(const QString &text) { m_text = text; } QString MacroKey::text() const { return m_text; } bool MacroKey::isValid() { const QString t = text(); static const QRegularExpression validMacroKey(QStringLiteral("^[a-z][-.:/+_a-z0-9]*$|^[0-9]+$"), QRegularExpression::CaseInsensitiveOption); const QRegularExpressionMatch match = validMacroKey.match(t); return match.hasMatch() && match.captured(0) == t; } void MacroKey::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { if (replaceMode == ValueItem::AnySubstring) m_text = m_text.replace(before, after); else if (replaceMode == ValueItem::CompleteMatch && m_text == before) m_text = after; } bool MacroKey::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { const QString text = QString(m_text).remove(ignoredInSorting); return text.contains(pattern, caseSensitive); } bool MacroKey::operator==(const ValueItem &other) const { const MacroKey *otherMacroKey = dynamic_cast(&other); if (otherMacroKey != nullptr) { return otherMacroKey->text() == text(); } else return false; } bool MacroKey::isMacroKey(const ValueItem &other) { return typeid(other) == typeid(MacroKey); } QDebug operator<<(QDebug dbg, const MacroKey ¯okey) { dbg.nospace() << "MacroKey " << macrokey.text(); return dbg; } PlainText::PlainText(const PlainText &other) : m_text(other.text()) { /// nothing } PlainText::PlainText(const QString &text) : m_text(text) { /// nothing } void PlainText::setText(const QString &text) { m_text = text; } QString PlainText::text() const { return m_text; } void PlainText::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { if (replaceMode == ValueItem::AnySubstring) m_text = m_text.replace(before, after); else if (replaceMode == ValueItem::CompleteMatch && m_text == before) m_text = after; } bool PlainText::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { const QString text = QString(m_text).remove(ignoredInSorting); return text.contains(pattern, caseSensitive); } bool PlainText::operator==(const ValueItem &other) const { const PlainText *otherPlainText = dynamic_cast(&other); if (otherPlainText != nullptr) { return otherPlainText->text() == text(); } else return false; } bool PlainText::isPlainText(const ValueItem &other) { return typeid(other) == typeid(PlainText); } QDebug operator<<(QDebug dbg, const PlainText &plainText) { dbg.nospace() << "PlainText " << plainText.text(); return dbg; } VerbatimText::VerbatimText(const VerbatimText &other) : m_text(other.text()) { /// nothing } VerbatimText::VerbatimText(const QString &text) : m_text(text) { /// nothing } void VerbatimText::setText(const QString &text) { m_text = text; } QString VerbatimText::text() const { return m_text; } void VerbatimText::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { if (replaceMode == ValueItem::AnySubstring) m_text = m_text.replace(before, after); else if (replaceMode == ValueItem::CompleteMatch && m_text == before) m_text = after; } bool VerbatimText::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { const QString text = QString(m_text).remove(ignoredInSorting); bool contained = text.contains(pattern, caseSensitive); -#ifdef HAVE_KF5 if (!contained) { /// Only if simple text match failed, check color labels /// For a match, the user's pattern has to be the start of the color label /// and this verbatim text has to contain the color as hex string for (QVector>::ConstIterator it = Preferences::instance().colorCodes().constBegin(); !contained && it != Preferences::instance().colorCodes().constEnd(); ++it) contained = text.compare(it->first.name(), Qt::CaseInsensitive) == 0 && it->second.contains(pattern, Qt::CaseInsensitive); } -#endif // HAVE_KF5 return contained; } bool VerbatimText::operator==(const ValueItem &other) const { const VerbatimText *otherVerbatimText = dynamic_cast(&other); if (otherVerbatimText != nullptr) { return otherVerbatimText->text() == text(); } else return false; } bool VerbatimText::isVerbatimText(const ValueItem &other) { return typeid(other) == typeid(VerbatimText); } QDebug operator<<(QDebug dbg, const VerbatimText &verbatimText) { dbg.nospace() << "VerbatimText " << verbatimText.text(); return dbg; } Value::Value() : QVector >() { /// nothing } Value::Value(const Value &other) : QVector >(other) { /// nothing } Value::Value(Value &&other) : QVector >(other) { /// nothing } Value::~Value() { clear(); } void Value::replace(const QString &before, const QString &after, ValueItem::ReplaceMode replaceMode) { QSet > unique; /// Delegate the replace operation to each individual ValueItem /// contained in this Value object for (Value::Iterator it = begin(); it != end();) { (*it)->replace(before, after, replaceMode); bool containedInUnique = false; for (const auto &valueItem : const_cast > &>(unique)) { containedInUnique = *valueItem.data() == *(*it).data(); if (containedInUnique) break; } if (containedInUnique) it = erase(it); else { unique.insert(*it); ++it; } } QSet uniqueValueItemTexts; for (int i = count() - 1; i >= 0; --i) { at(i)->replace(before, after, replaceMode); const QString valueItemText = PlainTextValue::text(*at(i).data()); if (uniqueValueItemTexts.contains(valueItemText)) { /// Due to a replace/delete operation above, an old ValueItem's text /// matches the replaced text. /// Therefore, remove the replaced text to avoid duplicates remove(i); ++i; /// compensate for for-loop's --i } else uniqueValueItemTexts.insert(valueItemText); } } void Value::replace(const QString &before, const QSharedPointer &after) { const QString valueText = PlainTextValue::text(*this); if (valueText == before) { clear(); append(after); } else { QSet uniqueValueItemTexts; for (int i = count() - 1; i >= 0; --i) { QString valueItemText = PlainTextValue::text(*at(i).data()); if (valueItemText == before) { /// Perform replacement operation QVector >::replace(i, after); valueItemText = PlainTextValue::text(*after.data()); // uniqueValueItemTexts.insert(PlainTextValue::text(*after.data())); } if (uniqueValueItemTexts.contains(valueItemText)) { /// Due to a previous replace operation, an existingValueItem's /// text matches a text which was inserted as an "after" ValueItem. /// Therefore, remove the old ValueItem to avoid duplicates. remove(i); } else { /// Neither a replacement, nor a duplicate. Keep this /// ValueItem (memorize as unique) and continue. uniqueValueItemTexts.insert(valueItemText); } } } } bool Value::containsPattern(const QString &pattern, Qt::CaseSensitivity caseSensitive) const { for (const auto &valueItem : const_cast(*this)) { if (valueItem->containsPattern(pattern, caseSensitive)) return true; } return false; } bool Value::contains(const ValueItem &item) const { for (const auto &valueItem : const_cast(*this)) if (valueItem->operator==(item)) return true; return false; } Value &Value::operator=(const Value &rhs) { return static_cast(QVector >::operator =((rhs))); } Value &Value::operator=(Value &&rhs) { return static_cast(QVector >::operator =((rhs))); } Value &Value::operator<<(const QSharedPointer &value) { return static_cast(QVector >::operator<<((value))); } bool Value::operator==(const Value &rhs) const { const Value &lhs = *this; ///< just for readability to have a 'lhs' matching 'rhs' /// Obviously, both Values must be of same size if (lhs.count() != rhs.count()) return false; /// Synchronously iterate over both Values' ValueItems for (Value::ConstIterator lhsIt = lhs.constBegin(), rhsIt = rhs.constBegin(); lhsIt != lhs.constEnd() && rhsIt != rhs.constEnd(); ++lhsIt, ++rhsIt) { /// Are both ValueItems PlainTexts and are both PlainTexts equal? const QSharedPointer lhsPlainText = lhsIt->dynamicCast<PlainText>(); const QSharedPointer<PlainText> rhsPlainText = rhsIt->dynamicCast<PlainText>(); if ((lhsPlainText.isNull() && !rhsPlainText.isNull()) || (!lhsPlainText.isNull() && rhsPlainText.isNull())) return false; if (!lhsPlainText.isNull() && !rhsPlainText.isNull()) { if (*lhsPlainText.data() != *rhsPlainText.data()) return false; } else { /// Remainder of comparisons is like for PlainText above, just for other descendants of ValueItem const QSharedPointer<MacroKey> lhsMacroKey = lhsIt->dynamicCast<MacroKey>(); const QSharedPointer<MacroKey> rhsMacroKey = rhsIt->dynamicCast<MacroKey>(); if ((lhsMacroKey.isNull() && !rhsMacroKey.isNull()) || (!lhsMacroKey.isNull() && rhsMacroKey.isNull())) return false; if (!lhsMacroKey.isNull() && !rhsMacroKey.isNull()) { if (*lhsMacroKey.data() != *rhsMacroKey.data()) return false; } else { const QSharedPointer<Person> lhsPerson = lhsIt->dynamicCast<Person>(); const QSharedPointer<Person> rhsPerson = rhsIt->dynamicCast<Person>(); if ((lhsPerson.isNull() && !rhsPerson.isNull()) || (!lhsPerson.isNull() && rhsPerson.isNull())) return false; if (!lhsPerson.isNull() && !rhsPerson.isNull()) { if (*lhsPerson.data() != *rhsPerson.data()) return false; } else { const QSharedPointer<VerbatimText> lhsVerbatimText = lhsIt->dynamicCast<VerbatimText>(); const QSharedPointer<VerbatimText> rhsVerbatimText = rhsIt->dynamicCast<VerbatimText>(); if ((lhsVerbatimText.isNull() && !rhsVerbatimText.isNull()) || (!lhsVerbatimText.isNull() && rhsVerbatimText.isNull())) return false; if (!lhsVerbatimText.isNull() && !rhsVerbatimText.isNull()) { if (*lhsVerbatimText.data() != *rhsVerbatimText.data()) return false; } else { const QSharedPointer<Keyword> lhsKeyword = lhsIt->dynamicCast<Keyword>(); const QSharedPointer<Keyword> rhsKeyword = rhsIt->dynamicCast<Keyword>(); if ((lhsKeyword.isNull() && !rhsKeyword.isNull()) || (!lhsKeyword.isNull() && rhsKeyword.isNull())) return false; if (!lhsKeyword.isNull() && !rhsKeyword.isNull()) { if (*lhsKeyword.data() != *rhsKeyword.data()) return false; } else { /// If there are other descendants of ValueItem, add tests here ... return false; } } } } } } /// No check failed, so equalness is proven return true; } bool Value::operator!=(const Value &rhs) const { return !operator ==(rhs); } QDebug operator<<(QDebug dbg, const Value &value) { dbg.nospace() << "Value"; if (value.isEmpty()) dbg << " is empty"; else dbg.nospace() << ": " << PlainTextValue::text(value); return dbg; } QString PlainTextValue::text(const Value &value) { ValueItemType vit = VITOther; ValueItemType lastVit = VITOther; QString result; for (const auto &valueItem : value) { QString nextText = text(*valueItem, vit); if (!nextText.isEmpty()) { if (lastVit == VITPerson && vit == VITPerson) result.append(i18n(" and ")); // TODO proper list of authors/editors, not just joined by "and" else if (lastVit == VITPerson && vit == VITOther && nextText == QStringLiteral("others")) { /// "and others" case: replace text to be appended by translated variant nextText = i18n(" and others"); } else if (lastVit == VITKeyword && vit == VITKeyword) result.append("; "); else if (!result.isEmpty()) result.append(" "); result.append(nextText); lastVit = vit; } } return result; } QString PlainTextValue::text(const QSharedPointer<const ValueItem> &valueItem) { const ValueItem *p = valueItem.data(); return text(*p); } QString PlainTextValue::text(const ValueItem &valueItem) { ValueItemType vit; return text(valueItem, vit); } QString PlainTextValue::text(const ValueItem &valueItem, ValueItemType &vit) { QString result; vit = VITOther; bool isVerbatim = false; const PlainText *plainText = dynamic_cast<const PlainText *>(&valueItem); if (plainText != nullptr) { result = plainText->text(); } else { const MacroKey *macroKey = dynamic_cast<const MacroKey *>(&valueItem); if (macroKey != nullptr) { result = macroKey->text(); // TODO Use File to resolve key to full text } else { const Person *person = dynamic_cast<const Person *>(&valueItem); if (person != nullptr) { result = Person::transcribePersonName(person, Preferences::instance().personNameFormat()); vit = VITPerson; } else { const Keyword *keyword = dynamic_cast<const Keyword *>(&valueItem); if (keyword != nullptr) { result = keyword->text(); vit = VITKeyword; } else { const VerbatimText *verbatimText = dynamic_cast<const VerbatimText *>(&valueItem); if (verbatimText != nullptr) { result = verbatimText->text(); isVerbatim = true; } else qCWarning(LOG_KBIBTEX_DATA) << "Cannot interpret ValueItem to one of its descendants"; } } } } /// clean up result string const int len = result.length(); int j = 0; static const QChar cbo = QLatin1Char('{'), cbc = QLatin1Char('}'), bs = QLatin1Char('\\'), mns = QLatin1Char('-'), comma = QLatin1Char(','), thinspace = QChar(0x2009), tilde = QLatin1Char('~'), nobreakspace = QChar(0x00a0); for (int i = 0; i < len; ++i) { if ((result[i] == cbo || result[i] == cbc) && (i < 1 || result[i - 1] != bs)) { /// hop over curly brackets } else if (i < len - 1 && result[i] == bs && result[i + 1] == mns) { /// hop over hyphenation commands ++i; } else if (i < len - 1 && result[i] == bs && result[i + 1] == comma) { /// place '\,' with a thin space result[j] = thinspace; ++i; ++j; } else if (!isVerbatim && result[i] == tilde && (i < 1 || result[i - 1] != bs)) { /// replace '~' with a non-breaking space, /// except if text was verbatim (e.g. a 'localfile' value /// like '~/document.pdf' or 'document.pdf~') result[j] = nobreakspace; ++j; } else { if (i > j) { /// move individual characters forward in result string result[j] = result[i]; } ++j; } } result.resize(j); return result; } diff --git a/src/networking/onlinesearch/onlinesearchabstract.cpp b/src/networking/onlinesearch/onlinesearchabstract.cpp index 6dd0decf..9742627a 100644 --- a/src/networking/onlinesearch/onlinesearchabstract.cpp +++ b/src/networking/onlinesearch/onlinesearchabstract.cpp @@ -1,702 +1,702 @@ /*************************************************************************** * Copyright (C) 2004-2019 by Thomas Fischer <fischer@unix-ag.uni-kl.de> * * * * This program is free software; you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation; either version 2 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program; if not, see <https://www.gnu.org/licenses/>. * ***************************************************************************/ #include "onlinesearchabstract.h" #include <QFileInfo> #include <QNetworkReply> #include <QDir> #include <QTimer> #include <QStandardPaths> #include <QRegularExpression> -#ifdef HAVE_QTWIDGETS -#include <QListWidgetItem> #include <QDBusConnection> #include <QDBusConnectionInterface> +#ifdef HAVE_QTWIDGETS +#include <QListWidgetItem> #endif // HAVE_QTWIDGETS #ifdef HAVE_KF5 #include <KLocalizedString> #include <KMessageBox> #endif // HAVE_KF5 #include <KBibTeX> #include <Encoder> #include "internalnetworkaccessmanager.h" #include "onlinesearchabstract_p.h" #include "logging_networking.h" const QString OnlineSearchAbstract::queryKeyFreeText = QStringLiteral("free"); const QString OnlineSearchAbstract::queryKeyTitle = QStringLiteral("title"); const QString OnlineSearchAbstract::queryKeyAuthor = QStringLiteral("author"); const QString OnlineSearchAbstract::queryKeyYear = QStringLiteral("year"); const int OnlineSearchAbstract::resultNoError = 0; const int OnlineSearchAbstract::resultCancelled = 0; /// may get redefined in the future! const int OnlineSearchAbstract::resultUnspecifiedError = 1; const int OnlineSearchAbstract::resultAuthorizationRequired = 2; const int OnlineSearchAbstract::resultNetworkError = 3; const int OnlineSearchAbstract::resultInvalidArguments = 4; const char *OnlineSearchAbstract::httpUnsafeChars = "%:/=+$?&\0"; #ifdef HAVE_QTWIDGETS OnlineSearchAbstract::Form::Private::Private() : config(KSharedConfig::openConfig(QStringLiteral("kbibtexrc"))) { /// nothing } QStringList OnlineSearchAbstract::Form::Private::authorLastNames(const Entry &entry) { const Encoder &encoder = Encoder::instance(); const Value v = entry[Entry::ftAuthor]; QStringList result; result.reserve(v.size()); for (const QSharedPointer<ValueItem> &vi : v) { QSharedPointer<const Person> p = vi.dynamicCast<const Person>(); if (!p.isNull()) result.append(encoder.convertToPlainAscii(p->lastName())); } return result; } QString OnlineSearchAbstract::Form::Private::guessFreeText(const Entry &entry) { /// If there is a DOI value in this entry, use it as free text static const QStringList doiKeys = {Entry::ftDOI, Entry::ftUrl}; for (const QString &doiKey : doiKeys) if (!entry.value(doiKey).isEmpty()) { const QString text = PlainTextValue::text(entry[doiKey]); const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(text); if (doiRegExpMatch.hasMatch()) return doiRegExpMatch.captured(0); } /// If there is no free text yet (e.g. no DOI number), try to identify an arXiv eprint number static const QStringList arxivKeys = {QStringLiteral("eprint"), Entry::ftNumber}; for (const QString &arxivKey : arxivKeys) if (!entry.value(arxivKey).isEmpty()) { const QString text = PlainTextValue::text(entry[arxivKey]); const QRegularExpressionMatch arXivRegExpMatch = KBibTeX::arXivRegExpWithPrefix.match(text); if (arXivRegExpMatch.hasMatch()) return arXivRegExpMatch.captured(1); } return QString(); } #endif // HAVE_QTWIDGETS OnlineSearchAbstract::OnlineSearchAbstract(QObject *parent) : QObject(parent), m_hasBeenCanceled(false), numSteps(0), curStep(0), m_previousBusyState(false), m_delayedStoppedSearchReturnCode(0) { m_parent = parent; } #ifdef HAVE_QTWIDGETS QIcon OnlineSearchAbstract::icon(QListWidgetItem *listWidgetItem) { static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9_]"), QRegularExpression::CaseInsensitiveOption); const QString cacheDirectory = QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/favicons/"); QDir().mkpath(cacheDirectory); const QString fileNameStem = cacheDirectory + QString(favIconUrl()).remove(invalidChars); const QStringList fileNameExtensions {QStringLiteral(".ico"), QStringLiteral(".png"), QString()}; for (const QString &extension : fileNameExtensions) { const QString fileName = fileNameStem + extension; if (QFileInfo::exists(fileName)) return QIcon(fileName); } QNetworkRequest request(favIconUrl()); QNetworkReply *reply = InternalNetworkAccessManager::instance().get(request); reply->setObjectName(fileNameStem); if (listWidgetItem != nullptr) m_iconReplyToListWidgetItem.insert(reply, listWidgetItem); connect(reply, &QNetworkReply::finished, this, &OnlineSearchAbstract::iconDownloadFinished); return QIcon::fromTheme(QStringLiteral("applications-internet")); } OnlineSearchAbstract::Form *OnlineSearchAbstract::customWidget(QWidget *) { return nullptr; } void OnlineSearchAbstract::startSearchFromForm() { m_hasBeenCanceled = false; curStep = numSteps = 0; delayedStoppedSearch(resultNoError); } #endif // HAVE_QTWIDGETS QString OnlineSearchAbstract::name() { if (m_name.isEmpty()) { static const QRegularExpression invalidChars(QStringLiteral("[^-a-z0-9]"), QRegularExpression::CaseInsensitiveOption); m_name = label().remove(invalidChars); } return m_name; } bool OnlineSearchAbstract::busy() const { return numSteps > 0 && curStep < numSteps; } void OnlineSearchAbstract::cancel() { m_hasBeenCanceled = true; curStep = numSteps = 0; refreshBusyProperty(); } QStringList OnlineSearchAbstract::splitRespectingQuotationMarks(const QString &text) { int p1 = 0, p2, max = text.length(); QStringList result; while (p1 < max) { while (text[p1] == ' ') ++p1; p2 = p1; if (text[p2] == '"') { ++p2; while (p2 < max && text[p2] != '"') ++p2; } else while (p2 < max && text[p2] != ' ') ++p2; result << text.mid(p1, p2 - p1 + 1).simplified(); p1 = p2 + 1; } return result; } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply) { QUrl url; return handleErrors(reply, url); } bool OnlineSearchAbstract::handleErrors(QNetworkReply *reply, QUrl &newUrl) { /// The URL to be shown or logged shall not contain any API key const QUrl urlToShow = InternalNetworkAccessManager::removeApiKey(reply->url()); newUrl = QUrl(); if (m_hasBeenCanceled) { stopSearch(resultCancelled); return false; } else if (reply->error() != QNetworkReply::NoError) { m_hasBeenCanceled = true; const QString errorString = reply->errorString(); qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "failed (error code" << reply->error() << "," << errorString << "), HTTP code" << reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() << ":" << reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toByteArray() << ") for URL" << urlToShow.toDisplayString(); const QNetworkRequest &request = reply->request(); /// Dump all HTTP headers that were sent with the original request (except for API keys) const QList<QByteArray> rawHeadersSent = request.rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersSent) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "SENT " << rawHeaderName << ":" << request.rawHeader(rawHeaderName); } /// Dump all HTTP headers that were received const QList<QByteArray> rawHeadersReceived = reply->rawHeaderList(); for (const QByteArray &rawHeaderName : rawHeadersReceived) { if (rawHeaderName.toLower().contains("apikey") || rawHeaderName.toLower().contains("api-key")) continue; ///< skip dumping header values containing an API key qCDebug(LOG_KBIBTEX_NETWORKING) << "RECVD " << rawHeaderName << ":" << reply->rawHeader(rawHeaderName); } #ifdef HAVE_KF5 sendVisualNotification(errorString.isEmpty() ? i18n("Searching '%1' failed for unknown reason.", label()) : i18n("Searching '%1' failed with error message:\n\n%2", label(), errorString), label(), QStringLiteral("kbibtex"), 7 * 1000); #endif // HAVE_KF5 int resultCode = resultUnspecifiedError; if (reply->error() == QNetworkReply::AuthenticationRequiredError || reply->error() == QNetworkReply::ProxyAuthenticationRequiredError) resultCode = resultAuthorizationRequired; else if (reply->error() == QNetworkReply::HostNotFoundError || reply->error() == QNetworkReply::TimeoutError) resultCode = resultNetworkError; stopSearch(resultCode); return false; } /** * Check the reply for various problems that might point to * more severe issues. Remember: those are only indicators * to problems which have to be handled elsewhere (therefore, * returning 'true' is totally ok here). */ if (reply->attribute(QNetworkRequest::RedirectionTargetAttribute).isValid()) { newUrl = reply->url().resolved(reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl()); } else if (reply->size() == 0) qCWarning(LOG_KBIBTEX_NETWORKING) << "Search using" << label() << "on url" << urlToShow.toDisplayString() << "returned no data"; return true; } QString OnlineSearchAbstract::htmlAttribute(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return QString(); ///< no closing angle bracket found const QString attributePattern = QString(QStringLiteral(" %1=")).arg(attribute); const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return QString(); ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const int openingQuotationMarkPos = attributePatternPos + attributePatternLen; const QChar quotationMark = htmlCode[openingQuotationMarkPos]; if (quotationMark != QLatin1Char('"') && quotationMark != QLatin1Char('\'')) { /// No valid opening quotation mark found int spacePos = openingQuotationMarkPos; while (spacePos < endPos && !htmlCode[spacePos].isSpace()) ++spacePos; if (spacePos > endPos) return QString(); ///< no closing space found return htmlCode.mid(openingQuotationMarkPos, spacePos - openingQuotationMarkPos); } else { /// Attribute has either single or double quotation marks const int closingQuotationMarkPos = htmlCode.indexOf(quotationMark, openingQuotationMarkPos + 1); if (closingQuotationMarkPos < 0 || closingQuotationMarkPos > endPos) return QString(); ///< closing quotation mark not found within limits return htmlCode.mid(openingQuotationMarkPos + 1, closingQuotationMarkPos - openingQuotationMarkPos - 1); } } bool OnlineSearchAbstract::htmlAttributeIsSelected(const QString &htmlCode, const int startPos, const QString &attribute) const { const int endPos = htmlCode.indexOf(QLatin1Char('>'), startPos); if (endPos < 0) return false; ///< no closing angle bracket found const QString attributePattern = QStringLiteral(" ") + attribute; const int attributePatternPos = htmlCode.indexOf(attributePattern, startPos, Qt::CaseInsensitive); if (attributePatternPos < 0 || attributePatternPos > endPos) return false; ///< attribute not found within limits const int attributePatternLen = attributePattern.length(); const QChar nextAfterAttributePattern = htmlCode[attributePatternPos + attributePatternLen]; if (nextAfterAttributePattern.isSpace() || nextAfterAttributePattern == QLatin1Char('>') || nextAfterAttributePattern == QLatin1Char('/')) /// No value given for attribute (old-style HTML), so assuming it means checked/selected return true; else if (nextAfterAttributePattern == QLatin1Char('=')) { /// Expecting value to attribute, so retrieve it and check for 'selected' or 'checked' const QString attributeValue = htmlAttribute(htmlCode, attributePatternPos, attribute).toLower(); return attributeValue == QStringLiteral("selected") || attributeValue == QStringLiteral("checked"); } /// Reaching this point only if HTML code is invalid return false; } #ifdef HAVE_KF5 /** * Display a passive notification popup using the D-Bus interface. * Copied from KDialog with modifications. */ void OnlineSearchAbstract::sendVisualNotification(const QString &text, const QString &title, const QString &icon, int timeout) { static const QString dbusServiceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusInterfaceName = QStringLiteral("org.freedesktop.Notifications"); static const QString dbusPath = QStringLiteral("/org/freedesktop/Notifications"); // check if service already exists on plugin instantiation QDBusConnectionInterface *interface = QDBusConnection::sessionBus().interface(); if (interface == nullptr || !interface->isServiceRegistered(dbusServiceName)) { return; } if (timeout <= 0) timeout = 10 * 1000; QDBusMessage m = QDBusMessage::createMethodCall(dbusServiceName, dbusPath, dbusInterfaceName, QStringLiteral("Notify")); const QList<QVariant> args {QStringLiteral("kdialog"), 0U, icon, title, text, QStringList(), QVariantMap(), timeout}; m.setArguments(args); QDBusMessage replyMsg = QDBusConnection::sessionBus().call(m); if (replyMsg.type() == QDBusMessage::ReplyMessage) { if (!replyMsg.arguments().isEmpty()) { return; } // Not displaying any error messages as this is optional for kdialog // and KPassivePopup is a perfectly valid fallback. //else { // qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: received reply with no arguments."; //} } else if (replyMsg.type() == QDBusMessage::ErrorMessage) { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Error: failed to send D-Bus message"; //qCDebug(LOG_KBIBTEX_NETWORKING) << replyMsg; } else { //qCDebug(LOG_KBIBTEX_NETWORKING) << "Unexpected reply type"; } } #endif // HAVE_KF5 QString OnlineSearchAbstract::encodeURL(QString rawText) { const char *cur = httpUnsafeChars; while (*cur != '\0') { rawText = rawText.replace(QChar(*cur), '%' + QString::number(*cur, 16)); ++cur; } rawText = rawText.replace(QLatin1Char(' '), QLatin1Char('+')); return rawText; } QString OnlineSearchAbstract::decodeURL(QString rawText) { static const QRegularExpression mimeRegExp(QStringLiteral("%([0-9A-Fa-f]{2})")); QRegularExpressionMatch mimeRegExpMatch; while ((mimeRegExpMatch = mimeRegExp.match(rawText)).hasMatch()) { bool ok = false; QChar c(mimeRegExpMatch.captured(1).toInt(&ok, 16)); if (ok) rawText = rawText.replace(mimeRegExpMatch.captured(0), c); } rawText = rawText.replace(QStringLiteral("&amp;"), QStringLiteral("&")).replace(QLatin1Char('+'), QStringLiteral(" ")); return rawText; } QMap<QString, QString> OnlineSearchAbstract::formParameters(const QString &htmlText, int startPos) const { /// how to recognize HTML tags static const QString formTagEnd = QStringLiteral("</form>"); static const QString inputTagBegin = QStringLiteral("<input "); static const QString selectTagBegin = QStringLiteral("<select "); static const QString selectTagEnd = QStringLiteral("</select>"); static const QString optionTagBegin = QStringLiteral("<option "); /// initialize result map QMap<QString, QString> result; /// determined boundaries of (only) "form" tag int endPos = htmlText.indexOf(formTagEnd, startPos, Qt::CaseInsensitive); if (startPos < 0 || endPos < 0) { qCWarning(LOG_KBIBTEX_NETWORKING) << "Could not locate form in text"; return result; } /// search for "input" tags within form int p = htmlText.indexOf(inputTagBegin, startPos, Qt::CaseInsensitive); while (p > startPos && p < endPos) { /// get "type", "name", and "value" attributes const QString inputType = htmlAttribute(htmlText, p, QStringLiteral("type")).toLower(); const QString inputName = htmlAttribute(htmlText, p, QStringLiteral("name")); const QString inputValue = htmlAttribute(htmlText, p, QStringLiteral("value")); if (!inputName.isEmpty()) { /// get value of input types if (inputType == QStringLiteral("hidden") || inputType == QStringLiteral("text") || inputType == QStringLiteral("submit")) result[inputName] = inputValue; else if (inputType == QStringLiteral("radio")) { /// must be selected if (htmlAttributeIsSelected(htmlText, p, QStringLiteral("checked"))) { result[inputName] = inputValue; } } else if (inputType == QStringLiteral("checkbox")) { /// must be checked if (htmlAttributeIsSelected(htmlText, p, QStringLiteral("checked"))) { /// multiple checkbox values with the same name are possible result.insertMulti(inputName, inputValue); } } } /// ignore input type "image" p = htmlText.indexOf(inputTagBegin, p + 1, Qt::CaseInsensitive); } /// search for "select" tags within form p = htmlText.indexOf(selectTagBegin, startPos, Qt::CaseInsensitive); while (p > startPos && p < endPos) { /// get "name" attribute from "select" tag const QString selectName = htmlAttribute(htmlText, p, QStringLiteral("name")); /// "select" tag contains one or several "option" tags, search all int popt = htmlText.indexOf(optionTagBegin, p, Qt::CaseInsensitive); int endSelect = htmlText.indexOf(selectTagEnd, p, Qt::CaseInsensitive); while (popt > p && popt < endSelect) { /// get "value" attribute from "option" tag const QString optionValue = htmlAttribute(htmlText, popt, QStringLiteral("value")); if (!selectName.isEmpty() && !optionValue.isEmpty()) { /// if this "option" tag is "selected", store value if (htmlAttributeIsSelected(htmlText, popt, QStringLiteral("selected"))) { result[selectName] = optionValue; } } popt = htmlText.indexOf(optionTagBegin, popt + 1, Qt::CaseInsensitive); } p = htmlText.indexOf(selectTagBegin, p + 1, Qt::CaseInsensitive); } return result; } #ifdef HAVE_QTWIDGETS void OnlineSearchAbstract::iconDownloadFinished() { QNetworkReply *reply = static_cast<QNetworkReply *>(sender()); if (reply->error() == QNetworkReply::NoError) { const QUrl redirUrl = reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl(); if (redirUrl.isValid()) { QNetworkRequest request(redirUrl); QNetworkReply *newReply = InternalNetworkAccessManager::instance().get(request); newReply->setObjectName(reply->objectName()); QListWidgetItem *listWidgetItem = m_iconReplyToListWidgetItem.value(reply, nullptr); m_iconReplyToListWidgetItem.remove(reply); if (listWidgetItem != nullptr) m_iconReplyToListWidgetItem.insert(newReply, listWidgetItem); connect(newReply, &QNetworkReply::finished, this, &OnlineSearchAbstract::iconDownloadFinished); return; } const QByteArray iconData = reply->readAll(); if (iconData.size() < 10) { /// Unlikely that an icon's data is less than 10 bytes, /// must be an error. qCWarning(LOG_KBIBTEX_NETWORKING) << "Received invalid icon data from " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); return; } QString extension; if (iconData[1] == 'P' && iconData[2] == 'N' && iconData[3] == 'G') { /// PNG files have string "PNG" at second to fourth byte extension = QStringLiteral(".png"); } else if (iconData[0] == static_cast<char>(0x00) && iconData[1] == static_cast<char>(0x00) && iconData[2] == static_cast<char>(0x01) && iconData[3] == static_cast<char>(0x00)) { /// Microsoft Icon have first two bytes always 0x0000, /// third and fourth byte is 0x0001 (for .ico) extension = QStringLiteral(".ico"); } else if (iconData[0] == '<') { /// HTML or XML code const QString htmlCode = QString::fromUtf8(iconData); qCDebug(LOG_KBIBTEX_NETWORKING) << "Received XML or HTML data from " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << ": " << htmlCode.left(128); return; } else { qCWarning(LOG_KBIBTEX_NETWORKING) << "Favicon is of unknown format: " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString(); return; } const QString filename = reply->objectName() + extension; QFile iconFile(filename); if (iconFile.open(QFile::WriteOnly)) { iconFile.write(iconData); iconFile.close(); QListWidgetItem *listWidgetItem = m_iconReplyToListWidgetItem.value(reply, nullptr); if (listWidgetItem != nullptr) { /// Disable signals while updating the widget and its items blockSignals(true); listWidgetItem->setIcon(QIcon(filename)); /// Re-enable signals after updating the widget and its items blockSignals(false); } } else { qCWarning(LOG_KBIBTEX_NETWORKING) << "Could not save icon data from URL" << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << "to file" << filename; return; } } else qCWarning(LOG_KBIBTEX_NETWORKING) << "Could not download icon from URL " << InternalNetworkAccessManager::removeApiKey(reply->url()).toDisplayString() << ": " << reply->errorString(); } #endif // HAVE_QTWIDGETS void OnlineSearchAbstract::dumpToFile(const QString &filename, const QString &text) { const QString usedFilename = QDir::tempPath() + QLatin1Char('/') + filename; QFile f(usedFilename); if (f.open(QFile::WriteOnly)) { qCDebug(LOG_KBIBTEX_NETWORKING) << "Dumping text" << KBibTeX::squeezeText(text, 96) << "to" << usedFilename; f.write(text.toUtf8()); f.close(); } } void OnlineSearchAbstract::delayedStoppedSearch(int returnCode) { m_delayedStoppedSearchReturnCode = returnCode; QTimer::singleShot(500, this, &OnlineSearchAbstract::delayedStoppedSearchTimer); } void OnlineSearchAbstract::delayedStoppedSearchTimer() { stopSearch(m_delayedStoppedSearchReturnCode); } void OnlineSearchAbstract::sanitizeEntry(QSharedPointer<Entry> entry) { if (entry.isNull()) return; /// Sometimes, there is no identifier, so set a random one if (entry->id().isEmpty()) entry->setId(QString(QStringLiteral("entry-%1")).arg(QString::number(qrand(), 36))); /// Missing entry type? Set it to 'misc' if (entry->type().isEmpty()) entry->setType(Entry::etMisc); static const QString ftIssue = QStringLiteral("issue"); if (entry->contains(ftIssue)) { /// ACM's Digital Library uses "issue" instead of "number" -> fix that Value v = entry->value(ftIssue); entry->remove(ftIssue); entry->insert(Entry::ftNumber, v); } /// If entry contains a description field but no abstract, /// rename description field to abstract static const QString ftDescription = QStringLiteral("description"); if (!entry->contains(Entry::ftAbstract) && entry->contains(ftDescription)) { Value v = entry->value(ftDescription); entry->remove(ftDescription); entry->insert(Entry::ftAbstract, v); } /// Remove "dblp" artifacts in abstracts and keywords if (entry->contains(Entry::ftAbstract)) { const QString abstract = PlainTextValue::text(entry->value(Entry::ftAbstract)); if (abstract == QStringLiteral("dblp")) entry->remove(Entry::ftAbstract); } if (entry->contains(Entry::ftKeywords)) { const QString keywords = PlainTextValue::text(entry->value(Entry::ftKeywords)); if (keywords == QStringLiteral("dblp")) entry->remove(Entry::ftKeywords); } if (entry->contains(Entry::ftMonth)) { /// Fix strings for months: "September" -> "sep" Value monthValue = entry->value(Entry::ftMonth); bool updated = false; for (Value::Iterator it = monthValue.begin(); it != monthValue.end(); ++it) { const QString valueItem = PlainTextValue::text(*it); static const QRegularExpression longMonth = QRegularExpression(QStringLiteral("(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*"), QRegularExpression::CaseInsensitiveOption); const QRegularExpressionMatch longMonthMatch = longMonth.match(valueItem); if (longMonthMatch.hasMatch()) { it = monthValue.erase(it); it = monthValue.insert(it, QSharedPointer<MacroKey>(new MacroKey(longMonthMatch.captured(1).toLower()))); updated = true; } } if (updated) entry->insert(Entry::ftMonth, monthValue); } if (entry->contains(Entry::ftDOI) && entry->contains(Entry::ftUrl)) { /// Remove URL from entry if contains a DOI and the DOI field matches the DOI in the URL const Value &doiValue = entry->value(Entry::ftDOI); for (const auto &doiValueItem : doiValue) { const QString doi = PlainTextValue::text(doiValueItem); Value v = entry->value(Entry::ftUrl); bool gotChanged = false; for (Value::Iterator it = v.begin(); it != v.end();) { const QSharedPointer<ValueItem> &vi = (*it); if (vi->containsPattern(QStringLiteral("/") + doi)) { it = v.erase(it); gotChanged = true; } else ++it; } if (v.isEmpty()) entry->remove(Entry::ftUrl); else if (gotChanged) entry->insert(Entry::ftUrl, v); } } else if (!entry->contains(Entry::ftDOI) && entry->contains(Entry::ftUrl)) { /// If URL looks like a DOI, remove URL and add a DOI field QSet<QString> doiSet; ///< using a QSet here to keep only unique DOIs Value v = entry->value(Entry::ftUrl); bool gotChanged = false; for (Value::Iterator it = v.begin(); it != v.end();) { const QString viText = PlainTextValue::text(*it); const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(viText); if (doiRegExpMatch.hasMatch()) { doiSet.insert(doiRegExpMatch.captured()); it = v.erase(it); gotChanged = true; } else ++it; } if (v.isEmpty()) entry->remove(Entry::ftUrl); else if (gotChanged) entry->insert(Entry::ftUrl, v); if (!doiSet.isEmpty()) { Value doiValue; /// Rewriting QSet<QString> doiSet into a (sorted) list for reproducibility /// (required for automated test in KBibTeXNetworkingTest) QStringList list; for (const QString &doi : doiSet) list.append(doi); list.sort(); for (const QString &doi : const_cast<const QStringList &>(list)) doiValue.append(QSharedPointer<PlainText>(new PlainText(doi))); entry->insert(Entry::ftDOI, doiValue); } } else if (!entry->contains(Entry::ftDOI)) { const QRegularExpressionMatch doiRegExpMatch = KBibTeX::doiRegExp.match(entry->id()); if (doiRegExpMatch.hasMatch()) { /// If entry id looks like a DOI, add a DOI field Value doiValue; doiValue.append(QSharedPointer<PlainText>(new PlainText(doiRegExpMatch.captured()))); entry->insert(Entry::ftDOI, doiValue); } } /// Referenced strings or entries do not exist in the search result /// and BibTeX breaks if it finds a reference to a non-existing string or entry entry->remove(Entry::ftCrossRef); } bool OnlineSearchAbstract::publishEntry(QSharedPointer<Entry> entry) { if (entry.isNull()) return false; Value v; v.append(QSharedPointer<PlainText>(new PlainText(label()))); entry->insert(QStringLiteral("x-fetchedfrom"), v); sanitizeEntry(entry); emit foundEntry(entry); return true; } void OnlineSearchAbstract::stopSearch(int errorCode) { if (errorCode == resultNoError) curStep = numSteps; else curStep = numSteps = 0; emit progress(curStep, numSteps); emit stoppedSearch(errorCode); } void OnlineSearchAbstract::refreshBusyProperty() { const bool newBusyState = busy(); if (newBusyState != m_previousBusyState) { m_previousBusyState = newBusyState; emit busyChanged(); } } #ifdef HAVE_QTWIDGETS OnlineSearchAbstract::Form::Form(QWidget *parent) : QWidget(parent), d(new OnlineSearchAbstract::Form::Private()) { /// nothing } OnlineSearchAbstract::Form::~Form() { delete d; } #endif // HAVE_QTWIDGETS